From 935ccc075d3c5f6567719cf2d90da5c215528513 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Wed, 28 Oct 2026 14:45:26 -0400 Subject: [PATCH 01/41] Fix benchmark health-check starvation and capture server diagnostics --- .github/workflows/llm-benchmark-periodic.yml | 10 ++ .../src/bench/publishers.rs | 99 +++++++++++++-- tools/xtask-llm-benchmark/src/bench/runner.rs | 117 ++++++++++++++++-- .../answers/csharp.cs | 2 +- .../tasks/csharp.txt | 2 +- .../t_043_role_based_access/answers/csharp.cs | 2 +- .../t_043_role_based_access/tasks/csharp.txt | 2 +- .../src/bin/llm_benchmark.rs | 96 +++++++++++++- tools/xtask-llm-benchmark/src/eval/scorers.rs | 2 +- 9 files changed, 301 insertions(+), 31 deletions(-) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 93f20145482..0533b7fac73 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -184,6 +184,7 @@ jobs: LLM_BENCH_CSHARP_CONCURRENCY: "2" LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || matrix.lang == 'csharp' && '1' || '2' }} LLM_BENCHMARK_REPORT_DIR: ${{ runner.temp }}/llm-benchmark-reports + LLM_BENCHMARK_DIAGNOSTICS_DIR: ${{ runner.temp }}/llm-benchmark-diagnostics INPUT_LANGUAGE: ${{ matrix.lang }} INPUT_MODEL_SET: ${{ inputs.model_set || 'website_active' }} INPUT_MODELS: ${{ inputs.models || '' }} @@ -270,6 +271,15 @@ jobs: if-no-files-found: ignore retention-days: 14 + - name: Upload server diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: llm-benchmark-diagnostics-${{ matrix.lang }} + path: ${{ runner.temp }}/llm-benchmark-diagnostics + if-no-files-found: ignore + retention-days: 14 + summarize: name: Summarize benchmark analysis if: always() diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index ad6131a54bb..94eb998f51e 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -217,7 +217,7 @@ pub struct PublishedDatabase { db: String, // Keep the publishing CLI session alive so deletion uses the database // owner's credentials without sharing mutable CLI state across routes. - cli_root: CliRootDir, + cli_root: Option, deleted: bool, } @@ -227,13 +227,13 @@ impl PublishedDatabase { Self { host_url: host_url.to_owned(), db, - cli_root, + cli_root: Some(cli_root), deleted: false, } } fn delete_inner(&self) -> Result<()> { - let mut cmd = spacetime_cmd(&self.cli_root); + let mut cmd = spacetime_cmd(self.cli_root.as_ref().expect("database owns CLI session")); cmd.arg("delete") .arg("-y") .arg("--no-config") @@ -251,8 +251,9 @@ impl PublishedDatabase { pub fn delete(mut self) -> Result<()> { self.delete_inner()?; self.deleted = true; - self.cli_root.remove()?; - self.cli_root.relinquish_cleanup(); + let cli_root = self.cli_root.as_mut().expect("database owns CLI session"); + cli_root.remove()?; + cli_root.relinquish_cleanup(); Ok(()) } @@ -260,21 +261,93 @@ impl PublishedDatabase { /// has been created by a migration republish. pub fn relinquish(mut self) { self.deleted = true; - self.cli_root.relinquish_cleanup(); + self.cli_root + .as_mut() + .expect("database owns CLI session") + .relinquish_cleanup(); } } impl Drop for PublishedDatabase { fn drop(&mut self) { - if !self.deleted - && let Err(error) = self.delete_inner() - { - eprintln!( - "[cleanup] failed to delete benchmark database `{}` during fallback cleanup: {error:#}", - self.db - ); + if !self.deleted { + let database = Self { + host_url: std::mem::take(&mut self.host_url), + db: std::mem::take(&mut self.db), + cli_root: self.cli_root.take(), + // Do not recursively retry if fallback deletion fails. + deleted: true, + }; + run_fallback_cleanup(move || { + if let Err(error) = database.delete_inner() { + eprintln!( + "[cleanup] failed to delete benchmark database `{}` during fallback cleanup: {error:#}", + database.db + ); + } + drop(database); + }); + } + } +} + +fn run_fallback_cleanup(cleanup: impl FnOnce() + Send + 'static) { + // If shutdown rejects the blocking task, dropping its closure must still clean up. + struct Cleanup(Option); + impl Drop for Cleanup { + fn drop(&mut self) { + if let Some(cleanup) = self.0.take() { + cleanup(); + } } } + let cleanup = Cleanup(Some(cleanup)); + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + // Runtime shutdown waits for blocking work, including cancelled-task cleanup. + runtime.spawn_blocking(move || drop(cleanup)); + } else { + drop(cleanup); + } +} + +#[cfg(test)] +mod fallback_cleanup_tests { + use super::*; + + #[test] + fn cancellation_cleanup_does_not_block_and_shutdown_waits_for_it() { + let runtime = tokio::runtime::Builder::new_current_thread().build().unwrap(); + let (release, wait) = std::sync::mpsc::channel(); + let (finished, completion) = std::sync::mpsc::channel(); + runtime.block_on(async { + run_fallback_cleanup(move || { + finished + .send(wait.recv_timeout(std::time::Duration::from_secs(5)).is_ok()) + .unwrap(); + }); + let _ = release.send(()); + }); + drop(runtime); + assert!(completion.try_recv().unwrap(), "cleanup blocked its async caller"); + } + + #[test] + fn cleanup_without_a_runtime_finishes_inline() { + let (finished, completion) = std::sync::mpsc::channel(); + run_fallback_cleanup(move || finished.send(()).unwrap()); + completion.try_recv().unwrap(); + } + + #[test] + fn cleanup_is_not_lost_when_runtime_has_shut_down() { + let runtime = tokio::runtime::Builder::new_current_thread().build().unwrap(); + let handle = runtime.handle().clone(); + drop(runtime); + let _entered = handle.enter(); + let (finished, completion) = std::sync::mpsc::channel(); + run_fallback_cleanup(move || finished.send(()).unwrap()); + completion.recv_timeout(std::time::Duration::from_secs(5)).unwrap(); + } } pub trait Publisher: Send + Sync { diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index e430a132501..324169c0155 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -18,7 +18,7 @@ use crate::bench::utils::{ }; use crate::bench::Publisher; use crate::eval::scorers::call_reducer_json_out; -use crate::eval::{Lang, ScoreDetails}; +use crate::eval::{Lang, ScoreDetails, Scorer}; use crate::generated::resolve_by_path; use crate::llm::model_routes::ModelRoute; @@ -107,6 +107,20 @@ async fn delete_database_async(database: PublishedDatabase) -> Result<()> { Ok(()) } +async fn score_async( + scorers: Vec>, + output: String, + database: Option, +) -> Result<(Vec<(&'static str, ScoreDetails)>, String, Option)> { + task::spawn_blocking(move || { + // Keep the database alive if the awaiting task is cancelled while scoring. + let scores = scorers.iter().map(|s| (s.id(), s.score(&output))).collect(); + (scores, output, database) + }) + .await + .context("scoring worker failed") +} + pub async fn delete_databases(databases: Vec) -> Result<()> { stream::iter(databases.into_iter().map(delete_database_async)) .buffer_unordered(8) @@ -307,7 +321,7 @@ impl TaskRunner { let generation_duration_ms = Some(gen_start.elapsed().as_millis() as u64); let input_tokens = llm_result.input_tokens; let output_tokens = llm_result.output_tokens; - let llm_output = llm_result.text; + let mut llm_output = llm_result.text; let migration_setup = load_migration_setup_source(task, cfg.lang)?; @@ -343,7 +357,7 @@ impl TaskRunner { Ok(None) }; - let (published_database, publish_error) = match setup_database { + let (mut published_database, publish_error) = match setup_database { Err(error) => (None, Some(format!("migration setup failed: {error:#}"))), Ok(mut setup_database) => { let publish_result = self @@ -391,13 +405,21 @@ impl TaskRunner { if publish_error.is_none() { println!("→ [{}] {}: scoring", cfg.lang_name, cfg.route.display_name); - for s in &scorers { - let r = s.score(&llm_output); + let (scores, output, database) = + score_async(scorers, llm_output, published_database) + .await + .map_err(|error| RunOneError::Infrastructure { + msg: format!("scoring worker failed: {error:#}"), + llm_output: None, + })?; + llm_output = output; + published_database = database; + for (id, r) in scores { if r.pass { passed += 1; } partial_sum += r.partial.clamp(0.0, 1.0); - scorer_details.insert(s.id().to_string(), r); + scorer_details.insert(id.to_string(), r); } } else { println!( @@ -1207,19 +1229,24 @@ pub async fn validate_goldens_for_lang( let ctor = resolve_by_path(&task.root)?; let spec = ctor(); - for scorer in spec.scorers_for(lang, route_tag, host_url) { - let details = scorer.score(&golden_src_text); + let (scores, _, candidate_database) = score_async( + spec.scorers_for(lang, route_tag, host_url), + golden_src_text, + Some(candidate_database), + ) + .await?; + for (id, details) in scores { if !details.pass { failures.push(format!( "{}/{} [{}]: {}", category, task_id, - scorer.id(), + id, serde_json::to_string(&details.notes).unwrap_or_else(|_| "".to_string()) )); } } - delete_database_async(candidate_database).await?; + delete_database_async(candidate_database.expect("candidate database retained during scoring")).await?; } delete_databases(golden_databases).await?; @@ -1381,6 +1408,76 @@ fn normalize_task_selector(raw: &str) -> Result { mod build_cleanup_tests { use super::*; + #[test] + fn slow_scoring_does_not_block_health_checks() { + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::mpsc; + + struct SlowScorer(mpsc::Receiver<()>); + impl Scorer for SlowScorer { + fn id(&self) -> &'static str { + "slow" + } + fn score(&self, output: &str) -> ScoreDetails { + assert_eq!(output, "candidate"); + ScoreDetails { + pass: self.0.recv_timeout(Duration::from_secs(8)).is_ok(), + partial: 0.5, + notes: json!({"preserved": true}), + } + } + } + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let host = format!("http://{}", listener.local_addr().unwrap()); + listener.set_nonblocking(true).unwrap(); + let server = std::thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + match listener.accept() { + Ok((mut socket, _)) => { + socket.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut request = [0; 4096]; + let n = socket.read(&mut request).unwrap(); + assert!(String::from_utf8_lossy(&request[..n]).starts_with("GET /v1/ping ")); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .unwrap(); + break; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + assert!(Instant::now() < deadline, "health check never reached mock server"); + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("mock server: {error}"), + } + } + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (release, wait) = mpsc::channel(); + let (scores, output, database) = runtime.block_on(async { + // Poll both futures in one task, as buffer_unordered does in the runner. + let scoring = score_async(vec![Box::new(SlowScorer(wait))], "candidate".into(), None); + let probe = async { + ensure_server_healthy(Some(&host)).await.unwrap(); + let _ = release.send(()); + }; + let (scores, ()) = futures::join!(scoring, probe); + scores.unwrap() + }); + server.join().unwrap(); + assert_eq!(output, "candidate"); + assert!(database.is_none()); + assert_eq!(scores[0].0, "slow"); + assert!(scores[0].1.pass, "scoring blocked the health check until its deadline"); + assert_eq!(scores[0].1.partial, 0.5); + assert_eq!(scores[0].1.notes, json!({"preserved": true})); + } + #[test] fn preserves_sources_and_shared_files_after_success_and_failure() { let root = std::env::temp_dir().join(format!("llm-build-cleanup-{}", std::process::id())); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/csharp.cs index 498eb336e7e..69aec852c85 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/answers/csharp.cs @@ -22,7 +22,7 @@ public partial struct Message } [Reducer] - public static void Register(ReducerContext ctx, string name) + public static void RegisterUser(ReducerContext ctx, string name) { if (ctx.Db.User.Identity.Find(ctx.Sender) is not null) { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt index 142af581a0f..2f2d5a38577 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt @@ -17,7 +17,7 @@ TABLES - Text: string REDUCERS -- Register(ctx, string name) +- RegisterUser(ctx, string name) - If a User row already exists for ctx.Sender, throw exception with "already registered" - Otherwise insert a new User with Identity = ctx.Sender and the given Name diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/csharp.cs index 346dc0a241a..0ff62a80629 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/csharp.cs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/answers/csharp.cs @@ -11,7 +11,7 @@ public partial struct User } [Reducer] - public static void Register(ReducerContext ctx) + public static void RegisterUser(ReducerContext ctx) { if (ctx.Db.User.Identity.Find(ctx.Sender) is not null) { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt index d0efc9dc564..67df07c65b8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt @@ -9,7 +9,7 @@ TABLES - Role: string ("admin" or "member") REDUCERS -- Register(ctx) +- RegisterUser(ctx) - If a User row already exists for ctx.Sender, throw exception with "already registered" - Insert User { Identity = ctx.Sender, Role = "member" } diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index a892a9a1d66..5fa14b49b79 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -298,7 +298,11 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { eprintln!("[warn] failed to upload task catalog: {e}"); } - let RuntimeInit { runtime, mut guard } = initialize_runtime(config.hash_only)?; + let initialized = initialize_runtime(config.hash_only)?; + let mut guard = initialized.guard; + // Drop the runtime first: finish blocking scoring/cleanup before stopping the server. + let runtime = initialized.runtime; + save_server_diagnostics(guard.as_mut(), "start"); config.host = guard.as_ref().map(|g| g.host_url.clone()); @@ -314,12 +318,14 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { if config.goldens_only { let rt = runtime.as_ref().expect("runtime required for --goldens-only"); - rt.block_on(validate_goldens_for_lang( + let result = rt.block_on(validate_goldens_for_lang( config.host.clone(), &bench_root, config.lang, selectors_ref, - ))?; + )); + save_server_diagnostics(guard.as_mut(), "end"); + result?; println!("[{}] goldens-only validation complete", config.lang.as_str()); return Ok(()); } @@ -355,6 +361,8 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { all_outcomes.extend(outcomes); } + save_server_diagnostics(guard.as_mut(), "end"); + // Write local run log on --dry-run so results aren't lost if dry_run && !all_outcomes.is_empty() { let runs_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("runs"); @@ -380,6 +388,7 @@ fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { let Some(guard) = guard else { return; }; + save_server_diagnostics(Some(guard), "failure"); match guard.child.try_wait() { Ok(Some(status)) => eprintln!("[server] local SpacetimeDB exited unexpectedly: {status}"), Ok(None) => eprintln!("[server] local SpacetimeDB is still running after benchmark failure"), @@ -387,6 +396,77 @@ fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { } } +// Capture before Drop kills the server, so cleanup cannot be mistaken for the cause. +fn save_server_diagnostics(guard: Option<&mut SpacetimeDbGuard>, phase: &str) { + let (Some(guard), Some(dir)) = (guard, std::env::var_os("LLM_BENCHMARK_DIAGNOSTICS_DIR")) else { + return; + }; + let result = (|| -> Result<()> { + let dir = PathBuf::from(dir); + fs::create_dir_all(&dir)?; + let pid = guard.child.id(); + let mut resources = format!( + "time={:?}\npid={pid}\nstatus={:?}\n", + std::time::SystemTime::now(), + guard.child.try_wait() + ); + for path in [ + "/proc/meminfo".to_owned(), + "/proc/pressure/memory".to_owned(), + "/proc/pressure/cpu".to_owned(), + "/proc/pressure/io".to_owned(), + format!("/proc/{pid}/status"), + format!("/proc/{pid}/stat"), + format!("/proc/{pid}/io"), + format!("/proc/{pid}/limits"), + format!("/proc/{pid}/cgroup"), + ] { + append_resource(&mut resources, Path::new(&path)); + } + // On cgroup v2, collect the server's group and its parents (limits can be inherited). + // The child may already be dead. The harness shares its cgroup in CI. + if let Ok(groups) = + fs::read_to_string(format!("/proc/{pid}/cgroup")).or_else(|_| fs::read_to_string("/proc/self/cgroup")) + && let Some(group) = groups.lines().find_map(|line| line.strip_prefix("0::/")) + { + let root = Path::new("/sys/fs/cgroup"); + let mut path = root.join(group); + while path.starts_with(root) { + for file in [ + "memory.current", + "memory.peak", + "memory.max", + "memory.events", + "cpu.stat", + ] { + append_resource(&mut resources, &path.join(file)); + } + if !path.pop() { + break; + } + } + } + fs::write(dir.join(format!("resources-{phase}.txt")), resources)?; + let logs = guard + .logs + .lock() + .map_err(|_| anyhow::anyhow!("server log lock poisoned"))?; + fs::write(dir.join("server.log"), logs.as_bytes())?; + Ok(()) + })(); + if let Err(error) = result { + eprintln!("[warn] failed to save server diagnostics: {error:#}"); + } +} + +fn append_resource(output: &mut String, path: &Path) { + output.push_str(&format!("\n--- {} ---\n", path.display())); + match fs::read_to_string(path) { + Ok(contents) => output.push_str(&contents), + Err(error) => output.push_str(&format!("unavailable: {error}\n")), + } +} + /* ------------------------------ analyze ------------------------------ */ fn cmd_analyze(args: AnalyzeArgs, reasoning: ReasoningEffort) -> Result<()> { @@ -962,6 +1042,16 @@ fn apply_category_filter( mod tests { use super::*; + #[test] + fn resource_diagnostics_include_contents_and_missing_file_errors() { + let mut output = String::new(); + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")).join("Cargo.toml"); + append_resource(&mut output, &manifest); + assert!(output.contains("[package]")); + append_resource(&mut output, &manifest.join("missing")); + assert!(output.contains("unavailable:")); + } + fn base_run_args() -> RunArgs { RunArgs { modes: None, diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index bfd9b7feec1..d1533d7b448 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -7,7 +7,7 @@ use std::path::Path; use std::time::{Duration, Instant}; use std::{io, thread}; -pub trait Scorer { +pub trait Scorer: Send { fn id(&self) -> &'static str; fn score(&self, llm_output: &str) -> ScoreDetails; } From e304d0fe4a3d82e21715e170b97dbf04535a923f Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Wed, 28 Oct 2026 17:34:30 -0400 Subject: [PATCH 02/41] Reject benchmark uploads with unresolved provider failures --- tools/xtask-llm-benchmark/src/bench/runner.rs | 95 ++++++++++++------- 1 file changed, 59 insertions(+), 36 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 324169c0155..7536c0eced4 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -526,7 +526,7 @@ fn partition_results( route, hash, anyhow::anyhow!(msg), - Some(llm_output), + llm_output, )); } Err(RunOneError::Infrastructure { msg, llm_output }) => { @@ -547,16 +547,15 @@ fn partition_results( Ok((good, retry)) } -fn pending_retries_to_outcomes( - pending: Vec, - lang_name: &str, - route: &ModelRoute, - hash: &str, -) -> Vec { - pending - .into_iter() - .map(|pending| build_fail_outcome(&pending.task, lang_name, route, hash, pending.error, None)) - .collect() +fn finalize_batch(outcomes: Vec, pending: Vec) -> Result> { + // A partial batch would change the score denominator. Reject it, not just the failed tasks. + if let Some(first) = pending.first() { + bail!( + "{} tasks still have provider/setup errors after retries; refusing to upload the entire batch. First failure at {}: {:#}", + pending.len(), first.task.root.display(), first.error + ); + } + Ok(outcomes) } fn dry_run_analysis_path(run_id: &str, lang_name: &str, mode: &str, route: &ModelRoute) -> PathBuf { @@ -764,14 +763,7 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu retry_tasks = still_failing; } - let dropped = retry_tasks.len(); - if dropped > 0 { - eprintln!( - "[runner] {} tasks still failing after retries — recording as provider failures", - dropped - ); - outcomes.extend(pending_retries_to_outcomes(retry_tasks, lang_name, cfg.route, cfg.hash)); - } + let outcomes = finalize_batch(outcomes, retry_tasks)?; println!("[runner] completed batch: {} results", outcomes.len()); @@ -886,8 +878,7 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> let (mut outcomes, retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; // Retry provider-error tasks until all pass or none make progress - let dropped; - { + let pending = { const MAX_RETRY_ROUNDS: usize = 3; let mut pending = retry_tasks; for round in 1..=MAX_RETRY_ROUNDS { @@ -938,7 +929,7 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> if new_outcomes.is_empty() && !still_failing.is_empty() { eprintln!( - "[runner] no tasks recovered in retry round {} — provider may be down, recording {} failures", + "[runner] no tasks recovered in retry round {} — provider may be down, keeping {} failures", round, still_failing.len() ); @@ -949,18 +940,9 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> outcomes.extend(new_outcomes); pending = still_failing; } - dropped = pending.len(); - if dropped > 0 { - outcomes.extend(pending_retries_to_outcomes(pending, lang_name, cfg.route, cfg.hash)); - } - } - - if dropped > 0 { - eprintln!( - "[runner] {} tasks still failing after retries — recording as provider failures", - dropped - ); - } + pending + }; + let outcomes = finalize_batch(outcomes, pending)?; ensure_server_healthy(cfg.host.as_deref()).await?; @@ -1285,7 +1267,7 @@ fn build_fail_outcome( route: &ModelRoute, hash: &str, err: anyhow::Error, - llm_output: Option, + llm_output: String, ) -> RunOutcome { let category = category_slug(&task.root); let task_id = task_slug(&task.root); @@ -1314,7 +1296,7 @@ fn build_fail_outcome( total_tests: 1, passed_tests: 0, - llm_output, + llm_output: Some(llm_output), route_api_model: Some(route.api_model.to_string()), golden_db: None, @@ -1408,6 +1390,47 @@ fn normalize_task_selector(raw: &str) -> Result { mod build_cleanup_tests { use super::*; + #[test] + fn provider_failures_reject_the_whole_batch_but_model_failures_remain_scores() { + let task = || TaskPaths { + root: PathBuf::from("basics/t_001"), + answers_csharp: PathBuf::new(), + answers_rust: PathBuf::new(), + answers_typescript: PathBuf::new(), + }; + let route = ModelRoute::new("Test", crate::llm::types::Vendor::OpenAi, "test", None); + let (outcomes, pending) = partition_results( + vec![ + ( + task(), + Err(RunOneError::WithOutput { + msg: "compile failed".into(), + llm_output: "invalid code".into(), + }), + ), + (task(), Err(RunOneError::Other(anyhow::anyhow!("402 Payment Required")))), + ], + "rust", + &route, + "test", + ) + .unwrap(); + assert_eq!(outcomes.len(), 1); + assert_eq!(outcomes[0].passed_tests, 0); + assert!(finalize_batch(outcomes.clone(), Vec::new()).is_ok()); + let error = finalize_batch(outcomes, pending).unwrap_err().to_string(); + assert!(error.contains("refusing to upload"), "{error}"); + assert!(error.contains("402 Payment Required"), "{error}"); + assert!(finalize_batch( + Vec::new(), + vec![PendingRetry { + task: task(), + error: anyhow::anyhow!("provider unavailable"), + }] + ) + .is_err()); + } + #[test] fn slow_scoring_does_not_block_health_checks() { use std::io::{Read, Write}; From 3d9f7c3ccdd4e060a5f7bd215cecf8a2766435ce Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Wed, 28 Oct 2026 17:51:36 -0400 Subject: [PATCH 03/41] Limit benchmark model concurrency to reduce credit reservations --- .github/workflows/llm-benchmark-periodic.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 0533b7fac73..3d3807599c8 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -179,10 +179,11 @@ jobs: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1" MSBUILDDISABLENODEREUSE: "1" DOTNET_CLI_USE_MSBUILD_SERVER: "0" + # Bound in-flight provider credit reservations across the three language jobs. LLM_BENCH_CONCURRENCY: "8" LLM_BENCH_RUST_CONCURRENCY: "4" LLM_BENCH_CSHARP_CONCURRENCY: "2" - LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || matrix.lang == 'csharp' && '1' || '2' }} + LLM_BENCH_ROUTE_CONCURRENCY: "1" LLM_BENCHMARK_REPORT_DIR: ${{ runner.temp }}/llm-benchmark-reports LLM_BENCHMARK_DIAGNOSTICS_DIR: ${{ runner.temp }}/llm-benchmark-diagnostics INPUT_LANGUAGE: ${{ matrix.lang }} From ead2965aa4d1ee99c719f1101b59348182e974a9 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Thu, 29 Oct 2026 17:37:33 -0400 Subject: [PATCH 04/41] Handle nullable provider responses and continue independent benchmark models --- tools/xtask-llm-benchmark/README.md | 6 + .../src/bin/llm_benchmark.rs | 150 ++++++++++-------- .../src/llm/clients/deepseek.rs | 6 +- .../src/llm/clients/meta.rs | 4 +- .../src/llm/clients/oa_compat.rs | 54 ++++++- .../src/llm/clients/openrouter.rs | 112 ++++++++++++- 6 files changed, 247 insertions(+), 85 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index 06b94a54599..1943341d3c7 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -4,6 +4,12 @@ The normal suite contains 95 tasks. The 12 `advanced` tasks are included by defa in all three languages and in both context modes. They are also included in the normal website task catalog. `--categories advanced` is only a development filter. +Provider errors or missing response content reject the whole model/language/mode +batch after retries. No partial scores from that batch are uploaded. Other models +still run, but the language job exits with failure if any batch fails. Response +errors include the generation ID and finish reason when supplied by the provider. +Generated code that fails compilation or grading remains a normal benchmark score. + ## New behavioral tasks | ID | Contract checked | diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index 5fa14b49b79..d97e4e8bc4b 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -2,7 +2,7 @@ use anyhow::{Context, Result}; use clap::{Args, Parser, Subcommand, ValueEnum}; -use futures::{StreamExt, TryStreamExt}; +use futures::{FutureExt, StreamExt}; use spacetimedb_data_structures::map::{HashCollectionExt as _, HashMap, HashSet}; use spacetimedb_guard::SpacetimeDbGuard; use std::fs; @@ -331,16 +331,13 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { } let llm_provider = if !config.goldens_only && !config.hash_only { - let provider = make_provider_from_env(reasoning)?; - let rt = runtime.as_ref().expect("failed to initialize runtime for preflight"); - let routes = filter_routes(&config); - preflight_llm_routes(rt, provider.as_ref(), &routes, &modes)?; - Some(provider) + Some(make_provider_from_env(reasoning)?) } else { None }; let mut all_outcomes: Vec = Vec::new(); + let mut run_status = Ok(()); for mode in modes { let result = run_mode_benchmarks( @@ -351,14 +348,19 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { runtime.as_ref(), llm_provider.as_ref(), ); - let outcomes = match result { - Ok(outcomes) => outcomes, + match result { + Ok((outcomes, status)) => { + all_outcomes.extend(outcomes); + run_status = status; + } Err(error) => { - report_server_status(guard.as_mut()); - return Err(error); + run_status = Err(error); } - }; - all_outcomes.extend(outcomes); + } + if run_status.is_err() { + report_server_status(guard.as_mut()); + break; + } } save_server_diagnostics(guard.as_mut(), "end"); @@ -381,7 +383,7 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { } } - Ok(()) + run_status } fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { @@ -674,51 +676,6 @@ fn should_fetch_remote_routes(args: &RunArgs) -> bool { && !args.goldens_only } -fn preflight_llm_routes( - runtime: &Runtime, - llm_provider: &dyn LlmProvider, - routes: &[ModelRoute], - modes: &[String], -) -> Result<()> { - if routes.is_empty() { - return Ok(()); - } - - let mut search_flags = Vec::new(); - if modes.iter().any(|mode| mode == "search") { - search_flags.push(true); - } - if modes.iter().any(|mode| mode != "search") { - search_flags.push(false); - } - - let mut failures = Vec::new(); - for route in routes { - for search_enabled in &search_flags { - let mode_label = if *search_enabled { - "search/OpenRouter online" - } else { - "standard" - }; - - if let Err(err) = runtime.block_on(llm_provider.preflight_route(route, *search_enabled)) { - let msg = format!("{} ({mode_label}): {err:#}", route.display_name); - eprintln!("[preflight] FAILED {msg}"); - failures.push(msg); - } - } - } - - if !failures.is_empty() { - anyhow::bail!( - "LLM provider preflight failed before benchmark run:\n - {}", - failures.join("\n - ") - ); - } - - Ok(()) -} - /// Run benchmarks for a single mode. fn run_mode_benchmarks( mode: &str, @@ -727,7 +684,7 @@ fn run_mode_benchmarks( bench_root: &Path, runtime: Option<&Runtime>, llm_provider: Option<&Arc>, -) -> Result> { +) -> Result<(Vec, Result<()>)> { let lang_str = lang.as_str(); let context = build_context(mode, Some(lang))?; // Use processed context hash so each lang/mode combination has its own unique hash @@ -737,7 +694,7 @@ fn run_mode_benchmarks( println!("{:<12} [{:<10}] hash: {}", mode, lang_str, short_hash(&hash)); if config.hash_only { - return Ok(Vec::new()); + return Ok((Vec::new(), Ok(()))); } // Run benchmarks for all matching routes @@ -745,13 +702,13 @@ fn run_mode_benchmarks( if routes.is_empty() { println!("{:<12} [{:<10}] no matching models to run", mode, lang_str); - return Ok(Vec::new()); + return Ok((Vec::new(), Ok(()))); } let runtime = runtime.expect("runtime required for normal runs"); let llm_provider = llm_provider.expect("llm provider required for normal runs"); - let route_runs = runtime.block_on(run_many_routes_for_mode( + let (route_runs, status) = runtime.block_on(run_many_routes_for_mode( bench_root, mode, &hash, @@ -760,7 +717,7 @@ fn run_mode_benchmarks( config, llm_provider.as_ref(), &routes, - ))?; + )); // Print summary sorted by pass rate descending let mut summary: Vec<(&str, u32, u32, f32)> = route_runs @@ -782,7 +739,7 @@ fn run_mode_benchmarks( } let all_outcomes: Vec = route_runs.into_iter().flat_map(|rr| rr.outcomes).collect(); - Ok(all_outcomes) + Ok((all_outcomes, status)) } /// Routes to run: when `model_filter` is set (from --models), only routes whose vendor and @@ -848,7 +805,7 @@ async fn run_many_routes_for_mode( config: &RunConfig, llm: &dyn LlmProvider, routes: &[ModelRoute], -) -> Result> { +) -> (Vec, Result<()>) { let rbuf = bench_route_concurrency(); let selectors = config.selectors.as_deref(); let host = config.host.clone(); @@ -857,13 +814,16 @@ async fn run_many_routes_for_mode( let local_analysis = config.local_analysis; let dry_run_id = config.dry_run_id.clone(); - futures::stream::iter(routes.iter().map(|route| { + let runs = futures::stream::iter(routes.iter().map(|route| { let host = host.clone(); let api_client = api_client.clone(); let dry_run_id = dry_run_id.clone(); async move { println!("\u{2192} running {}", route.display_name); + llm.preflight_route(route, mode == "search") + .await + .context("LLM provider preflight failed before model batch")?; let golden_scope = xtask_llm_benchmark::bench::run_scope_tag(mode, route.vendor.slug(), &route.api_model); let golden_databases = @@ -902,10 +862,36 @@ async fn run_many_routes_for_mode( outcomes, }) } + .map(move |result| result.with_context(|| format!("model {}:{}", route.vendor.slug(), route.api_model))) })) - .buffer_unordered(rbuf) - .try_collect::>() - .await + .buffer_unordered(rbuf); + collect_model_runs(runs).await +} + +/// A rejected batch must not cancel independent models or make the job look successful. +async fn collect_model_runs(runs: impl futures::Stream>) -> (Vec, Result<()>) { + futures::pin_mut!(runs); + let mut completed = Vec::new(); + let mut failures = Vec::new(); + while let Some(result) = runs.next().await { + match result { + Ok(run) => completed.push(run), + Err(error) => { + eprintln!("[runner] model batch failed: {error:#}"); + failures.push(format!("{error:#}")); + } + } + } + let status = if failures.is_empty() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "{} model batches failed:\n{}", + failures.len(), + failures.join("\n") + )) + }; + (completed, status) } fn categories_to_set(v: Option>) -> Option> { @@ -1100,6 +1086,32 @@ mod tests { assert_eq!(overridden.reasoning, ReasoningEffort::High); } + #[tokio::test] + async fn failed_model_does_not_skip_later_models_or_report_success() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let completed = AtomicUsize::new(0); + let runs = futures::stream::iter(["broken", "healthy", "also-broken"].into_iter().map(|model| { + let completed = &completed; + async move { + completed.fetch_add(1, Ordering::SeqCst); + if model == "healthy" { + Ok(model) + } else { + anyhow::bail!("{model}: provider error") + } + } + })) + .buffer_unordered(1); + let (successful, status) = collect_model_runs(runs).await; + assert_eq!(successful, ["healthy"]); + let error = status.unwrap_err().to_string(); + assert_eq!(completed.load(Ordering::SeqCst), 3); + assert!(error.contains("broken") && error.contains("also-broken"), "{error}"); + let (valid, status) = collect_model_runs(futures::stream::iter([Ok("first"), Ok("second")])).await; + status.unwrap(); + assert_eq!(valid, ["first", "second"]); + } + #[test] fn task_catalog_upload_can_be_skipped() { let cli = Cli::try_parse_from(["llm", "run", "--hash-only", "--skip-task-catalog-upload"]).unwrap(); diff --git a/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs b/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs index ccf3164d856..b98ab5146d5 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result}; use serde::Serialize; use super::http::HttpClient; @@ -93,7 +93,9 @@ impl DeepSeekClient { let resp: OACompatResp = serde_json::from_str(&body).context("parse deepseek resp")?; let input_tokens = resp.usage.as_ref().and_then(|u| u.prompt_tokens); let output_tokens = resp.usage.as_ref().and_then(|u| u.completion_tokens); - let text = resp.first_text().ok_or_else(|| anyhow!("no content from DeepSeek"))?; + let text = resp + .first_text() + .with_context(|| format!("DeepSeek completion failed (model={model})"))?; Ok(LlmOutput { text, input_tokens, diff --git a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs index 9bf4d87df9d..4218c029d85 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result}; use serde::Serialize; use super::http::HttpClient; @@ -101,7 +101,7 @@ impl MetaLlamaClient { let output_tokens = resp.usage.as_ref().and_then(|u| u.completion_tokens); let text = resp .first_text() - .ok_or_else(|| anyhow!("no content from Meta/OpenRouter"))?; + .with_context(|| format!("Meta/OpenRouter completion failed (model={model})"))?; Ok(LlmOutput { text, input_tokens, diff --git a/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs b/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs index a199b3b3c0c..9b427d1a48c 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs @@ -1,10 +1,13 @@ //! Shared types for OpenAI-compatible chat completions responses. //! Used by DeepSeek, Meta/OpenRouter, and OpenRouter clients. +use anyhow::{anyhow, bail, Result}; use serde::Deserialize; #[derive(Debug, Deserialize)] pub struct OACompatResp { + pub id: Option, + pub provider: Option, #[serde(default)] pub choices: Vec, #[serde(default)] @@ -17,17 +20,21 @@ pub struct OACompatResp { #[derive(Debug, Deserialize)] pub struct OAError { - pub message: String, + pub code: Option, + pub message: Option, } #[derive(Debug, Deserialize)] pub struct Choice { - pub message: MsgOut, + pub message: Option, + pub finish_reason: Option, + pub error: Option, } #[derive(Debug, Deserialize)] pub struct MsgOut { - pub content: String, + pub content: Option, + pub refusal: Option, } #[derive(Debug, Deserialize)] @@ -39,7 +46,44 @@ pub struct UsageInfo { } impl OACompatResp { - pub fn first_text(self) -> Option { - self.choices.into_iter().next().map(|c| c.message.content) + pub fn first_text(self) -> Result { + let choice = self.choices.first(); + let finish = choice.and_then(|c| c.finish_reason.as_deref()); + // Keep diagnostic fields, not prompts, generated content, or raw response bodies. + let diagnostic = format!( + "id={}, provider={}, finish_reason={}, refusal={}", + diagnostic_field(self.id.as_deref()), + diagnostic_field(self.provider.as_deref()), + diagnostic_field(finish), + choice + .and_then(|c| c.message.as_ref()) + .is_some_and(|m| m.refusal.is_some()), + ); + if let Some(error) = self.error.as_ref().or_else(|| choice.and_then(|c| c.error.as_ref())) { + bail!( + "upstream error ({diagnostic}, code={}): {}", + diagnostic_field(error.code.as_ref().map(|v| v.to_string()).as_deref()), + diagnostic_field(error.message.as_deref()), + ); + } + if matches!(finish, Some("error" | "content_filter")) { + bail!("unsuccessful completion ({diagnostic})"); + } + self.choices + .into_iter() + .next() + .and_then(|c| c.message) + .and_then(|m| m.content) + .filter(|text| !text.trim().is_empty()) + .ok_or_else(|| anyhow!("no nonempty completion content ({diagnostic})")) } } + +fn diagnostic_field(value: Option<&str>) -> String { + value + .unwrap_or("unknown") + .chars() + .filter(|c| !c.is_control()) + .take(512) + .collect() +} diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs index c4e806666ee..20fb60cbdb8 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{bail, Context, Result}; use serde::{Deserialize, Serialize}; use std::env; @@ -240,16 +240,13 @@ impl OpenRouterClient { .await .with_context(|| format!("OpenRouter POST {}", url))?; - let resp: OACompatResp = serde_json::from_str(&body).context("parse OpenRouter response")?; - // Check for upstream provider errors returned by OpenRouter. - if let Some(err) = resp.error { - anyhow::bail!("OpenRouter upstream error (model={}): {}", model, err.message); - } + let resp: OACompatResp = + serde_json::from_str(&body).with_context(|| format!("parse OpenRouter response (model={model})"))?; let input_tokens = resp.usage.as_ref().and_then(|u| u.prompt_tokens); let output_tokens = resp.usage.as_ref().and_then(|u| u.completion_tokens); let text = resp .first_text() - .ok_or_else(|| anyhow!("no content from OpenRouter (model={})", model))?; + .with_context(|| format!("OpenRouter completion failed (model={model})"))?; Ok(LlmOutput { text, input_tokens, @@ -423,6 +420,107 @@ pub fn openrouter_ctx_limit_tokens(model: &str) -> usize { mod tests { use super::{parse_env_flag, parse_min_credits_threshold}; + #[tokio::test] + async fn response_errors_keep_diagnostics_and_never_become_generated_code() { + use super::*; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::time::Duration; + + let cases = [ + ( + r#"{"choices":[{"message":{"content":"valid code"}}],"usage":{"prompt_tokens":10,"completion_tokens":2}}"#, + None, + ), + ( + r#"{"id":"gen-null","provider":"test-provider","choices":[{"finish_reason":"length","message":{"content":null}}]}"#, + Some("gen-null"), + ), + ( + r#"{"id":"gen-error","choices":[{"finish_reason":"error","message":{"content":"partial code"},"error":{"code":503,"message":"upstream unavailable"}}]}"#, + Some("upstream unavailable"), + ), + ( + r#"{"id":"gen-top","error":{"code":429,"message":"rate limited"}}"#, + Some("rate limited"), + ), + ( + r#"{"id":"gen-empty","choices":[{"finish_reason":"stop","message":{"content":" "}}]}"#, + Some("gen-empty"), + ), + ( + r#"{"id":"gen-filter","choices":[{"finish_reason":"content_filter","message":{"content":null,"refusal":"refusal text must not be logged"}}]}"#, + Some("content_filter"), + ), + ]; + for (body, expected_error) in cases { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut socket = loop { + match listener.accept() { + Ok((socket, _)) => break socket, + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + assert!(std::time::Instant::now() < deadline, "mock request timed out"); + std::thread::sleep(Duration::from_millis(5)); + } + Err(e) => panic!("{e}"), + } + }; + socket.set_nonblocking(false).unwrap(); + socket.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut request = Vec::new(); + let mut buf = [0; 4096]; + loop { + let n = socket.read(&mut buf).unwrap(); + assert!(n > 0); + request.extend_from_slice(&buf[..n]); + if let Some(end) = request.windows(4).position(|w| w == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&request[..end]).to_ascii_lowercase(); + let length: usize = headers + .lines() + .find_map(|line| line.strip_prefix("content-length: ")) + .unwrap() + .parse() + .unwrap(); + if request.len() >= end + 4 + length { + break; + } + } + } + write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body).unwrap(); + }); + let client = OpenRouterClient::with_base(HttpClient::new().unwrap(), base, "mock-key".into()); + let prompt = BuiltPrompt { + system: None, + static_prefix: None, + segments: vec![Segment::new("user", "test")], + search_enabled: false, + }; + let result = client.generate("test/model", &prompt, ReasoningEffort::Medium).await; + server.join().unwrap(); + match expected_error { + Some(expected) => { + let error = format!("{:#}", result.expect_err("response must not be scored")); + assert!(error.contains(expected), "{error}"); + assert!(error.contains("test/model"), "{error}"); + assert!( + !error.contains("partial code") && !error.contains("refusal text"), + "{error}" + ); + } + None => { + let output = result.unwrap(); + assert_eq!(output.text, "valid code"); + assert_eq!(output.input_tokens, Some(10)); + assert_eq!(output.output_tokens, Some(2)); + } + } + } + } + #[test] fn openrouter_min_credits_overrides_global_threshold() { assert_eq!(parse_min_credits_threshold(Some("2.5"), Some("1.0")), 2.5); From 3d56bfa22a2e96d413b58092e16a912c02b88224 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Thu, 29 Oct 2026 17:41:05 -0400 Subject: [PATCH 05/41] Finish remaining benchmark modes after a batch failure --- .../src/bin/llm_benchmark.rs | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index d97e4e8bc4b..769371a79b0 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -337,7 +337,7 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { }; let mut all_outcomes: Vec = Vec::new(); - let mut run_status = Ok(()); + let mut mode_failures = Vec::new(); for mode in modes { let result = run_mode_benchmarks( @@ -348,18 +348,16 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { runtime.as_ref(), llm_provider.as_ref(), ); - match result { + let status = match result { Ok((outcomes, status)) => { all_outcomes.extend(outcomes); - run_status = status; - } - Err(error) => { - run_status = Err(error); + status } - } - if run_status.is_err() { + Err(error) => Err(error), + }; + if let Err(error) = status { report_server_status(guard.as_mut()); - break; + mode_failures.push(format!("{mode}: {error:#}")); } } @@ -383,7 +381,12 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { } } - run_status + anyhow::ensure!( + mode_failures.is_empty(), + "benchmark modes failed:\n{}", + mode_failures.join("\n") + ); + Ok(()) } fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { From 274f46f29bc32886bdac6c3a7a36e29c40c6bb96 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Thu, 29 Oct 2026 22:16:51 -0400 Subject: [PATCH 06/41] Isolate C# benchmark servers and handle startup failures --- crates/guard/src/lib.rs | 182 ++++++++++----- tools/xtask-llm-benchmark/src/bench/types.rs | 1 + .../t_074_fetch_and_store/tasks/csharp.txt | 2 + .../t_074_fetch_and_store/tasks/rust.txt | 2 + .../tasks/typescript.txt | 2 + .../tasks/csharp.txt | 2 + .../t_079_external_upload_flow/tasks/rust.txt | 2 + .../tasks/typescript.txt | 2 + .../src/bin/llm_benchmark.rs | 213 +++++++++++++++--- 9 files changed, 323 insertions(+), 85 deletions(-) diff --git a/crates/guard/src/lib.rs b/crates/guard/src/lib.rs index eead324c936..b0cbe101ce6 100644 --- a/crates/guard/src/lib.rs +++ b/crates/guard/src/lib.rs @@ -121,12 +121,20 @@ impl SpacetimeDbGuard { Self::spawn_in_temp_data_dir_with_pg_port(None) } + /// Start a temporary server, returning startup errors after cleaning up the child. + pub fn try_spawn_in_temp_data_dir() -> std::io::Result { + let temp_dir = tempfile::tempdir()?; + let data_dir = temp_dir.path().to_path_buf(); + Self::spawn_spacetime_start_with_data_dir(false, None, data_dir, Some(temp_dir)) + } + /// Start `spacetimedb` in a temporary data directory with optional PostgreSQL wire protocol. pub fn spawn_in_temp_data_dir_with_pg_port(pg_port: Option) -> Self { let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); let data_dir_path = temp_dir.path().to_path_buf(); Self::spawn_spacetime_start_with_data_dir(false, pg_port, data_dir_path, Some(temp_dir)) + .expect("failed to start SpacetimeDB") } /// Start `spacetimedb` in a temporary data directory via: @@ -136,6 +144,7 @@ impl SpacetimeDbGuard { let data_dir_path = temp_dir.path().to_path_buf(); Self::spawn_spacetime_start_with_data_dir(true, None, data_dir_path, Some(temp_dir)) + .expect("failed to start SpacetimeDB") } /// Start `spacetimedb` with an explicit data directory (for restart scenarios). @@ -143,7 +152,7 @@ impl SpacetimeDbGuard { /// Unlike `spawn_in_temp_data_dir`, this method does not create a temporary directory. /// The caller is responsible for managing the data directory lifetime. pub fn spawn_with_data_dir(data_dir: PathBuf, pg_port: Option) -> Self { - Self::spawn_spacetime_start_with_data_dir(false, pg_port, data_dir, None) + Self::spawn_spacetime_start_with_data_dir(false, pg_port, data_dir, None).expect("failed to start SpacetimeDB") } fn spawn_spacetime_start_with_data_dir( @@ -151,11 +160,11 @@ impl SpacetimeDbGuard { pg_port: Option, data_dir: PathBuf, _data_dir_handle: Option, - ) -> Self { + ) -> std::io::Result { let spawn_id = next_spawn_id(); let (child, logs, host_url, reader_threads) = - Self::spawn_server(&data_dir, pg_port, spawn_id, use_installed_cli); - SpacetimeDbGuard { + Self::spawn_server(&data_dir, pg_port, spawn_id, use_installed_cli)?; + Ok(SpacetimeDbGuard { child, host_url, logs, @@ -164,7 +173,7 @@ impl SpacetimeDbGuard { _data_dir_handle, reader_threads, use_installed_cli, - } + }) } /// Stop the server process without dropping the guard. @@ -192,7 +201,8 @@ impl SpacetimeDbGuard { eprintln!("[RESTART-{:03}] Spawning new server", spawn_id); let (child, logs, host_url, reader_threads) = - Self::spawn_server(&self.data_dir, self.pg_port, spawn_id, self.use_installed_cli); + Self::spawn_server(&self.data_dir, self.pg_port, spawn_id, self.use_installed_cli) + .expect("failed to restart SpacetimeDB"); eprintln!( "[RESTART-{:03}] New server ready, pid={}, url={}", spawn_id, @@ -214,7 +224,11 @@ impl SpacetimeDbGuard { /// Kills the current server process and waits for it to exit. fn kill_process(&mut self) { - let pid = self.child.id(); + Self::kill_child(&mut self.child, &mut self.reader_threads); + } + + fn kill_child(child: &mut Child, reader_threads: &mut Vec>) { + let pid = child.id(); eprintln!("[KILL] Killing process tree for pid={}", pid); // Kill the process tree to ensure all child processes are terminated. @@ -232,16 +246,16 @@ impl SpacetimeDbGuard { #[cfg(not(windows))] { - let result = self.child.kill(); + let result = child.kill(); eprintln!("[KILL] kill result for pid={}: {:?}", pid, result); } - let wait_result = self.child.wait(); + let wait_result = child.wait(); eprintln!("[KILL] wait() result for pid={}: {:?}", pid, wait_result); // Join reader threads to prevent leaks. // The threads will exit naturally once the process is killed and pipes close. - let threads = std::mem::take(&mut self.reader_threads); + let threads = std::mem::take(reader_threads); for handle in threads { let _ = handle.join(); } @@ -250,18 +264,19 @@ impl SpacetimeDbGuard { /// Spawns a new server process with the given data directory. /// Returns (child, logs, host_url, reader_threads). + #[allow(clippy::type_complexity)] fn spawn_server( data_dir: &Path, pg_port: Option, spawn_id: u64, use_installed_cli: bool, - ) -> (Child, Arc>, String, Vec>) { + ) -> std::io::Result<(Child, Arc>, String, Vec>)> { let cmd = if use_installed_cli { eprintln!("[SPAWN-{:03}] START Using installed CLI", spawn_id); Command::new("spacetime") } else { // Use the built CLI (common case) - let cli_path = ensure_binaries_built(); + let cli_path = cli_binary_path(); Command::new(&cli_path) }; eprintln!( @@ -288,64 +303,73 @@ impl SpacetimeDbGuard { } eprintln!("[SPAWN-{:03}] Spawning child process", spawn_id); - let (child, logs, reader_threads) = Self::spawn_child(cmd, &args, spawn_id); + let (mut child, logs, mut reader_threads) = Self::spawn_child(cmd, &args, spawn_id)?; eprintln!("[SPAWN-{:03}] Child spawned pid={}", spawn_id, child.id()); + let host_url = Self::finish_server_start( + &mut child, + &logs, + &mut reader_threads, + spawn_id, + Duration::from_secs(10), + )?; + Ok((child, logs, host_url, reader_threads)) + } - // Wait for the server to be ready - eprintln!("[SPAWN-{:03}] Waiting for listen address", spawn_id); - let listen_addr = wait_for_listen_addr(&logs, Duration::from_secs(10), spawn_id).unwrap_or_else(|| { - // Dump diagnostic info on failure - let buf = logs.lock().unwrap(); - eprintln!("[SPAWN-{:03}] TIMEOUT after 10s", spawn_id); - eprintln!( - "[SPAWN-{:03}] Captured {} bytes, {} lines", - spawn_id, - buf.len(), - buf.lines().count() - ); - eprintln!( - "[SPAWN-{:03}] Contains 'Starting SpacetimeDB': {}", - spawn_id, - buf.contains("Starting SpacetimeDB") - ); - // Check if process is still running - drop(buf); // Release lock before try_wait - panic!("Timed out waiting for SpacetimeDB to report listen address") - }); - eprintln!("[SPAWN-{:03}] Got listen_addr={}", spawn_id, listen_addr); - - let host_url = format!("http://{}", listen_addr); - - // Wait until HTTP is ready - eprintln!("[SPAWN-{:03}] Waiting for HTTP ready", spawn_id); - let client = Client::new(); - let deadline = Instant::now() + Duration::from_secs(10); - while Instant::now() < deadline { - let url = format!("{}/v1/ping", host_url); - if let Ok(resp) = client.get(&url).send() - && resp.status().is_success() - { - eprintln!("[SPAWN-{:03}] HTTP ready at {}", spawn_id, host_url); - return (child, logs, host_url, reader_threads); + fn finish_server_start( + child: &mut Child, + logs: &Arc>, + reader_threads: &mut Vec>, + spawn_id: u64, + timeout: Duration, + ) -> std::io::Result { + let result = (|| { + let listen_addr = wait_for_listen_addr(logs, timeout, spawn_id).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::TimedOut, + "SpacetimeDB did not report its listen address", + ) + })?; + let host_url = format!("http://{listen_addr}"); + let client = Client::builder() + .timeout(timeout) + .build() + .map_err(std::io::Error::other)?; + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if let Ok(resp) = client + .get(format!("{host_url}/v1/ping")) + .timeout(deadline.saturating_duration_since(Instant::now())) + .send() + && resp.status().is_success() + { + eprintln!("[SPAWN-{:03}] HTTP ready at {}", spawn_id, host_url); + return Ok(host_url); + } + sleep(Duration::from_millis(50)); + } + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!("SpacetimeDB HTTP /v1/ping not ready at {host_url}"), + )) + })(); + if result.is_err() { + Self::kill_child(child, reader_threads); + if let Ok(logs) = logs.lock() { + eprintln!("[SPAWN-{spawn_id:03}] startup failed; server logs:\n{logs}"); } - sleep(Duration::from_millis(50)); } - panic!("Timed out waiting for SpacetimeDB HTTP /v1/ping at {}", host_url); + result } + #[allow(clippy::type_complexity)] fn spawn_child( mut cmd: Command, args: &[&str], spawn_id: u64, - ) -> (Child, Arc>, Vec>) { + ) -> std::io::Result<(Child, Arc>, Vec>)> { eprintln!("[SPAWN-{:03}] spawn_child: about to spawn", spawn_id); - let mut child = cmd - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .expect("failed to spawn spacetimedb-cli"); + let mut child = cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; let pid = child.id(); eprintln!("[SPAWN-{:03}] spawn_child: spawned pid={}", spawn_id, pid); @@ -406,7 +430,7 @@ impl SpacetimeDbGuard { } eprintln!("[SPAWN-{:03}] spawn_child: readers attached", spawn_id); - (child, logs, reader_threads) + Ok((child, logs, reader_threads)) } } @@ -496,3 +520,43 @@ impl Drop for SpacetimeDbGuard { } } } + +#[cfg(test)] +mod startup_tests { + use super::*; + + #[test] + fn waiting_child() { + if env::var_os("SPACETIMEDB_GUARD_TEST_CHILD").is_some() { + sleep(Duration::from_secs(60)); + } + } + + #[test] + fn startup_timeout_reaps_child_for_both_readiness_phases() { + for reported_address in [false, true] { + let mut command = Command::new(env::current_exe().unwrap()); + command.env("SPACETIMEDB_GUARD_TEST_CHILD", "1"); + let (mut child, logs, mut readers) = + SpacetimeDbGuard::spawn_child(command, &["--exact", "startup_tests::waiting_child"], 0).unwrap(); + assert!(!readers.is_empty()); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let log = if reported_address { + format!("Starting SpacetimeDB listening on {}\n", listener.local_addr().unwrap()) + } else { + String::new() + }; + *logs.lock().unwrap() = log; + let error = + SpacetimeDbGuard::finish_server_start(&mut child, &logs, &mut readers, 0, Duration::from_millis(10)) + .unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); + assert_eq!(error.to_string().contains("HTTP"), reported_address); + assert!( + child.try_wait().unwrap().is_some(), + "failed startup left its child alive" + ); + assert!(readers.is_empty()); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/bench/types.rs b/tools/xtask-llm-benchmark/src/bench/types.rs index c99ca37ccd5..eb6a44a6696 100644 --- a/tools/xtask-llm-benchmark/src/bench/types.rs +++ b/tools/xtask-llm-benchmark/src/bench/types.rs @@ -193,6 +193,7 @@ pub struct BenchRunContext<'a> { pub dry_run_id: Option, } +#[derive(Clone)] pub struct RunConfig { pub modes: Option>, pub hash_only: bool, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt index 2e042f4e476..e74391b2da4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in C# that fetches external data before storing a result in a transaction. +The caller supplies a public example-page URL, such as https://example.com. Fetch that page, then store the response metadata. + TABLE - FetchedRecord (public, accessor FetchedRecord) - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt index 5b42d9d9811..67d18b777af 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in Rust that fetches external data before storing a result in a transaction. +The caller supplies a public example-page URL, such as https://example.com. Fetch that page, then store the response metadata. + TABLE - fetched_record (public), struct FetchedRecord - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt index 6ae5f60bfc8..087c8bcce0d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that fetches external data before storing a result in a transaction. +The caller supplies a public example-page URL, such as https://example.com. Fetch that page, then store the response metadata. + TABLE - fetched_record (public) - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt index 5817b13ce75..84443a7d58b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in C# that exposes an upload route and an external upload procedure. +The caller supplies sample bytes and a public HTTP echo endpoint, such as https://postman-echo.com/post. Send only those supplied bytes, then store the response metadata. + TABLE - UploadedAsset (public, accessor UploadedAsset) - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt index f528ede25ac..aaa89d38223 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in Rust that exposes an upload route and an external upload procedure. +The caller supplies sample bytes and a public HTTP echo endpoint, such as https://postman-echo.com/post. Send only those supplied bytes, then store the response metadata. + TABLE - uploaded_asset (public), struct UploadedAsset - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt index 3f06b4e2dcb..057328fad14 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that exposes an upload route and an external upload procedure. +The caller supplies sample bytes and a public HTTP echo endpoint, such as https://postman-echo.com/post. Send only those supplied bytes, then store the response metadata. + TABLE - uploaded_asset (public) - Fields: diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index 769371a79b0..ce5150058ea 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -298,11 +298,12 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { eprintln!("[warn] failed to upload task catalog: {e}"); } - let initialized = initialize_runtime(config.hash_only)?; + let isolated_csharp = config.lang == Lang::CSharp && !config.goldens_only; + let initialized = initialize_runtime(config.hash_only || isolated_csharp)?; let mut guard = initialized.guard; // Drop the runtime first: finish blocking scoring/cleanup before stopping the server. let runtime = initialized.runtime; - save_server_diagnostics(guard.as_mut(), "start"); + save_server_diagnostics(guard.as_mut(), "start", None); config.host = guard.as_ref().map(|g| g.host_url.clone()); @@ -324,7 +325,7 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { config.lang, selectors_ref, )); - save_server_diagnostics(guard.as_mut(), "end"); + save_server_diagnostics(guard.as_mut(), "end", None); result?; println!("[{}] goldens-only validation complete", config.lang.as_str()); return Ok(()); @@ -356,12 +357,12 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { Err(error) => Err(error), }; if let Err(error) = status { - report_server_status(guard.as_mut()); + report_server_status(guard.as_mut(), None); mode_failures.push(format!("{mode}: {error:#}")); } } - save_server_diagnostics(guard.as_mut(), "end"); + save_server_diagnostics(guard.as_mut(), "end", None); // Write local run log on --dry-run so results aren't lost if dry_run && !all_outcomes.is_empty() { @@ -389,11 +390,11 @@ fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { Ok(()) } -fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { +fn report_server_status(guard: Option<&mut SpacetimeDbGuard>, scope: Option<&str>) { let Some(guard) = guard else { return; }; - save_server_diagnostics(Some(guard), "failure"); + save_server_diagnostics(Some(guard), "failure", scope); match guard.child.try_wait() { Ok(Some(status)) => eprintln!("[server] local SpacetimeDB exited unexpectedly: {status}"), Ok(None) => eprintln!("[server] local SpacetimeDB is still running after benchmark failure"), @@ -402,12 +403,15 @@ fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { } // Capture before Drop kills the server, so cleanup cannot be mistaken for the cause. -fn save_server_diagnostics(guard: Option<&mut SpacetimeDbGuard>, phase: &str) { +fn save_server_diagnostics(guard: Option<&mut SpacetimeDbGuard>, phase: &str, scope: Option<&str>) { let (Some(guard), Some(dir)) = (guard, std::env::var_os("LLM_BENCHMARK_DIAGNOSTICS_DIR")) else { return; }; let result = (|| -> Result<()> { - let dir = PathBuf::from(dir); + let mut dir = PathBuf::from(dir); + if let Some(scope) = scope { + dir.push(scope); + } fs::create_dir_all(&dir)?; let pid = guard.child.id(); let mut resources = format!( @@ -708,19 +712,47 @@ fn run_mode_benchmarks( return Ok((Vec::new(), Ok(()))); } - let runtime = runtime.expect("runtime required for normal runs"); let llm_provider = llm_provider.expect("llm provider required for normal runs"); - let (route_runs, status) = runtime.block_on(run_many_routes_for_mode( - bench_root, - mode, - &hash, - &context, - lang, - config, - llm_provider.as_ref(), - &routes, - )); + let (route_runs, status) = if lang == Lang::CSharp { + // C# batches run sequentially with fresh server state to bound cross-model memory growth. + let runs = routes.iter().enumerate().map(|(index, route)| { + let tag = xtask_llm_benchmark::bench::run_scope_tag(mode, route.vendor.slug(), &route.api_model); + let scope = format!("csharp-{index}-{tag}"); + with_isolated_server(&scope, |host| async { + let mut config = config.clone(); + config.host = Some(host); + let (runs, status) = run_many_routes_for_mode( + bench_root, + mode, + &hash, + &context, + lang, + &config, + llm_provider.as_ref(), + std::slice::from_ref(route), + ) + .await; + status?; + Ok(runs) + }) + .with_context(|| format!("model {}:{}", route.vendor.slug(), route.api_model)) + }); + collect_sequential_model_runs(runs) + } else { + runtime + .expect("runtime required for normal runs") + .block_on(run_many_routes_for_mode( + bench_root, + mode, + &hash, + &context, + lang, + config, + llm_provider.as_ref(), + &routes, + )) + }; // Print summary sorted by pass rate descending let mut summary: Vec<(&str, u32, u32, f32)> = route_runs @@ -885,7 +917,11 @@ async fn collect_model_runs(runs: impl futures::Stream>) -> } } } - let status = if failures.is_empty() { + (completed, model_batch_status(failures)) +} + +fn model_batch_status(failures: Vec) -> Result<()> { + if failures.is_empty() { Ok(()) } else { Err(anyhow::anyhow!( @@ -893,8 +929,20 @@ async fn collect_model_runs(runs: impl futures::Stream>) -> failures.len(), failures.join("\n") )) - }; - (completed, status) + } +} + +// Consume every lazy batch, including those after a failure, without starting servers concurrently. +fn collect_sequential_model_runs(runs: impl Iterator>>) -> (Vec, Result<()>) { + let mut completed = Vec::new(); + let mut failures = Vec::new(); + for result in runs { + match result { + Ok(runs) => completed.extend(runs), + Err(error) => failures.push(format!("{error:#}")), + } + } + (completed, model_batch_status(failures)) } fn categories_to_set(v: Option>) -> Option> { @@ -912,8 +960,8 @@ pub struct RuntimeInit { pub guard: Option, } -fn initialize_runtime(hash_only: bool) -> Result { - if hash_only { +fn initialize_runtime(skip_runtime: bool) -> Result { + if skip_runtime { return Ok(RuntimeInit { runtime: None, guard: None, @@ -921,7 +969,7 @@ fn initialize_runtime(hash_only: bool) -> Result { } // Use locally built SpacetimeDB so server supports spacetime:sys@2.0 (required by local TypeScript SDK). - let spacetime = SpacetimeDbGuard::spawn_in_temp_data_dir(); + let spacetime = SpacetimeDbGuard::try_spawn_in_temp_data_dir().context("start local benchmark server")?; let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; @@ -931,6 +979,47 @@ fn initialize_runtime(hash_only: bool) -> Result { }) } +/// Own the runtime and server together so even failed batches drain blocking work before shutdown. +fn with_isolated_server(scope: &str, run: impl FnOnce(String) -> F) -> Result +where + F: std::future::Future>, +{ + let initialized = initialize_runtime(false)?; + let mut guard = initialized.guard; + let runtime = initialized.runtime.expect("isolated batch requires a runtime"); + let host = guard + .as_ref() + .expect("isolated batch requires a server") + .host_url + .clone(); + println!("[server] starting isolated batch {scope} at {host}"); + save_server_diagnostics(guard.as_mut(), "start", Some(scope)); + let result = runtime.block_on(async { + let period = std::time::Duration::from_secs(60); + let mut samples = tokio::time::interval_at(tokio::time::Instant::now() + period, period); + samples.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let work = run(host); + tokio::pin!(work); + let mut sample = 0; + loop { + tokio::select! { + result = &mut work => break result, + _ = samples.tick() => { + sample += 1; + save_server_diagnostics(guard.as_mut(), &format!("sample-{sample}"), Some(scope)); + } + } + } + }); + if result.is_err() { + report_server_status(guard.as_mut(), Some(scope)); + } + drop(runtime); + save_server_diagnostics(guard.as_mut(), "end", Some(scope)); + // The guard now stops the server and removes this batch's temporary database directory. + result +} + fn find_bench_root() -> PathBuf { let start = PathBuf::from(env!("CARGO_MANIFEST_DIR")); for dir in start.ancestors() { @@ -1031,6 +1120,60 @@ fn apply_category_filter( mod tests { use super::*; + #[test] + #[ignore = "starts two local servers; requires built CLI and standalone binaries, no LLM calls"] + fn isolated_server_stops_after_failure_and_success() { + use std::sync::atomic::{AtomicBool, Ordering}; + let drained = Arc::new(AtomicBool::new(false)); + let mut hosts = Vec::new(); + for fail in [true, false] { + let worker_drained = drained.clone(); + let scope = if fail { + "isolation-test-failure" + } else { + "isolation-test-success" + }; + let result = with_isolated_server(scope, |host| { + hosts.push(host.clone()); + async move { + reqwest::Client::new() + .get(format!("{host}/v1/ping")) + .send() + .await? + .error_for_status()?; + tokio::task::spawn_blocking(move || { + std::thread::sleep(std::time::Duration::from_millis(50)); + let server_alive = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(2)) + .build() + .unwrap() + .get(format!("{host}/v1/ping")) + .send() + .is_ok_and(|response| response.status().is_success()); + worker_drained.store(server_alive, Ordering::SeqCst); + }); + anyhow::ensure!(!fail, "injected batch failure"); + Ok(()) + } + }); + if fail { + assert_eq!(result.unwrap_err().to_string(), "injected batch failure"); + } else { + result.unwrap(); + } + assert!( + drained.load(Ordering::SeqCst), + "blocking work must finish before server shutdown" + ); + let client = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(2)) + .build() + .unwrap(); + assert!(client.get(format!("{}/v1/ping", hosts.last().unwrap())).send().is_err()); + drained.store(false, Ordering::SeqCst); + } + } + #[test] fn resource_diagnostics_include_contents_and_missing_file_errors() { let mut output = String::new(); @@ -1041,6 +1184,24 @@ mod tests { assert!(output.contains("unavailable:")); } + #[test] + fn failed_sequential_model_keeps_later_results_and_failure_status() { + let mut attempted = Vec::new(); + let runs = ["failed", "healthy", "last"].into_iter().map(|model| { + attempted.push(model); + anyhow::ensure!(model != "failed", "injected batch failure"); + Ok(vec![model]) + }); + let (completed, status) = collect_sequential_model_runs(runs); + assert_eq!(attempted, ["failed", "healthy", "last"]); + assert_eq!(completed, ["healthy", "last"]); + assert!(status.unwrap_err().to_string().contains("injected batch failure")); + + let (completed, status) = collect_sequential_model_runs([Ok(vec!["healthy"])].into_iter()); + assert_eq!(completed, ["healthy"]); + assert!(status.is_ok()); + } + fn base_run_args() -> RunArgs { RunArgs { modes: None, From 0cd339c21dc489ddbda086ee026e58f40e08e97e Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Fri, 30 Oct 2026 15:34:26 -0400 Subject: [PATCH 07/41] Preserve incomplete benchmark results and refusal evidence --- tools/xtask-llm-benchmark/README.md | 30 ++ tools/xtask-llm-benchmark/src/api/client.rs | 1 + tools/xtask-llm-benchmark/src/bench/mod.rs | 1 + .../xtask-llm-benchmark/src/bench/records.rs | 320 +++++++++++++ tools/xtask-llm-benchmark/src/bench/runner.rs | 442 +++++++++++++++--- .../src/llm/clients/anthropic.rs | 90 +++- .../src/llm/clients/google.rs | 119 ++++- .../src/llm/clients/http.rs | 96 ++-- .../src/llm/clients/oa_compat.rs | 152 +++++- .../src/llm/clients/openrouter.rs | 45 +- 10 files changed, 1173 insertions(+), 123 deletions(-) create mode 100644 tools/xtask-llm-benchmark/src/bench/records.rs diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index 1943341d3c7..3e3c12c9a0e 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -10,6 +10,36 @@ still run, but the language job exits with failure if any batch fails. Response errors include the generation ID and finish reason when supplied by the provider. Generated code that fails compilation or grading remains a normal benchmark score. +Each batch also saves an append-only JSONL record under +`LLM_BENCHMARK_REPORT_DIR/batches`, or `tools/xtask-llm-benchmark/runs/batches` +when that variable is not set. CI retains these files in its analysis artifacts, +including on failure. They are not website uploads. + +The record contains the planned tasks, model identity, each finished task attempt, +generated code, scorer details, and provider refusal details. Outer task retries +remain in the record. The latest task attempt determines its status and the +summary. Inner provider-call retries are not separate artifact entries; their +status and retry counts remain in console logs. Raw refusal text, partial provider +content, and HTTP error bodies stay in artifacts, not console errors. Treat these +artifacts as source and provider-response data. + +Refusals and provider or infrastructure errors have no code score. Summaries keep +task counts separate from scorer-check counts and always include coverage. For +example, 99 passing tasks and one refusal means **99/99 scored tasks passed, with +99/100 tasks scored and one refusal**. It does not mean a complete 100% benchmark. +Zero scored tasks means no score, not 0%. An incomplete batch stays off the website. +Migration setup failures and publish failures without a recognized source-code +diagnostic also block the batch, even when the server responds to a health check. +This is conservative: an unknown code error can block a batch until classified. + +Each finished attempt is saved before an error can stop the batch. If a task +reports an infrastructure error, new tasks stop, but started tasks finish and save +their results before the batch returns failure. If the process stops during a +write, only the last JSONL line can be incomplete; earlier lines +remain usable. Only a final `complete` event after the server health check marks +the batch complete for scoring. It does not confirm a website upload. A record +write failure blocks upload. This does not resume or upload partial runs. + ## New behavioral tasks | ID | Contract checked | diff --git a/tools/xtask-llm-benchmark/src/api/client.rs b/tools/xtask-llm-benchmark/src/api/client.rs index 328693e57d2..02bab7af50a 100644 --- a/tools/xtask-llm-benchmark/src/api/client.rs +++ b/tools/xtask-llm-benchmark/src/api/client.rs @@ -457,6 +457,7 @@ mod tests { Err(e) => panic!("upload request was not received: {e}"), } }; + stream.set_nonblocking(false).unwrap(); stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); stream.set_write_timeout(Some(Duration::from_secs(10))).unwrap(); let mut reader = BufReader::new(&stream); diff --git a/tools/xtask-llm-benchmark/src/bench/mod.rs b/tools/xtask-llm-benchmark/src/bench/mod.rs index 7d900be053a..db4f86ba64e 100644 --- a/tools/xtask-llm-benchmark/src/bench/mod.rs +++ b/tools/xtask-llm-benchmark/src/bench/mod.rs @@ -1,6 +1,7 @@ pub mod analysis; pub mod normalize; pub mod publishers; +mod records; pub mod runner; mod templates; pub mod types; diff --git a/tools/xtask-llm-benchmark/src/bench/records.rs b/tools/xtask-llm-benchmark/src/bench/records.rs new file mode 100644 index 00000000000..2d09a6166d0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/bench/records.rs @@ -0,0 +1,320 @@ +//! Append-only local evidence. A failed or interrupted batch never gets a complete marker. +use super::types::{BenchRunContext, RunOneError, RunOutcome, TaskPaths}; +use crate::llm::clients::oa_compat::ProviderRefusal; +use anyhow::{bail, Context, Result}; +use chrono::Utc; +use serde_json::{json, Value}; +use std::collections::BTreeMap; +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +pub(super) struct BatchRecord { + path: PathBuf, + file: File, + // Latest state determines coverage. Earlier attempts remain in the journal. + tasks: BTreeMap, +} + +enum TaskState { + Pending, + Scored { passed: u32, total: u32 }, + Refused, + ProviderError, + InfrastructureError, +} + +impl BatchRecord { + pub(super) fn new(cfg: &BenchRunContext<'_>, tasks: &[TaskPaths]) -> Result { + let root = std::env::var_os("LLM_BENCHMARK_REPORT_DIR") + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("runs")) + .join("batches"); + let record = Self::create( + &root, + json!({ + "hash": cfg.hash, "lang": cfg.lang.as_str(), "mode": cfg.mode, + "model": cfg.route.model_id(), "api_model": cfg.route.api_model, + "vendor": cfg.route.vendor.slug(), "dry_run": cfg.dry_run, + }), + tasks + .iter() + .map(|task| task.root.to_string_lossy().into_owned()) + .collect(), + )?; + eprintln!("[runner] local batch record: {}", record.path.display()); + Ok(record) + } + + fn create(root: &Path, identity: Value, tasks: Vec) -> Result { + std::fs::create_dir_all(root)?; + let path = root.join(format!( + "{}-{}.jsonl", + Utc::now().timestamp_nanos_opt().context("timestamp out of range")?, + std::process::id() + )); + let file = OpenOptions::new().write(true).create_new(true).open(&path)?; + let mut record = Self { + path, + file, + tasks: tasks.iter().map(|task| (task.clone(), TaskState::Pending)).collect(), + }; + record.append( + json!({"event": "start", "schema_version": 1, "identity": identity, "tasks": tasks}), + false, + )?; + Ok(record) + } + + pub(super) fn record(&mut self, task: &str, result: &Result) -> Result<()> { + let (state, attempt) = match result { + Ok(outcome) => ( + TaskState::Scored { + passed: outcome.passed_tests, + total: outcome.total_tests, + }, + json!({"status": "scored", "outcome": outcome}), + ), + Err(RunOneError::Other(error)) => match error.downcast_ref::() { + Some(refusal) => (TaskState::Refused, json!({"status": "refused", "refusal": refusal})), + None => ( + TaskState::ProviderError, + json!({"status": "provider_or_setup_error", "error": format!("{error:#}"), + "provider_response": error.downcast_ref::(), + "http_response": error.downcast_ref::()}), + ), + }, + Err(RunOneError::Infrastructure { msg, llm_output }) => ( + TaskState::InfrastructureError, + json!({"status": "infrastructure_error", "error": msg, "llm_output": llm_output}), + ), + Err(RunOneError::WithOutput { .. }) => bail!("normalize generated-code failures before recording"), + }; + *self.tasks.get_mut(task).context("task not in planned batch")? = state; + self.append(json!({"event": "attempt", "task": task, "attempt": attempt}), false) + } + + pub(super) fn complete(&mut self) -> Result<()> { + if !self + .tasks + .values() + .all(|state| matches!(state, TaskState::Scored { .. })) + { + bail!("incomplete batch record; refusing to upload"); + } + self.append(json!({"event": "complete"}), true) + } + + fn append(&mut self, mut event: Value, complete: bool) -> Result<()> { + let (mut scored, mut passed, mut refused, mut provider, mut infrastructure, mut pending) = (0, 0, 0, 0, 0, 0); + let (mut passed_checks, mut scored_checks) = (0u64, 0u64); + for state in self.tasks.values() { + match state { + TaskState::Pending => pending += 1, + TaskState::Refused => refused += 1, + TaskState::ProviderError => provider += 1, + TaskState::InfrastructureError => infrastructure += 1, + TaskState::Scored { passed: p, total } => { + scored += 1; + passed += usize::from(*total > 0 && p == total); + passed_checks += u64::from(*p); + scored_checks += u64::from(*total); + } + } + } + event["at"] = json!(Utc::now()); + event["complete"] = json!(complete); + event["summary"] = json!({ + "planned_tasks": self.tasks.len(), "scored_tasks": scored, "passed_tasks": passed, + "refused_tasks": refused, "provider_or_setup_errors": provider, + "infrastructure_errors": infrastructure, "pending_tasks": pending, + "passed_checks": passed_checks, "scored_checks": scored_checks, + }); + // Serialize first. An interrupted write can affect only the last line, not earlier evidence. + let mut bytes = serde_json::to_vec(&event)?; + bytes.push(b'\n'); + self.file + .write_all(&bytes) + .with_context(|| format!("write {}", self.path.display()))?; + self.file + .sync_data() + .with_context(|| format!("sync {}", self.path.display()))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::llm::clients::oa_compat::OACompatResp; + + fn outcome(passed: u32) -> RunOutcome { + serde_json::from_value(json!({ + "hash": "test", "task": "a", "lang": "rust", "golden_published": true, + "model_name": "test", "total_tests": 3, "passed_tests": passed, + "llm_output": "generated code", "scorer_details": {}, "vendor": "test" + })) + .unwrap() + } + + #[test] + fn incomplete_records_keep_scores_refusals_and_retry_history() { + let root = std::env::temp_dir().join(format!( + "benchmark-record-test-{}-{}", + std::process::id(), + Utc::now().timestamp_nanos_opt().unwrap() + )); + let mut record = BatchRecord::create(&root, json!({"model": "test"}), vec!["a".into(), "b".into()]).unwrap(); + record.record("a", &Ok(outcome(3))).unwrap(); + let response: OACompatResp = serde_json::from_value(json!({ + "id": "gen-test", "provider": "test-provider", "choices": [{ + "finish_reason": "content_filter", "native_finish_reason": "refusal", + "message": {"content": null, "refusal": "full refusal reason"} + }] + })) + .unwrap(); + record + .record("b", &Err(RunOneError::Other(response.first_text().unwrap_err()))) + .unwrap(); + assert!(record.complete().is_err()); + let rows = read_rows(&record.path); + let last = rows.last().unwrap(); + assert_eq!(last["summary"]["scored_tasks"], 1); + assert_eq!(last["summary"]["passed_tasks"], 1); + assert_eq!(last["summary"]["refused_tasks"], 1); + assert_eq!(last["summary"]["passed_checks"], 3); + assert_eq!(last["summary"]["scored_checks"], 3); + assert_eq!(last["complete"], false); + assert_eq!(last["attempt"]["refusal"]["refusal"], "full refusal reason"); + assert_eq!(rows[1]["attempt"]["outcome"]["llm_output"], "generated code"); + // A generated-code failure is a scored result, unlike a provider refusal. + record.record("b", &Ok(outcome(0))).unwrap(); + record.complete().unwrap(); + let rows = read_rows(&record.path); + let last = rows.last().unwrap(); + assert_eq!(last["complete"], true); + assert_eq!(last["summary"]["scored_tasks"], 2); + assert_eq!(last["summary"]["passed_tasks"], 1); + assert_eq!(last["summary"]["refused_tasks"], 0); + assert_eq!(last["summary"]["scored_checks"], 6); + assert_eq!(rows[2]["attempt"]["refusal"]["generation_id"], "gen-test"); + drop(record); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn infrastructure_failure_keeps_prior_results_and_pending_tasks() { + let root = std::env::temp_dir().join(format!( + "benchmark-infra-test-{}-{}", + std::process::id(), + Utc::now().timestamp_nanos_opt().unwrap() + )); + let mut record = BatchRecord::create(&root, json!({}), vec!["a".into(), "b".into(), "c".into()]).unwrap(); + record.record("a", &Ok(outcome(3))).unwrap(); + record + .record( + "b", + &Err(RunOneError::Infrastructure { + msg: "server unavailable".into(), + llm_output: Some("unscored code".into()), + }), + ) + .unwrap(); + assert!(record.complete().is_err()); + let rows = read_rows(&record.path); + let last = rows.last().unwrap(); + assert_eq!(last["summary"]["scored_tasks"], 1); + assert_eq!(last["summary"]["infrastructure_errors"], 1); + assert_eq!(last["summary"]["pending_tasks"], 1); + assert_eq!(last["attempt"]["llm_output"], "unscored code"); + drop(record); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn no_scored_tasks_have_no_score_denominator_and_write_errors_fail_closed() { + let root = std::env::temp_dir().join(format!( + "benchmark-write-test-{}-{}", + std::process::id(), + Utc::now().timestamp_nanos_opt().unwrap() + )); + let mut record = BatchRecord::create(&root, json!({}), vec!["a".into()]).unwrap(); + record + .record("a", &Err(RunOneError::Other(anyhow::anyhow!("402 Payment Required")))) + .unwrap(); + record + .record( + "a", + &Err(RunOneError::Other( + crate::llm::clients::http::HttpResponseError { + status: 403, + body: "full HTTP refusal details".into(), + } + .into(), + )), + ) + .unwrap(); + assert!(record.complete().is_err()); + let rows = read_rows(&record.path); + assert_eq!(rows.last().unwrap()["summary"]["scored_checks"], 0); + assert_eq!(rows.last().unwrap()["summary"]["scored_tasks"], 0); + assert_eq!(rows.last().unwrap()["summary"]["provider_or_setup_errors"], 1); + assert_eq!( + rows.last().unwrap()["attempt"]["http_response"]["body"], + "full HTTP refusal details" + ); + let full_message = "refusal details ".repeat(100); + let response: OACompatResp = serde_json::from_value(json!({ + "id": "gen-error", "choices": [{ + "finish_reason": "error", "error": {"code": 403, "message": full_message}, + "message": {"content": "unscored partial code", "refusal": "private refusal detail"} + }] + })) + .unwrap(); + record + .record("a", &Err(RunOneError::Other(response.first_text().unwrap_err()))) + .unwrap(); + let rows = read_rows(&record.path); + assert_eq!( + rows.last().unwrap()["attempt"]["provider_response"]["error"]["message"], + full_message + ); + assert_eq!( + rows.last().unwrap()["attempt"]["provider_response"]["content"], + "unscored partial code" + ); + assert_eq!( + rows.last().unwrap()["attempt"]["provider_response"]["refusal"], + "private refusal detail" + ); + let response: OACompatResp = serde_json::from_value(json!({ + "id": "gen-finish-error", "choices": [{"finish_reason": "error", + "message": {"content": "partial code without an error object"}}] + })) + .unwrap(); + record + .record("a", &Err(RunOneError::Other(response.first_text().unwrap_err()))) + .unwrap(); + let rows = read_rows(&record.path); + assert_eq!( + rows.last().unwrap()["attempt"]["provider_response"]["content"], + "partial code without an error object" + ); + // Replace the writer with a read-only handle to inject an actual I/O failure. + record.file = File::open(&record.path).unwrap(); + assert!(record.record("a", &Ok(outcome(3))).is_err()); + assert!(record.complete().is_err()); + assert_eq!(read_rows(&record.path), rows, "prior evidence must remain intact"); + drop(record); + std::fs::remove_dir_all(root).unwrap(); + } + + fn read_rows(path: &Path) -> Vec { + std::fs::read_to_string(path) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } +} diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 7536c0eced4..47f8b761a18 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -5,10 +5,12 @@ use serde_json::json; use spacetimedb_data_structures::map::{HashCollectionExt as _, HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use tokio::task; use crate::bench::publishers::{DotnetPublisher, PublishedDatabase, SpacetimeRustPublisher, TypeScriptPublisher}; +use crate::bench::records::BatchRecord; use crate::bench::templates::materialize_project; use crate::bench::types::{BenchRunContext, PublishParams, RunContext, RunOneError}; pub(crate) use crate::bench::types::{RunOutcome, TaskPaths}; @@ -34,11 +36,57 @@ struct PendingRetry { error: anyhow::Error, } -fn task_result_or_infrastructure_error( +async fn collect_recorded_attempts( + attempts: impl IntoIterator, + concurrency: usize, + mut record: impl FnMut(T) -> Result, +) -> Result> +where + F: std::future::Future, +{ + let stopped = AtomicBool::new(false); + stream::iter(attempts) + .take_while(|_| std::future::ready(!stopped.load(Ordering::Relaxed))) + .buffer_unordered(concurrency) + .map(|attempt| { + let result = record(attempt); + if result.is_err() { + stopped.store(true, Ordering::Relaxed); + } + result + }) + // Drain in-flight attempts before returning an error. Do not lose completed work. + .collect::>() + .await + .into_iter() + .collect() +} + +fn record_task_result( + record: &mut BatchRecord, + cfg: &BenchRunContext<'_>, task: TaskPaths, result: Result, started: chrono::DateTime, ) -> Result<(TaskPaths, Result)> { + let result = match result { + Err(RunOneError::WithOutput { msg, llm_output }) => Ok(build_fail_outcome( + &task, + cfg.lang.as_str(), + cfg.route, + cfg.hash, + anyhow!(msg), + llm_output, + )), + other => other, + } + .map(|mut outcome| { + outcome.started_at.get_or_insert(started); + outcome + }); + // Save before propagating an infrastructure failure; the collector drains started tasks. + // A record write failure also aborts the batch before any website upload. + record.record(&task.root.to_string_lossy(), &result)?; match result { Err(RunOneError::Infrastructure { msg, llm_output }) => { let output_note = llm_output @@ -47,13 +95,7 @@ fn task_result_or_infrastructure_error( .unwrap_or_default(); bail!("benchmark infrastructure failure: {msg}{output_note}"); } - other => Ok(( - task, - other.map(|mut outcome| { - outcome.started_at.get_or_insert(started); - outcome - }), - )), + other => Ok((task, other)), } } @@ -111,14 +153,18 @@ async fn score_async( scorers: Vec>, output: String, database: Option, -) -> Result<(Vec<(&'static str, ScoreDetails)>, String, Option)> { +) -> Result<(Vec<(&'static str, ScoreDetails)>, String, Option), RunOneError> { + let failure_output = output.clone(); task::spawn_blocking(move || { // Keep the database alive if the awaiting task is cancelled while scoring. let scores = scorers.iter().map(|s| (s.id(), s.score(&output))).collect(); (scores, output, database) }) .await - .context("scoring worker failed") + .map_err(|error| RunOneError::Infrastructure { + msg: format!("scoring worker failed: {error:#}"), + llm_output: Some(failure_output), + }) } pub async fn delete_databases(databases: Vec) -> Result<()> { @@ -150,6 +196,116 @@ async fn ensure_server_healthy(host: Option<&str>) -> Result<()> { Ok(()) } +fn provider_retry_delay(error: &anyhow::Error) -> Option { + use crate::llm::clients::{ + http::HttpResponseError, + oa_compat::{ProviderRefusal, ProviderResponseError}, + }; + let status_delay = |status| match status { + 429 => Some(60), + 502..=504 => Some(30), + _ => None, + }; + if error.downcast_ref::().is_some() { + return None; + } + if let Some(response) = error.downcast_ref::() { + return status_delay(u64::from(response.status)); + } + // Inspect private fields for retry decisions, never for console output. + let message = if let Some(response) = error.downcast_ref::() { + if let Some(code) = response.error.code.as_ref().and_then(|code| { + code.as_u64() + .or_else(|| code.as_str().and_then(|value| value.parse().ok())) + }) { + return status_delay(code); + } + response + .error + .message + .as_deref() + .unwrap_or_default() + .to_ascii_lowercase() + } else { + format!("{error:#}").to_ascii_lowercase() + }; + if message.contains("429") || message.contains("rate limit") { + Some(60) + } else if message.contains("timed out") || ["502", "503", "504"].iter().any(|status| message.contains(status)) { + Some(30) + } else { + None + } +} + +fn is_code_publish_failure(message: &str, source_root: &Path, lang: Lang) -> bool { + // Unknown publish failures are unscored. A healthy ping does not prove a valid build environment. + let message = message.replace('\\', "/").to_ascii_lowercase(); + if [ + "error[e0463]", + "error[e0460]", + "error[e0514]", + "error cs0006", + "error cs0009", + "error cs0012", + "error cs0518", + "error cs1705", + ] + .iter() + .any(|code| message.contains(code)) + { + return false; + } + let relative = match lang { + Lang::Rust => "src/lib.rs", + Lang::CSharp => "lib.cs", + Lang::TypeScript => "src/index.ts", + }; + let absolute = source_root + .canonicalize() + .or_else(|_| std::path::absolute(source_root)) + .unwrap_or_else(|_| source_root.to_path_buf()) + .join(relative) + .to_string_lossy() + .replace('\\', "/") + .to_ascii_lowercase(); + let is_generated_location = |location: &str| { + let location = location.trim().trim_start_matches("//?/").trim_start_matches("./"); + [relative, absolute.trim_start_matches("//?/")].iter().any(|path| { + location + .strip_prefix(path) + .is_some_and(|rest| rest.starts_with(':') || rest.starts_with('(')) + }) + }; + let lines: Vec<_> = message.lines().map(str::trim).collect(); + let mut found = false; + for (index, line) in lines.iter().enumerate() { + let rust_error = matches!(lang, Lang::Rust) && (line.starts_with("error[e") || line.starts_with("error:")); + if rust_error { + // Cargo/rustc summaries add no source diagnostic of their own. + if line.starts_with("error: could not compile") || line.starts_with("error: aborting due to") { + continue; + } + let location = lines[index + 1..] + .iter() + .take_while(|line| !line.starts_with("error") && !line.starts_with("warning")) + .find_map(|line| line.strip_prefix("--> ")); + if !location.is_some_and(is_generated_location) { + return false; + } + found = true; + } else if (matches!(lang, Lang::CSharp) && (line.contains(": error cs") || line.contains(": error stdb"))) + || (matches!(lang, Lang::TypeScript) && line.contains(": error ts")) + { + if !is_generated_location(line) { + return false; + } + found = true; + } + } + found +} + impl TaskRunner { pub fn new( bench_root: PathBuf, @@ -278,18 +434,7 @@ impl TaskRunner { } Ok(Err(e)) => { let msg = format!("{e:#}"); - let retryable = msg.contains("timed out") - || msg.contains("429") - || msg.contains("502") - || msg.contains("503") - || msg.contains("504") - || msg.contains("rate limit"); - if retryable && attempt < MAX_ATTEMPTS { - let delay = if msg.contains("429") || msg.contains("rate limit") { - 60 - } else { - 30 - }; + if let Some(delay) = provider_retry_delay(&e).filter(|_| attempt < MAX_ATTEMPTS) { eprintln!( "⚠️ [{}/{}] provider error (attempt {}/{}), retrying in {delay}s: {}", cfg.lang_name, cfg.route.display_name, attempt, MAX_ATTEMPTS, msg @@ -323,7 +468,11 @@ impl TaskRunner { let output_tokens = llm_result.output_tokens; let mut llm_output = llm_result.text; - let migration_setup = load_migration_setup_source(task, cfg.lang)?; + let migration_setup = + load_migration_setup_source(task, cfg.lang).map_err(|error| RunOneError::Infrastructure { + msg: format!("load migration setup failed: {error:#}"), + llm_output: Some(llm_output.clone()), + })?; if debug_llm() { print_llm_output(&cfg.route.display_name, &task_id, &llm_output); @@ -347,8 +496,12 @@ impl TaskRunner { .await; match setup_result { Ok(database) => { - call_reducer_json_out(&llm_db, "seed", &[], Some(cfg.host.as_deref().unwrap_or("local"))) - .map_err(anyhow::Error::msg)?; + call_reducer_json_out(&llm_db, "seed", &[], Some(cfg.host.as_deref().unwrap_or("local"))).map_err( + |error| RunOneError::Infrastructure { + msg: format!("migration seed failed: {error}"), + llm_output: Some(llm_output.clone()), + }, + )?; Ok(Some(database)) } Err(error) => Err(error), @@ -358,7 +511,12 @@ impl TaskRunner { }; let (mut published_database, publish_error) = match setup_database { - Err(error) => (None, Some(format!("migration setup failed: {error:#}"))), + Err(error) => { + return Err(RunOneError::Infrastructure { + msg: format!("migration setup failed: {error:#}"), + llm_output: Some(llm_output), + }) + } Ok(mut setup_database) => { let publish_result = self .publish_llm(PublishParams { @@ -380,6 +538,13 @@ impl TaskRunner { (Some(database), None) } Err(error) => { + let source_root = work_server_dir_scoped(&category, &task_id, cfg.lang_name, "llm", &route_tag); + if !is_code_publish_failure(&format!("{error:#}"), &source_root, cfg.lang) { + return Err(RunOneError::Infrastructure { + msg: format!("publish failed without a source-code diagnostic: {error:#}"), + llm_output: Some(llm_output), + }); + } eprintln!( "⚠️ publish failed for {}/{}/{}: {error:#}", category, task_id, cfg.route.display_name @@ -405,13 +570,7 @@ impl TaskRunner { if publish_error.is_none() { println!("→ [{}] {}: scoring", cfg.lang_name, cfg.route.display_name); - let (scores, output, database) = - score_async(scorers, llm_output, published_database) - .await - .map_err(|error| RunOneError::Infrastructure { - msg: format!("scoring worker failed: {error:#}"), - llm_output: None, - })?; + let (scores, output, database) = score_async(scorers, llm_output, published_database).await?; llm_output = output; published_database = database; for (id, r) in scores { @@ -649,6 +808,7 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu // 1) run per-task LLM builds + scoring let tasks = discover_tasks(cfg.bench_root)?; + let mut record = BatchRecord::new(cfg, &tasks)?; let runner = TaskRunner::new( PathBuf::from(cfg.bench_root), SpacetimeRustPublisher, @@ -662,8 +822,8 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu _ => bench_concurrency(), }; - let results: Vec<(TaskPaths, Result)> = - futures::stream::iter(tasks.into_iter().map(|task| { + let results: Vec<(TaskPaths, Result)> = collect_recorded_attempts( + tasks.into_iter().map(|task| { let runner = &runner; let route = cfg.route; let lang = cfg.lang; @@ -687,12 +847,13 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu }; let result = runner.run_one(&task, &run_cfg).await; - task_result_or_infrastructure_error(task, result, started) + (task, result, started) } - })) - .buffer_unordered(buf) - .try_collect() - .await?; + }), + buf, + |(task, result, started)| record_task_result(&mut record, cfg, task, result, started), + ) + .await?; let (mut outcomes, mut retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; @@ -710,8 +871,8 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu ); tokio::time::sleep(std::time::Duration::from_secs(30)).await; - let retry_results: Vec<(TaskPaths, Result)> = - futures::stream::iter(retry_tasks.drain(..).map(|pending| { + let retry_results: Vec<(TaskPaths, Result)> = collect_recorded_attempts( + retry_tasks.drain(..).map(|pending| { let runner = &runner; let route = cfg.route; let lang = cfg.lang; @@ -735,12 +896,13 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu host, }; let result = runner.run_one(&task, &run_cfg).await; - task_result_or_infrastructure_error(task, result, started) + (task, result, started) } - })) - .buffer_unordered(buf) - .try_collect() - .await?; + }), + buf, + |(task, result, started)| record_task_result(&mut record, cfg, task, result, started), + ) + .await?; let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash)?; @@ -770,6 +932,7 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu // A dead local server turns later publish failures into misleading zeroes. // Refuse to analyze or upload any batch that cannot pass a final health check. ensure_server_healthy(cfg.host.as_deref()).await?; + record.complete()?; if cfg.dry_run { let _ = match maybe_generate_analysis(cfg, &outcomes).await { @@ -831,6 +994,7 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> bail!("no tasks matched {:?}", wanted); } + let mut record = BatchRecord::new(cfg, &selected)?; let runner = TaskRunner::new( PathBuf::from(cfg.bench_root), SpacetimeRustPublisher, @@ -844,8 +1008,8 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> _ => bench_concurrency(), }; - let results: Vec<(TaskPaths, Result)> = - futures::stream::iter(selected.into_iter().map(|task| { + let results: Vec<(TaskPaths, Result)> = collect_recorded_attempts( + selected.into_iter().map(|task| { let runner = &runner; let route = cfg.route; let lang = cfg.lang; @@ -868,12 +1032,13 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> }; let result = runner.run_one(&task, &run_cfg).await; - task_result_or_infrastructure_error(task, result, started) + (task, result, started) } - })) - .buffer_unordered(buf) - .try_collect() - .await?; + }), + buf, + |(task, result, started)| record_task_result(&mut record, cfg, task, result, started), + ) + .await?; let (mut outcomes, retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; @@ -893,8 +1058,8 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> ); tokio::time::sleep(std::time::Duration::from_secs(30)).await; - let retry_results: Vec<(TaskPaths, Result)> = - futures::stream::iter(pending.drain(..).map(|pending_retry| { + let retry_results: Vec<(TaskPaths, Result)> = collect_recorded_attempts( + pending.drain(..).map(|pending_retry| { let runner = &runner; let route = cfg.route; let lang = cfg.lang; @@ -918,12 +1083,13 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> host, }; let result = runner.run_one(&task, &run_cfg).await; - task_result_or_infrastructure_error(task, result, started) + (task, result, started) } - })) - .buffer_unordered(buf) - .try_collect() - .await?; + }), + buf, + |(task, result, started)| record_task_result(&mut record, cfg, task, result, started), + ) + .await?; let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash)?; @@ -945,6 +1111,7 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> let outcomes = finalize_batch(outcomes, pending)?; ensure_server_healthy(cfg.host.as_deref()).await?; + record.complete()?; if cfg.dry_run { let _ = match maybe_generate_analysis(cfg, &outcomes).await { @@ -1390,6 +1557,128 @@ fn normalize_task_selector(raw: &str) -> Result { mod build_cleanup_tests { use super::*; + #[tokio::test] + async fn failed_attempt_drains_started_work_without_starting_more_tasks() { + let mut recorded = Vec::new(); + let error = collect_recorded_attempts( + (0..4).map(|id| async move { + if id == 1 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + id + }), + 2, + |id| { + recorded.push(id); + if id == 0 { + bail!("infrastructure failure"); + } + Ok(id) + }, + ) + .await + .unwrap_err(); + assert_eq!(error.to_string(), "infrastructure failure"); + assert_eq!( + recorded, + [0, 1], + "retain in-flight evidence, but stop scheduling new tasks" + ); + } + + #[test] + fn only_explicit_code_diagnostics_are_scored_as_publish_failures() { + let root = std::env::temp_dir().join("benchmark-module"); + let root = root.as_path(); + for diagnostic in [ + "error[E0425]: cannot find value `missing` in this scope", + "error: expected one of `!` or `::`, found `main`", + "error: unexpected closing delimiter: `}`", + "error: mismatched closing delimiter: `)`", + "error: invalid suffix `words` for number literal", + "error: this file contains an unclosed delimiter", + "error: unknown start of token: \\", + ] { + let diagnostic = + format!("{diagnostic}\n --> src/lib.rs:10:5\nerror: could not compile `test` due to 1 previous error"); + assert!(is_code_publish_failure(&diagnostic, root, Lang::Rust), "{diagnostic}"); + } + for (lang, diagnostic) in [ + ( + Lang::CSharp, + "Lib.cs(10,5): error CS0103: The name 'missing' does not exist", + ), + ( + Lang::TypeScript, + "src/index.ts(2,3): error TS2304: Cannot find name 'missing'", + ), + ] { + assert!(is_code_publish_failure(diagnostic, root, lang), "{diagnostic}"); + } + for root in [root, Path::new("target/llm-runs/test-relative-module")] { + let source = std::path::absolute(root).unwrap().join("Lib.cs"); + let diagnostic = format!("{}(10,5): error CS0103: missing name", source.display()); + assert!(is_code_publish_failure(&diagnostic, root, Lang::CSharp), "{diagnostic}"); + } + for diagnostic in [ + "HTTP 503 Service Unavailable", + "authentication token expired", + "No space left on device", + "failed to download package", + "command dotnet publish exited with code 1", + "migration setup failed", + "unknown publish error", + "error: linking with `cc` failed", + "error: could not compile `test` due to 1 previous error", + "error[E0463]: can't find crate for `std`\n --> src/lib.rs:1:1", + "error[E0425]: cannot find value\n --> /dependencies/src/lib.rs:1:1", + "Lib.cs(1,1): error CS0006: Metadata file could not be found", + "/dependencies/Lib.cs(1,1): error CS0103: missing name", + "node_modules/package/index.ts(1,1): error TS2304: missing name", + ] { + for lang in [Lang::Rust, Lang::CSharp, Lang::TypeScript] { + assert!(!is_code_publish_failure(diagnostic, root, lang), "{diagnostic}"); + } + } + } + + #[test] + fn private_provider_errors_keep_their_retry_policy() { + use crate::llm::clients::oa_compat::OACompatResp; + for (error, expected) in [ + (serde_json::json!({"message": "rate limit private detail"}), Some(60)), + ( + serde_json::json!({"code": "429", "message": "private detail"}), + Some(60), + ), + (serde_json::json!({"code": 503, "message": "private detail"}), Some(30)), + (serde_json::json!({"code": 402, "message": "private detail"}), None), + ( + serde_json::json!({"message": "request timed out: private detail"}), + Some(30), + ), + ] { + let response: OACompatResp = serde_json::from_value(serde_json::json!({"error": error})).unwrap(); + let error = response.first_text().unwrap_err().context("model test"); + assert_eq!(provider_retry_delay(&error), expected); + assert!(!format!("{error:?}").contains("private detail")); + } + let refusal: OACompatResp = serde_json::from_value(serde_json::json!({ + "id": "gen-429", "choices": [{"finish_reason":"content_filter", "message":{"content":null}}] + })) + .unwrap(); + assert_eq!(provider_retry_delay(&refusal.first_text().unwrap_err()), None); + for (status, expected) in [(429, Some(60)), (503, Some(30)), (402, None)] { + let error = crate::llm::clients::http::HttpResponseError { + status, + body: "private detail".into(), + } + .into(); + assert_eq!(provider_retry_delay(&error), expected); + assert!(!format!("{error:?}").contains("private detail")); + } + } + #[test] fn provider_failures_reject_the_whole_batch_but_model_failures_remain_scores() { let task = || TaskPaths { @@ -1409,6 +1698,11 @@ mod build_cleanup_tests { }), ), (task(), Err(RunOneError::Other(anyhow::anyhow!("402 Payment Required")))), + (task(), Err(RunOneError::Other( + serde_json::from_value::(json!({ + "choices": [{"finish_reason": "content_filter", "message": {"content": null, "refusal": "provider refusal"}}] + })).unwrap().first_text().unwrap_err() + ))), ], "rust", &route, @@ -1416,6 +1710,7 @@ mod build_cleanup_tests { ) .unwrap(); assert_eq!(outcomes.len(), 1); + assert_eq!(pending.len(), 2, "refusals and provider failures are not scored zeroes"); assert_eq!(outcomes[0].passed_tests, 0); assert!(finalize_batch(outcomes.clone(), Vec::new()).is_ok()); let error = finalize_batch(outcomes, pending).unwrap_err().to_string(); @@ -1431,6 +1726,30 @@ mod build_cleanup_tests { .is_err()); } + #[tokio::test] + async fn scorer_panic_preserves_generated_code_as_unscored_evidence() { + struct PanickingScorer; + impl Scorer for PanickingScorer { + fn id(&self) -> &'static str { + "panic" + } + fn score(&self, _: &str) -> ScoreDetails { + panic!("test scorer failure") + } + } + let error = score_async(vec![Box::new(PanickingScorer)], "generated code".into(), None) + .await + .err() + .unwrap(); + match error { + RunOneError::Infrastructure { msg, llm_output } => { + assert!(msg.contains("test scorer failure")); + assert_eq!(llm_output.as_deref(), Some("generated code")); + } + other => panic!("expected unscored infrastructure error: {other:?}"), + } + } + #[test] fn slow_scoring_does_not_block_health_checks() { use std::io::{Read, Write}; @@ -1460,6 +1779,7 @@ mod build_cleanup_tests { loop { match listener.accept() { Ok((mut socket, _)) => { + socket.set_nonblocking(false).unwrap(); socket.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); let mut request = [0; 4096]; let n = socket.read(&mut request).unwrap(); diff --git a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs index b8bd6cd55e6..4bcc0426dc1 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs @@ -1,3 +1,5 @@ +use super::http::HttpResponseError; +use super::oa_compat::ProviderRefusal; use crate::bench::utils::debug_llm_verbose; use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ @@ -5,7 +7,7 @@ use crate::llm::segmentation::{ estimate_tokens, headroom_tokens_env, non_context_reserve_tokens_env, }; use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{anyhow, Context, Result}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue, CONTENT_TYPE}; use reqwest::{Client, StatusCode}; use serde::{Deserialize, Serialize}; @@ -115,15 +117,7 @@ impl AnthropicClient { let url = self.url_messages(); let (status, body) = self.post_with_retries(&url, hm, &req).await?; if debug_llm_verbose() { - let preview = if body.len() > 2000 { - format!("{}…", &body[..2000]) - } else { - body.clone() - }; - eprintln!("[anthropic] http_status={} body_preview={}", status, preview); - } - if !status.is_success() { - bail!("POST {} -> {}: {}", url, status, body); + eprintln!("[anthropic] http_status={}", status); } #[derive(Deserialize)] @@ -148,7 +142,19 @@ impl AnthropicClient { text: Option, } - let parsed: MsgResp = serde_json::from_str(&body).context("parse anthropic resp")?; + let value: serde_json::Value = serde_json::from_str(&body).context("parse anthropic resp")?; + if value["stop_reason"].as_str() == Some("refusal") { + return Err(ProviderRefusal { + generation_id: value["id"].as_str().map(str::to_owned), + provider: Some("Anthropic".into()), + finish_reason: Some("content_filter".into()), + native_finish_reason: Some("refusal".into()), + refusal: Some(body), + content: None, + } + .into()); + } + let parsed: MsgResp = serde_json::from_value(value).context("parse anthropic resp")?; let input_tokens = parsed.usage.as_ref().and_then(|u| u.input_tokens); let output_tokens = parsed.usage.as_ref().and_then(|u| u.output_tokens); let text = parsed @@ -194,7 +200,11 @@ impl AnthropicClient { || status == StatusCode::GATEWAY_TIMEOUT; if !retryable || attempt >= 10 { - bail!("POST {} -> {}: {}", url, status, body); + return Err(HttpResponseError { + status: status.as_u16(), + body, + } + .into()); } let wait = compute_backoff(status, &h, attempt); @@ -265,3 +275,59 @@ pub fn normalize_anthropic_model(id: &str) -> &str { _ => id, // return the original input; never the temporary } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn http_refusal_body_is_saved_but_not_logged() { + let (base, server) = super::super::http::mock_response(403, "private refusal details").await; + let client = AnthropicClient { + base, + api_key: "mock-key".into(), + client: Client::new(), + }; + let error = client + .post_with_retries(&client.url_messages(), HeaderMap::new(), &serde_json::json!({})) + .await + .unwrap_err(); + server.await.unwrap(); + assert!(!format!("{error:?}").contains("private refusal details")); + assert_eq!( + error + .downcast_ref::() + .unwrap() + .body, + "private refusal details" + ); + } + + #[tokio::test] + async fn successful_http_refusal_is_not_generated_code() { + let body = r#"{"id":"gen-block","stop_reason":"refusal","content":[{"type":"text","text":"private refusal details"}]}"#; + let (base, server) = super::super::http::mock_response(200, body).await; + let client = AnthropicClient { + base, + api_key: "mock-key".into(), + client: Client::new(), + }; + let prompt = BuiltPrompt { + system: None, + static_prefix: None, + segments: vec![], + search_enabled: false, + }; + let error = client + .generate("test-model", &prompt, ReasoningEffort::Medium) + .await + .unwrap_err(); + server.await.unwrap(); + let refusal = error + .downcast_ref::() + .unwrap(); + assert_eq!(refusal.generation_id.as_deref(), Some("gen-block")); + assert_eq!(refusal.refusal.as_deref(), Some(body)); + assert!(!format!("{error:?}").contains("private refusal details")); + } +} diff --git a/tools/xtask-llm-benchmark/src/llm/clients/google.rs b/tools/xtask-llm-benchmark/src/llm/clients/google.rs index f822be51e5f..811cea2ac81 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/google.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/google.rs @@ -1,16 +1,17 @@ use anyhow::{anyhow, Context, Result}; -use reqwest::header::HeaderMap; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; -use super::http::HttpClient; +use super::http::{HttpClient, HttpResponseError}; +use super::oa_compat::ProviderRefusal; use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, gemini_ctx_limit_tokens, non_context_reserve_tokens_env, Segment, }; use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; -/// Google uses API key in the query string rather than Authorization header. +/// Keep the API key in a header, out of URLs and transport-error diagnostics. #[derive(Clone)] pub struct GoogleGeminiClient { base: String, // e.g. https://generativelanguage.googleapis.com @@ -37,11 +38,13 @@ impl GoogleGeminiClient { static_prefix = deterministic_trim_prefix(&static_prefix, allowance); let url = format!( - "{}/v1beta/models/{}:generateContent?key={}", + "{}/v1beta/models/{}:generateContent", self.base.trim_end_matches('/'), urlencoding::encode(model), - urlencoding::encode(&self.api_key) ); + let mut api_key = HeaderValue::from_str(&self.api_key)?; + api_key.set_sensitive(true); + let headers = [(HeaderName::from_static("x-goog-api-key"), api_key)]; // ----- Request payload ----- #[derive(Serialize)] @@ -133,7 +136,7 @@ impl GoogleGeminiClient { // Use raw so we can inspect status/headers before consuming the body let resp = self .http - .post_json_raw(&url, &[], &req) + .post_json_raw(&url, &headers, &req) .await .with_context(|| format!("POST {} send failed", url))?; @@ -152,7 +155,11 @@ impl GoogleGeminiClient { || status == StatusCode::GATEWAY_TIMEOUT; if !retryable || attempt >= 10 { - return Err(anyhow!("POST {} -> {}: {}", url, status, text)); + return Err(HttpResponseError { + status: status.as_u16(), + body: text, + } + .into()); } let wait = compute_backoff_google(status, &headers, attempt); @@ -202,7 +209,39 @@ impl GoogleGeminiClient { } } - let resp: GeminiResp = serde_json::from_str(&body).context("parse gemini response")?; + let value: serde_json::Value = serde_json::from_str(&body).context("parse gemini response")?; + let blocked = value["promptFeedback"]["blockReason"] + .as_str() + .filter(|reason| !reason.is_empty() && *reason != "BLOCK_REASON_UNSPECIFIED") + .or_else(|| { + value["candidates"][0]["finishReason"].as_str().filter(|reason| { + matches!( + *reason, + "SAFETY" + | "RECITATION" + | "BLOCKLIST" + | "PROHIBITED_CONTENT" + | "SPII" + | "IMAGE_SAFETY" + | "IMAGE_PROHIBITED_CONTENT" + | "IMAGE_RECITATION" + | "ESCALATION" + | "PUP_LIMITED_DISABLED" + ) + }) + }); + if let Some(reason) = blocked { + return Err(ProviderRefusal { + generation_id: value["responseId"].as_str().map(str::to_owned), + provider: Some("Google".into()), + finish_reason: Some("content_filter".into()), + native_finish_reason: Some(reason.into()), + refusal: Some(body), + content: None, + } + .into()); + } + let resp: GeminiResp = serde_json::from_value(value).context("parse gemini response")?; let input_tokens = resp.usage_metadata.as_ref().and_then(|u| u.prompt_token_count); let output_tokens = resp.usage_metadata.as_ref().and_then(|u| u.candidates_token_count); let text = resp.first_text().ok_or_else(|| anyhow!("no text in Gemini response"))?; @@ -244,3 +283,67 @@ fn compute_backoff_google(status: StatusCode, headers: &HeaderMap, attempt: u32) fn header_f64(h: &HeaderMap, k: &str) -> Option { h.get(k).and_then(|v| v.to_str().ok())?.trim().parse::().ok() } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn request_errors_keep_bodies_private_and_keys_out_of_urls() { + let (base, server) = super::super::http::mock_response(403, "private refusal details").await; + let client = GoogleGeminiClient::new(HttpClient::new().unwrap(), base, "test-secret-key".into()); + let prompt = BuiltPrompt { + system: None, + static_prefix: None, + segments: vec![Segment::new("user", "test")], + search_enabled: false, + }; + let error = client + .generate("test-model", &prompt, ReasoningEffort::Medium) + .await + .unwrap_err(); + let request = server.await.unwrap(); + assert!(!request.lines().next().unwrap().contains("test-secret-key")); + assert!(request.to_ascii_lowercase().contains("x-goog-api-key: test-secret-key")); + let diagnostic = format!("{error:?}"); + assert!(!diagnostic.contains("test-secret-key")); + assert!(!diagnostic.contains("private refusal details")); + assert_eq!( + error + .downcast_ref::() + .unwrap() + .body, + "private refusal details" + ); + } + + #[tokio::test] + async fn successful_http_refusals_remain_unscored_with_full_evidence() { + for body in [ + r#"{"responseId":"gen-block","promptFeedback":{"blockReason":"SAFETY","blockReasonMessage":"private detail"}}"#, + r#"{"responseId":"gen-block","candidates":[{"finishReason":"SAFETY","content":{"parts":[{"text":"partial code"}]}}]}"#, + ] { + let (base, server) = super::super::http::mock_response(200, body).await; + let client = GoogleGeminiClient::new(HttpClient::new().unwrap(), base, "mock-key".into()); + let prompt = BuiltPrompt { + system: None, + static_prefix: None, + segments: vec![Segment::new("user", "test")], + search_enabled: false, + }; + let error = client + .generate("test-model", &prompt, ReasoningEffort::Medium) + .await + .unwrap_err(); + server.await.unwrap(); + let refusal = error + .downcast_ref::() + .unwrap(); + assert_eq!(refusal.generation_id.as_deref(), Some("gen-block")); + assert_eq!(refusal.native_finish_reason.as_deref(), Some("SAFETY")); + assert_eq!(refusal.refusal.as_deref(), Some(body)); + assert!(!format!("{error:?}").contains("private detail")); + assert!(!format!("{error:?}").contains("partial code")); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/llm/clients/http.rs b/tools/xtask-llm-benchmark/src/llm/clients/http.rs index 8e061cfae4a..f5a9eb59a6c 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/http.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/http.rs @@ -5,6 +5,66 @@ use reqwest::{ }; use std::time::Duration; +#[derive(serde::Serialize)] +pub(crate) struct HttpResponseError { + pub status: u16, + pub body: String, +} + +impl std::fmt::Display for HttpResponseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "provider HTTP {} (response body omitted from console)", self.status) + } +} + +impl std::fmt::Debug for HttpResponseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self, f) + } +} + +impl std::error::Error for HttpResponseError {} + +#[cfg(test)] +pub(super) async fn mock_response(status: u16, body: &'static str) -> (String, tokio::task::JoinHandle) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + tokio::time::timeout(Duration::from_secs(10), async { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + loop { + let mut buf = [0; 4096]; + let n = socket.read(&mut buf).await.unwrap(); + assert!(n > 0); + request.extend_from_slice(&buf[..n]); + if let Some(end) = request.windows(4).position(|w| w == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&request[..end]).to_ascii_lowercase(); + let length: usize = headers + .lines() + .find_map(|line| line.strip_prefix("content-length: ")) + .unwrap() + .parse() + .unwrap(); + if request.len() >= end + 4 + length { + break; + } + } + } + let response = format!( + "HTTP/1.1 {status} Mock\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + socket.write_all(response.as_bytes()).await.unwrap(); + String::from_utf8(request).unwrap() + }) + .await + .unwrap() + }); + (base, server) +} + #[derive(Clone)] pub struct HttpClient { inner: Client, @@ -40,19 +100,11 @@ impl HttpClient { let status = resp.status(); if !status.is_success() { let bytes = resp.bytes().await.context("read error response body")?; - let body_preview = String::from_utf8_lossy(&bytes); - let body_str = body_preview.trim(); - let body_short: String = if body_str.len() > 500 { - format!("{}...", &body_str[..500]) - } else { - body_str.into() - }; - anyhow::bail!( - "GET {url} status {status} body: {body_short}", - url = url, - status = status, - body_short = body_short - ); + return Err(HttpResponseError { + status: status.as_u16(), + body: String::from_utf8_lossy(&bytes).into_owned(), + } + .into()); } let bytes = resp.bytes().await.context("read GET body")?; @@ -87,19 +139,11 @@ impl HttpClient { let status = resp.status(); if !status.is_success() { let bytes = resp.bytes().await.context("read error response body")?; - let body_preview = String::from_utf8_lossy(&bytes); - let body_str = body_preview.trim(); - let body_short: String = if body_str.len() > 500 { - format!("{}...", &body_str[..500]) - } else { - body_str.into() - }; - anyhow::bail!( - "POST {url} status {status} body: {body_short}", - url = url, - status = status, - body_short = body_short - ); + return Err(HttpResponseError { + status: status.as_u16(), + body: String::from_utf8_lossy(&bytes).into_owned(), + } + .into()); } let bytes = resp.bytes().await.context("read POST body")?; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs b/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs index 9b427d1a48c..0e64459ccdc 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs @@ -1,8 +1,73 @@ //! Shared types for OpenAI-compatible chat completions responses. //! Used by DeepSeek, Meta/OpenRouter, and OpenRouter clients. -use anyhow::{anyhow, bail, Result}; -use serde::Deserialize; +use anyhow::{anyhow, Result}; +use serde::{Deserialize, Serialize}; + +/// Retain provider evidence in local artifacts, never in console error messages. +#[derive(Serialize)] +pub(crate) struct ProviderRefusal { + pub generation_id: Option, + pub provider: Option, + pub finish_reason: Option, + pub native_finish_reason: Option, + pub refusal: Option, + pub content: Option, +} + +impl std::fmt::Display for ProviderRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "provider refusal (id={}, provider={}, finish_reason={}, native_finish_reason={}, refusal=true)", + diagnostic_field(self.generation_id.as_deref()), + diagnostic_field(self.provider.as_deref()), + diagnostic_field(self.finish_reason.as_deref()), + diagnostic_field(self.native_finish_reason.as_deref()) + ) + } +} + +impl std::fmt::Debug for ProviderRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self, f) + } +} + +impl std::error::Error for ProviderRefusal {} + +#[derive(Serialize)] +pub(crate) struct ProviderResponseError { + diagnostic: String, + pub error: OAError, + pub content: Option, + pub refusal: Option, + pub native_finish_reason: Option, +} + +impl std::fmt::Display for ProviderResponseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "upstream error ({}, code={})", + self.diagnostic, + self.error + .code + .as_ref() + .and_then(serde_json::Value::as_u64) + .map(|code| code.to_string()) + .unwrap_or_else(|| "unknown".into()) + ) + } +} + +impl std::fmt::Debug for ProviderResponseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self, f) + } +} + +impl std::error::Error for ProviderResponseError {} #[derive(Debug, Deserialize)] pub struct OACompatResp { @@ -18,7 +83,7 @@ pub struct OACompatResp { pub error: Option, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct OAError { pub code: Option, pub message: Option, @@ -28,6 +93,7 @@ pub struct OAError { pub struct Choice { pub message: Option, pub finish_reason: Option, + pub native_finish_reason: Option, pub error: Option, } @@ -59,15 +125,37 @@ impl OACompatResp { .and_then(|c| c.message.as_ref()) .is_some_and(|m| m.refusal.is_some()), ); - if let Some(error) = self.error.as_ref().or_else(|| choice.and_then(|c| c.error.as_ref())) { - bail!( - "upstream error ({diagnostic}, code={}): {}", - diagnostic_field(error.code.as_ref().map(|v| v.to_string()).as_deref()), - diagnostic_field(error.message.as_deref()), - ); + let error = self.error.as_ref().or_else(|| choice.and_then(|c| c.error.as_ref())); + if error.is_some() || finish == Some("error") { + return Err(ProviderResponseError { + diagnostic, + error: error.cloned().unwrap_or_else(|| OAError { + code: None, + message: Some("unsuccessful completion".into()), + }), + content: choice.and_then(|c| c.message.as_ref()).and_then(|m| m.content.clone()), + refusal: choice.and_then(|c| c.message.as_ref()).and_then(|m| m.refusal.clone()), + native_finish_reason: choice.and_then(|c| c.native_finish_reason.clone()), + } + .into()); } - if matches!(finish, Some("error" | "content_filter")) { - bail!("unsuccessful completion ({diagnostic})"); + if let Some(choice) = choice + && (finish == Some("content_filter") + || choice.native_finish_reason.as_deref() == Some("refusal") + || choice + .message + .as_ref() + .is_some_and(|m| m.refusal.as_deref().is_some_and(|text| !text.trim().is_empty()))) + { + return Err(ProviderRefusal { + generation_id: self.id.clone(), + provider: self.provider.clone(), + finish_reason: choice.finish_reason.clone(), + native_finish_reason: choice.native_finish_reason.clone(), + refusal: choice.message.as_ref().and_then(|m| m.refusal.clone()), + content: choice.message.as_ref().and_then(|m| m.content.clone()), + } + .into()); } self.choices .into_iter() @@ -87,3 +175,45 @@ fn diagnostic_field(value: Option<&str>) -> String { .take(512) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_refusal_field_does_not_reject_valid_content() { + let response: OACompatResp = serde_json::from_value(serde_json::json!({ + "choices": [{"finish_reason": "stop", "message": {"content": "valid code", "refusal": ""}}] + })) + .unwrap(); + assert_eq!(response.first_text().unwrap(), "valid code"); + } + + #[test] + fn refusals_preserve_details_without_logging_text_or_scoring_partial_code() { + for (finish, native, refusal) in [ + ("content_filter", "refusal", Some("full refusal details")), + ("stop", "refusal", None), + ("stop", "stop", Some("full refusal details")), + ] { + let response: OACompatResp = serde_json::from_value(serde_json::json!({ + "id": "generation-1", "provider": "test-provider", + "choices": [{"finish_reason": finish, "native_finish_reason": native, + "message": {"content": "partial code", "refusal": refusal}}] + })) + .unwrap(); + let error = response.first_text().unwrap_err().context("model test"); + let detail = error + .downcast_ref::() + .expect("typed refusal survives context"); + assert_eq!(detail.generation_id.as_deref(), Some("generation-1")); + assert_eq!(detail.provider.as_deref(), Some("test-provider")); + assert_eq!(detail.native_finish_reason.as_deref(), Some(native)); + assert_eq!(detail.refusal.as_deref(), refusal); + assert_eq!(detail.content.as_deref(), Some("partial code")); + let logged = format!("{error:?} {error:#}"); + assert!(!logged.contains("full refusal details")); + assert!(!logged.contains("partial code")); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs index 20fb60cbdb8..4c8adb0eb06 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs @@ -238,7 +238,7 @@ impl OpenRouterClient { .http .post_json(&url, &[auth], &req) .await - .with_context(|| format!("OpenRouter POST {}", url))?; + .with_context(|| format!("OpenRouter POST {url} (model={model})"))?; let resp: OACompatResp = serde_json::from_str(&body).with_context(|| format!("parse OpenRouter response (model={model})"))?; @@ -428,6 +428,10 @@ mod tests { use std::time::Duration; let cases = [ + ( + r#"{"error":{"message":"refusal text must not be logged"}}"#, + Some("HTTP 403"), + ), ( r#"{"choices":[{"message":{"content":"valid code"}}],"usage":{"prompt_tokens":10,"completion_tokens":2}}"#, None, @@ -438,11 +442,11 @@ mod tests { ), ( r#"{"id":"gen-error","choices":[{"finish_reason":"error","message":{"content":"partial code"},"error":{"code":503,"message":"upstream unavailable"}}]}"#, - Some("upstream unavailable"), + Some("gen-error"), ), ( r#"{"id":"gen-top","error":{"code":429,"message":"rate limited"}}"#, - Some("rate limited"), + Some("gen-top"), ), ( r#"{"id":"gen-empty","choices":[{"finish_reason":"stop","message":{"content":" "}}]}"#, @@ -490,7 +494,12 @@ mod tests { } } } - write!(socket, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body).unwrap(); + let status = if expected_error == Some("HTTP 403") { + "403 Forbidden" + } else { + "200 OK" + }; + write!(socket, "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", body.len(), body).unwrap(); }); let client = OpenRouterClient::with_base(HttpClient::new().unwrap(), base, "mock-key".into()); let prompt = BuiltPrompt { @@ -503,7 +512,33 @@ mod tests { server.join().unwrap(); match expected_error { Some(expected) => { - let error = format!("{:#}", result.expect_err("response must not be scored")); + let error = result.expect_err("response must not be scored"); + if expected == "HTTP 403" { + let response = error.downcast_ref::().unwrap(); + assert_eq!(response.status, 403); + assert!(response.body.contains("refusal text")); + assert!(!format!("{error:?}").contains("refusal text")); + } + if expected == "content_filter" { + let refusal = error + .downcast_ref::() + .unwrap(); + assert_eq!(refusal.generation_id.as_deref(), Some("gen-filter")); + assert_eq!(refusal.refusal.as_deref(), Some("refusal text must not be logged")); + } + if matches!(expected, "gen-error" | "gen-top") { + let response = error + .downcast_ref::() + .unwrap(); + let expected_message = if expected == "gen-error" { + "upstream unavailable" + } else { + "rate limited" + }; + assert_eq!(response.error.message.as_deref(), Some(expected_message)); + assert!(!format!("{error:?}").contains(expected_message)); + } + let error = format!("{error:#}"); assert!(error.contains(expected), "{error}"); assert!(error.contains("test/model"), "{error}"); assert!( From 214e8cc139ab09a4f3cc90a06c0bd0842bc0d20c Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Fri, 30 Oct 2026 21:12:08 -0400 Subject: [PATCH 08/41] Bound C# benchmark reference database lifetimes --- .../llm-benchmark-validate-goldens.yml | 14 +- tools/xtask-llm-benchmark/README.md | 9 + .../xtask-llm-benchmark/src/bench/records.rs | 60 ++++- tools/xtask-llm-benchmark/src/bench/runner.rs | 246 +++++++++++++++++- tools/xtask-llm-benchmark/src/bench/types.rs | 8 +- .../src/bin/llm_benchmark.rs | 7 +- 6 files changed, 330 insertions(+), 14 deletions(-) diff --git a/.github/workflows/llm-benchmark-validate-goldens.yml b/.github/workflows/llm-benchmark-validate-goldens.yml index b2709a3a98c..74bef1948ea 100644 --- a/.github/workflows/llm-benchmark-validate-goldens.yml +++ b/.github/workflows/llm-benchmark-validate-goldens.yml @@ -27,7 +27,7 @@ concurrency: jobs: validate-goldens: runs-on: spacetimedb-linux - timeout-minutes: 60 + timeout-minutes: ${{ matrix.lang == 'csharp' && 180 || 60 }} env: CARGO_TARGET_DIR: ${{ github.workspace }}/target TARGET_TRIPLE: x86_64-unknown-linux-gnu @@ -121,5 +121,15 @@ jobs: MSBUILDDISABLENODEREUSE: "1" DOTNET_CLI_USE_MSBUILD_SERVER: "0" LLM_BENCH_CSHARP_CONCURRENCY: "1" + LLM_BENCHMARK_DIAGNOSTICS_DIR: ${{ runner.temp }}/llm-golden-diagnostics run: | - llm_benchmark run --goldens-only --lang ${{ matrix.lang }} + llm_benchmark run --goldens-only --lang ${{ matrix.lang }} --dry-run --skip-task-catalog-upload + + - name: Upload server diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: llm-golden-diagnostics-${{ matrix.lang }} + path: ${{ runner.temp }}/llm-golden-diagnostics + if-no-files-found: ignore + retention-days: 14 diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index 3e3c12c9a0e..a0a884df759 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -10,6 +10,15 @@ still run, but the language job exits with failure if any batch fails. Response errors include the generation ID and finish reason when supplied by the provider. Generated code that fails compilation or grading remains a normal benchmark score. +C# uses a fresh server per model batch. Each active task publishes its own reference +database and deletes it after scoring or a provider failure. This bounds live C# +references by task concurrency, rather than loading all 95 before model generation. +`--goldens-only` also validates and deletes each C# reference/candidate pair before +starting the next task. Rust and TypeScript keep their batched reference builds. +Cleanup errors block the batch, but completed scoring remains in the local record. +Compiler errors from SDK-generated files remain unscored; +they can expose SDK defects and must not automatically become model failures. + Each batch also saves an append-only JSONL record under `LLM_BENCHMARK_REPORT_DIR/batches`, or `tools/xtask-llm-benchmark/runs/batches` when that variable is not set. CI retains these files in its analysis artifacts, diff --git a/tools/xtask-llm-benchmark/src/bench/records.rs b/tools/xtask-llm-benchmark/src/bench/records.rs index 2d09a6166d0..643d9a51152 100644 --- a/tools/xtask-llm-benchmark/src/bench/records.rs +++ b/tools/xtask-llm-benchmark/src/bench/records.rs @@ -77,7 +77,10 @@ impl BatchRecord { json!({"status": "scored", "outcome": outcome}), ), Err(RunOneError::Other(error)) => match error.downcast_ref::() { - Some(refusal) => (TaskState::Refused, json!({"status": "refused", "refusal": refusal})), + Some(refusal) => ( + TaskState::Refused, + json!({"status": "refused", "refusal": refusal, "error": format!("{error:#}")}), + ), None => ( TaskState::ProviderError, json!({"status": "provider_or_setup_error", "error": format!("{error:#}"), @@ -85,9 +88,17 @@ impl BatchRecord { "http_response": error.downcast_ref::()}), ), }, - Err(RunOneError::Infrastructure { msg, llm_output }) => ( + Err(RunOneError::Infrastructure { + msg, + llm_output, + provider_error, + completed_outcome, + }) => ( TaskState::InfrastructureError, - json!({"status": "infrastructure_error", "error": msg, "llm_output": llm_output}), + json!({"status": "infrastructure_error", "error": msg, "llm_output": llm_output, "outcome": completed_outcome, + "refusal": provider_error.as_ref().and_then(|error| error.downcast_ref::()), + "provider_response": provider_error.as_ref().and_then(|error| error.downcast_ref::()), + "http_response": provider_error.as_ref().and_then(|error| error.downcast_ref::())}), ), Err(RunOneError::WithOutput { .. }) => bail!("normalize generated-code failures before recording"), }; @@ -218,6 +229,8 @@ mod tests { &Err(RunOneError::Infrastructure { msg: "server unavailable".into(), llm_output: Some("unscored code".into()), + provider_error: None, + completed_outcome: None, }), ) .unwrap(); @@ -228,6 +241,47 @@ mod tests { assert_eq!(last["summary"]["infrastructure_errors"], 1); assert_eq!(last["summary"]["pending_tasks"], 1); assert_eq!(last["attempt"]["llm_output"], "unscored code"); + let refusal = serde_json::from_value::(json!({ + "id": "gen-cleanup", "provider": "test-provider", + "choices": [{"finish_reason": "content_filter", "message": {"content": null, "refusal": "private detail"}}] + })) + .unwrap() + .first_text() + .unwrap_err(); + record + .record( + "b", + &Err(RunOneError::Infrastructure { + msg: "reference database cleanup failed".into(), + llm_output: None, + provider_error: Some(refusal), + completed_outcome: None, + }), + ) + .unwrap(); + assert!(record.complete().is_err()); + let rows = read_rows(&record.path); + let last = rows.last().unwrap(); + assert_eq!(last["summary"]["infrastructure_errors"], 1); + assert_eq!(last["summary"]["scored_tasks"], 1); + assert_eq!(last["attempt"]["refusal"]["generation_id"], "gen-cleanup"); + assert_eq!(last["attempt"]["refusal"]["refusal"], "private detail"); + record + .record( + "b", + &Err(RunOneError::Infrastructure { + msg: "reference database cleanup failed".into(), + llm_output: Some("generated code".into()), + provider_error: None, + completed_outcome: Some(Box::new(outcome(3))), + }), + ) + .unwrap(); + assert!(record.complete().is_err()); + let rows = read_rows(&record.path); + let last = rows.last().unwrap(); + assert_eq!(last["attempt"]["outcome"]["passed_tests"], 3); + assert_eq!(last["attempt"]["outcome"]["llm_output"], "generated code"); drop(record); std::fs::remove_dir_all(root).unwrap(); } diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 47f8b761a18..5df4a7c5e28 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -88,7 +88,7 @@ fn record_task_result( // A record write failure also aborts the batch before any website upload. record.record(&task.root.to_string_lossy(), &result)?; match result { - Err(RunOneError::Infrastructure { msg, llm_output }) => { + Err(RunOneError::Infrastructure { msg, llm_output, .. }) => { let output_note = llm_output .as_deref() .map(|output| format!("; generated output retained in work directory ({} bytes)", output.len())) @@ -164,6 +164,8 @@ async fn score_async( .map_err(|error| RunOneError::Infrastructure { msg: format!("scoring worker failed: {error:#}"), llm_output: Some(failure_output), + provider_error: None, + completed_outcome: None, }) } @@ -390,6 +392,30 @@ impl TaskRunner { } pub async fn run_one(&self, task: &TaskPaths, cfg: &RunContext<'_>) -> Result { + if cfg.lang != Lang::CSharp { + return self.run_one_prepared(task, cfg).await; + } + // A C# reference hosts a .NET runtime. Keep only references for active tasks loaded. + let scope = run_scope_tag(cfg.mode, cfg.route.vendor.slug(), &cfg.route.api_model); + let references = build_goldens_only_for_lang( + cfg.host.clone(), + &self.bench_root, + cfg.lang, + Some(&[task_slug(&task.root)]), + &scope, + ) + .await + .map_err(|error| RunOneError::Infrastructure { + msg: format!("reference setup failed for {}: {error:#}", task.root.display()), + llm_output: None, + provider_error: None, + completed_outcome: None, + })?; + let result = self.run_one_prepared(task, cfg).await; + finish_reference_cleanup(result, delete_databases(references).await) + } + + async fn run_one_prepared(&self, task: &TaskPaths, cfg: &RunContext<'_>) -> Result { let wall = Instant::now(); let started = Utc::now(); @@ -472,6 +498,8 @@ impl TaskRunner { load_migration_setup_source(task, cfg.lang).map_err(|error| RunOneError::Infrastructure { msg: format!("load migration setup failed: {error:#}"), llm_output: Some(llm_output.clone()), + provider_error: None, + completed_outcome: None, })?; if debug_llm() { @@ -500,6 +528,8 @@ impl TaskRunner { |error| RunOneError::Infrastructure { msg: format!("migration seed failed: {error}"), llm_output: Some(llm_output.clone()), + provider_error: None, + completed_outcome: None, }, )?; Ok(Some(database)) @@ -515,6 +545,8 @@ impl TaskRunner { return Err(RunOneError::Infrastructure { msg: format!("migration setup failed: {error:#}"), llm_output: Some(llm_output), + provider_error: None, + completed_outcome: None, }) } Ok(mut setup_database) => { @@ -543,6 +575,8 @@ impl TaskRunner { return Err(RunOneError::Infrastructure { msg: format!("publish failed without a source-code diagnostic: {error:#}"), llm_output: Some(llm_output), + provider_error: None, + completed_outcome: None, }); } eprintln!( @@ -561,6 +595,8 @@ impl TaskRunner { return Err(RunOneError::Infrastructure { msg: format!("publish failed and the benchmark server is unavailable: {error:#}"), llm_output: Some(llm_output), + provider_error: None, + completed_outcome: None, }); } @@ -604,6 +640,8 @@ impl TaskRunner { return Err(RunOneError::Infrastructure { msg: format!("database cleanup failed for `{llm_db}`: {error:#}"), llm_output: Some(llm_output), + provider_error: None, + completed_outcome: None, }); } @@ -661,6 +699,47 @@ impl TaskRunner { } } +fn finish_reference_cleanup( + result: Result, + cleanup: Result<()>, +) -> Result { + let Err(error) = cleanup else { + return result; + }; + let cleanup = format!("reference database cleanup failed: {error:#}"); + match result { + Err(RunOneError::Other(error)) => Err(RunOneError::Infrastructure { + msg: format!("{cleanup}; provider request also failed: {error:#}"), + llm_output: None, + provider_error: Some(error), + completed_outcome: None, + }), + Err(RunOneError::Infrastructure { + msg, + llm_output, + provider_error, + completed_outcome, + }) => Err(RunOneError::Infrastructure { + msg: format!("{msg}; {cleanup}"), + llm_output, + provider_error, + completed_outcome, + }), + Err(RunOneError::WithOutput { msg, llm_output }) => Err(RunOneError::Infrastructure { + msg: format!("{msg}; {cleanup}"), + llm_output: Some(llm_output), + provider_error: None, + completed_outcome: None, + }), + Ok(outcome) => Err(RunOneError::Infrastructure { + msg: cleanup, + llm_output: outcome.llm_output.clone(), + provider_error: None, + completed_outcome: Some(Box::new(outcome)), + }), + } +} + /// Partition task results into completed outcomes and retry candidates. /// Tasks where the LLM never responded (provider/setup error) are returned for retry. /// Tasks where the LLM responded but the code failed to compile/score are kept as outcomes. @@ -688,7 +767,7 @@ fn partition_results( llm_output, )); } - Err(RunOneError::Infrastructure { msg, llm_output }) => { + Err(RunOneError::Infrastructure { msg, llm_output, .. }) => { let output_note = llm_output .as_deref() .map(|output| format!("; generated output retained in work directory ({} bytes)", output.len())) @@ -1285,7 +1364,11 @@ pub async fn validate_goldens_for_lang( selectors: Option<&[String]>, ) -> Result<()> { let route_tag = "golden-validation"; - let golden_databases = build_goldens_only_for_lang(host.clone(), bench_root, lang, selectors, route_tag).await?; + let shared_goldens = if lang == Lang::CSharp { + Vec::new() + } else { + build_goldens_only_for_lang(host.clone(), bench_root, lang, selectors, route_tag).await? + }; let tasks = if let Some(sels) = selectors { let wanted: HashSet = sels.iter().map(|s| normalize_task_selector(s)).collect::>()?; @@ -1317,6 +1400,18 @@ pub async fn validate_goldens_for_lang( // Scorers can mutate both databases, so validate one task at a time and // preserve the scorer ordering declared by its spec. for task in tasks { + let golden_databases = if lang == Lang::CSharp { + build_goldens_only_for_lang( + host.clone(), + bench_root, + lang, + Some(&[task_slug(&task.root)]), + route_tag, + ) + .await? + } else { + Vec::new() + }; let category = category_slug(&task.root); let task_id = task_slug(&task.root); let candidate_db = sanitize_db_name(&format!("{}-{}-{}-llm", category, task_id, route_tag)); @@ -1396,9 +1491,9 @@ pub async fn validate_goldens_for_lang( } } delete_database_async(candidate_database.expect("candidate database retained during scoring")).await?; + delete_databases(golden_databases).await?; } - - delete_databases(golden_databases).await?; + delete_databases(shared_goldens).await?; if !failures.is_empty() { bail!( @@ -1557,6 +1652,98 @@ fn normalize_task_selector(raw: &str) -> Result { mod build_cleanup_tests { use super::*; + #[test] + #[ignore = "requires local server binaries and packed C# SDK; no model calls or uploads"] + fn csharp_references_are_scoped_to_successful_and_failed_tasks() { + struct LocalProvider { + host: String, + reference: String, + output: Option, + } + #[async_trait::async_trait] + impl crate::llm::LlmProvider for LocalProvider { + async fn preflight_route(&self, _: &ModelRoute, _: bool) -> Result<()> { + Ok(()) + } + async fn generate( + &self, + _: &ModelRoute, + _: &crate::llm::prompt::BuiltPrompt, + ) -> Result { + reqwest::Client::new() + .get(format!("{}/v1/database/{}/schema?version=9", self.host, self.reference)) + .send() + .await? + .error_for_status()?; + Ok(crate::llm::LlmOutput { + text: self.output.clone().context("injected provider failure")?, + input_tokens: None, + output_tokens: None, + }) + } + } + let server = spacetimedb_guard::SpacetimeDbGuard::try_spawn_in_temp_data_dir().unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/benchmarks"); + let task = discover_tasks(&root) + .unwrap() + .into_iter() + .find(|task| task.root.ends_with("basics/t_004_insert")) + .unwrap(); + let route = ModelRoute::new( + "local-fixture", + crate::llm::types::Vendor::OpenAi, + "local-fixture", + None, + ); + let scope = run_scope_tag("no_context", route.vendor.slug(), &route.api_model); + let reference = golden_db_name("basics", "t_004_insert", &scope); + let runner = TaskRunner::new(root, SpacetimeRustPublisher, DotnetPublisher, TypeScriptPublisher); + let source = load_golden_source(&task, Lang::CSharp).unwrap(); + for output in [Some(source), Some("this is invalid C#".into()), None] { + let provider = LocalProvider { + host: server.host_url.clone(), + reference: reference.clone(), + output, + }; + let cfg = RunContext { + lang_name: "csharp", + lang: Lang::CSharp, + mode: "no_context", + route: &route, + context: "", + hash: "local-validation", + llm: &provider, + host: Some(server.host_url.clone()), + }; + let result = runtime.block_on(runner.run_one(&task, &cfg)); + match provider.output.as_deref() { + None => assert!(matches!(result, Err(RunOneError::Other(_)))), + Some("this is invalid C#") => { + let outcome = result.unwrap(); + assert_eq!(outcome.passed_tests, 0); + } + Some(_) => { + let outcome = result.unwrap(); + assert!(outcome.total_tests > 0); + assert_eq!(outcome.passed_tests, outcome.total_tests); + } + } + let status = runtime.block_on(async { + reqwest::Client::new() + .get(format!("{}/v1/database/{reference}/schema?version=9", server.host_url)) + .send() + .await + .unwrap() + .status() + }); + assert_eq!(status, reqwest::StatusCode::NOT_FOUND, "reference leaked after task"); + } + } + #[tokio::test] async fn failed_attempt_drains_started_work_without_starting_more_tasks() { let mut recorded = Vec::new(); @@ -1742,7 +1929,7 @@ mod build_cleanup_tests { .err() .unwrap(); match error { - RunOneError::Infrastructure { msg, llm_output } => { + RunOneError::Infrastructure { msg, llm_output, .. } => { assert!(msg.contains("test scorer failure")); assert_eq!(llm_output.as_deref(), Some("generated code")); } @@ -1750,6 +1937,53 @@ mod build_cleanup_tests { } } + #[test] + fn reference_cleanup_failure_preserves_unscored_evidence() { + let completed: RunOutcome = serde_json::from_value(json!({ + "hash": "test", "task": "a", "lang": "csharp", "model_name": "test", "vendor": "test", + "golden_published": true, "passed_tests": 1, "total_tests": 2, "llm_output": "candidate code", + "input_tokens": 123, "scorer_details": {"parity": {"pass": false, "partial": 0.5, "notes": {"difference": "retained"}}} + })).unwrap(); + let expected = serde_json::to_value(&completed).unwrap(); + match finish_reference_cleanup(Ok(completed), Err(anyhow!("delete failed"))) { + Err(RunOneError::Infrastructure { completed_outcome, .. }) => { + assert_eq!(serde_json::to_value(completed_outcome.unwrap()).unwrap(), expected); + } + other => panic!("completed scoring lost after cleanup failure: {other:?}"), + } + let failed_code = Err(RunOneError::WithOutput { + msg: "compile failed".into(), + llm_output: "candidate code".into(), + }); + match finish_reference_cleanup(failed_code, Err(anyhow!("delete failed"))) { + Err(RunOneError::Infrastructure { msg, llm_output, .. }) => { + assert!(msg.contains("compile failed") && msg.contains("delete failed")); + assert_eq!(llm_output.as_deref(), Some("candidate code")); + } + other => panic!("expected unscored cleanup failure: {other:?}"), + } + let refusal = serde_json::from_value::(json!({ + "choices": [{"finish_reason":"content_filter", "message":{"content":null,"refusal":"private detail"}}] + })) + .unwrap() + .first_text() + .unwrap_err(); + match finish_reference_cleanup(Err(RunOneError::Other(refusal)), Err(anyhow!("delete failed"))) { + Err(RunOneError::Infrastructure { + msg, provider_error, .. + }) => { + assert!(msg.contains("delete failed")); + assert!(!msg.contains("private detail")); + let error = provider_error.unwrap(); + let evidence = error + .downcast_ref::() + .unwrap(); + assert_eq!(evidence.refusal.as_deref(), Some("private detail")); + } + other => panic!("cleanup failure must stop the batch: {other:?}"), + } + } + #[test] fn slow_scoring_does_not_block_health_checks() { use std::io::{Read, Write}; diff --git a/tools/xtask-llm-benchmark/src/bench/types.rs b/tools/xtask-llm-benchmark/src/bench/types.rs index eb6a44a6696..3359f041186 100644 --- a/tools/xtask-llm-benchmark/src/bench/types.rs +++ b/tools/xtask-llm-benchmark/src/bench/types.rs @@ -136,7 +136,13 @@ pub enum RunOneError { #[error("{msg}")] WithOutput { msg: String, llm_output: String }, #[error("{msg}")] - Infrastructure { msg: String, llm_output: Option }, + Infrastructure { + msg: String, + llm_output: Option, + // Provider details stay in local evidence, not console messages. + provider_error: Option, + completed_outcome: Option>, + }, #[error(transparent)] Other(#[from] anyhow::Error), } diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index ce5150058ea..376a1dd5ecf 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -861,8 +861,11 @@ async fn run_many_routes_for_mode( .context("LLM provider preflight failed before model batch")?; let golden_scope = xtask_llm_benchmark::bench::run_scope_tag(mode, route.vendor.slug(), &route.api_model); - let golden_databases = - ensure_goldens_built_once(host.clone(), bench_root, lang, selectors, &golden_scope).await?; + let golden_databases = if lang == Lang::CSharp { + Vec::new() // C# references are owned and deleted by each active task. + } else { + ensure_goldens_built_once(host.clone(), bench_root, lang, selectors, &golden_scope).await? + }; let per = BenchRunContext { bench_root, From a92afc7f8f3df0c293320ec45716b1ad79900a1c Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Sat, 31 Oct 2026 02:56:04 -0400 Subject: [PATCH 09/41] Score rejected benchmark migrations as code failures A candidate that builds but needs a manual migration is a generated-code failure, not an infrastructure failure. Score it normally so one task cannot stop the batch. --- tools/xtask-llm-benchmark/README.md | 1 + tools/xtask-llm-benchmark/src/bench/runner.rs | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index a0a884df759..a84bce4776c 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -40,6 +40,7 @@ Zero scored tasks means no score, not 0%. An incomplete batch stays off the webs Migration setup failures and publish failures without a recognized source-code diagnostic also block the batch, even when the server responds to a health check. This is conservative: an unknown code error can block a batch until classified. +A candidate whose schema the server rejects as a manual migration is scored normally. Each finished attempt is saved before an error can stop the batch. If a task reports an infrastructure error, new tasks stop, but started tasks finish and save diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 5df4a7c5e28..fe2e2e522a1 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -243,6 +243,10 @@ fn provider_retry_delay(error: &anyhow::Error) -> Option { fn is_code_publish_failure(message: &str, source_root: &Path, lang: Lang) -> bool { // Unknown publish failures are unscored. A healthy ping does not prove a valid build environment. let message = message.replace('\\', "/").to_ascii_lowercase(); + // The candidate built, but its schema cannot migrate the task's existing data. + if message.contains("aborting publish due to required manual migration") { + return true; + } if [ "error[e0463]", "error[e0460]", @@ -1807,6 +1811,10 @@ mod build_cleanup_tests { let diagnostic = format!("{}(10,5): error CS0103: missing name", source.display()); assert!(is_code_publish_failure(&diagnostic, root, Lang::CSharp), "{diagnostic}"); } + let rejected_migration = "Errors occurred:\nAdding a column enabled to table widget requires a default value annotation\n\nAborting publish due to required manual migration."; + for lang in [Lang::Rust, Lang::CSharp, Lang::TypeScript] { + assert!(is_code_publish_failure(rejected_migration, root, lang)); + } for diagnostic in [ "HTTP 503 Service Unavailable", "authentication token expired", From b680ad8168cdf3d376dca4b73490ca340d7b07b9 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Sat, 31 Oct 2026 03:16:54 -0400 Subject: [PATCH 10/41] Score SDK-generated compile errors as code failures A candidate that fails in SDK-generated code still failed to build. Score it normally so one task cannot stop the batch, and keep the evidence so suspected SDK defects can go to the core team. --- tools/xtask-llm-benchmark/README.md | 4 ++-- tools/xtask-llm-benchmark/src/bench/runner.rs | 22 +++++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index a84bce4776c..501d00bf52c 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -16,8 +16,8 @@ references by task concurrency, rather than loading all 95 before model generati `--goldens-only` also validates and deletes each C# reference/candidate pair before starting the next task. Rust and TypeScript keep their batched reference builds. Cleanup errors block the batch, but completed scoring remains in the local record. -Compiler errors from SDK-generated files remain unscored; -they can expose SDK defects and must not automatically become model failures. +Compiler errors in SDK-generated files score as normal failures. Saved batch +records keep the evidence so suspected SDK defects can go to the core team. Each batch also saves an append-only JSONL record under `LLM_BENCHMARK_REPORT_DIR/batches`, or `tools/xtask-llm-benchmark/runs/batches` diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index fe2e2e522a1..a8cc0dae3e2 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -267,21 +267,25 @@ fn is_code_publish_failure(message: &str, source_root: &Path, lang: Lang) -> boo Lang::CSharp => "lib.cs", Lang::TypeScript => "src/index.ts", }; - let absolute = source_root + let root = source_root .canonicalize() .or_else(|_| std::path::absolute(source_root)) .unwrap_or_else(|_| source_root.to_path_buf()) - .join(relative) .to_string_lossy() .replace('\\', "/") .to_ascii_lowercase(); + let root = root.trim_start_matches("//?/"); + let absolute = format!("{root}/{relative}"); + let sdk_output = format!("{root}/obj/"); let is_generated_location = |location: &str| { let location = location.trim().trim_start_matches("//?/").trim_start_matches("./"); - [relative, absolute.trim_start_matches("//?/")].iter().any(|path| { - location - .strip_prefix(path) - .is_some_and(|rest| rest.starts_with(':') || rest.starts_with('(')) - }) + // SDK source generators compile the candidate's types from its own obj directory. + (matches!(lang, Lang::CSharp) && location.starts_with(&sdk_output)) + || [relative, absolute.as_str()].iter().any(|path| { + location + .strip_prefix(path) + .is_some_and(|rest| rest.starts_with(':') || rest.starts_with('(')) + }) }; let lines: Vec<_> = message.lines().map(str::trim).collect(); let mut found = false; @@ -1810,6 +1814,9 @@ mod build_cleanup_tests { let source = std::path::absolute(root).unwrap().join("Lib.cs"); let diagnostic = format!("{}(10,5): error CS0103: missing name", source.display()); assert!(is_code_publish_failure(&diagnostic, root, Lang::CSharp), "{diagnostic}"); + let generated = std::path::absolute(root).unwrap().join("obj/Release/Shape.cs"); + let diagnostic = format!("{}(27,33): error CS0144: abstract type", generated.display()); + assert!(is_code_publish_failure(&diagnostic, root, Lang::CSharp), "{diagnostic}"); } let rejected_migration = "Errors occurred:\nAdding a column enabled to table widget requires a default value annotation\n\nAborting publish due to required manual migration."; for lang in [Lang::Rust, Lang::CSharp, Lang::TypeScript] { @@ -1829,6 +1836,7 @@ mod build_cleanup_tests { "error[E0425]: cannot find value\n --> /dependencies/src/lib.rs:1:1", "Lib.cs(1,1): error CS0006: Metadata file could not be found", "/dependencies/Lib.cs(1,1): error CS0103: missing name", + "/dependencies/obj/Shape.cs(1,1): error CS0144: abstract type", "node_modules/package/index.ts(1,1): error TS2304: missing name", ] { for lang in [Lang::Rust, Lang::CSharp, Lang::TypeScript] { From c4a2f1c96d3e8ab7663e673987649956f72a9d8d Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Sat, 31 Oct 2026 15:34:39 -0400 Subject: [PATCH 11/41] Match schema parity tables across naming styles Prompts do not fix a table's casing, and the data checks find the table either way. A candidate that used the prompt's name, such as User against the reference's user, failed schema parity while passing every other check. Column types, keys, indexes and access are still compared exactly. --- tools/xtask-llm-benchmark/README.md | 2 + tools/xtask-llm-benchmark/src/eval/scorers.rs | 67 ++++++++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index 501d00bf52c..b4d72ffb518 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -18,6 +18,8 @@ starting the next task. Rust and TypeScript keep their batched reference builds. Cleanup errors block the batch, but completed scoring remains in the local record. Compiler errors in SDK-generated files score as normal failures. Saved batch records keep the evidence so suspected SDK defects can go to the core team. +Schema parity treats table names that differ only by case or underscores as the +same table, because prompts do not fix a table's casing. Each batch also saves an append-only JSONL record under `LLM_BENCHMARK_REPORT_DIR/batches`, or `tools/xtask-llm-benchmark/runs/batches` diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index d1533d7b448..598eff4279e 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -108,7 +108,7 @@ impl Scorer for SchemaParityScorer { let schema_a = extract_schema(&golden); let schema_b = extract_schema(&llm); - let tables_diff = diff_maps(&schema_a.tables, &schema_b.tables); + let tables_diff = diff_maps(&schema_a.tables, &align_table_names(&schema_a.tables, &schema_b.tables)); let reducers_diff = diff_sets(&schema_a.reducers, &schema_b.reducers); let exports_diff = diff_sets(&schema_a.exports, &schema_b.exports); let rls_diff = diff_sets(&schema_a.row_level_security, &schema_b.row_level_security); @@ -510,6 +510,24 @@ fn insert_schema_property(columns: &mut BTreeMap, name: &str, va } } +// Prompts do not fix a table's casing, so a name differing only by case or underscores is the same table. +// shortcut: `User` and `USER` beside a golden `user` collapse into one table; compare counts if a model ever does this. +fn align_table_names( + golden: &BTreeMap>, + candidate: &BTreeMap>, +) -> BTreeMap> { + let key = |name: &str| name.to_lowercase().replace('_', ""); + candidate + .iter() + .map(|(name, table)| { + let golden_name = golden + .keys() + .find(|golden_name| !candidate.contains_key(*golden_name) && key(golden_name) == key(name)); + (golden_name.unwrap_or(name).clone(), table.clone()) + }) + .collect() +} + fn diff_maps(a: &BTreeMap>, b: &BTreeMap>) -> Value { let mut only_a = BTreeMap::new(); let mut only_b = BTreeMap::new(); @@ -1440,6 +1458,53 @@ mod tests { assert!(!diff_maps(&golden.tables, &candidate.tables).is_null()); } + fn tables_named(names: &[&str]) -> BTreeMap> { + let table = extract_schema(¤t_schema(true)) + .tables + .remove("child_item") + .unwrap(); + names.iter().map(|name| (name.to_string(), table.clone())).collect() + } + + #[test] + fn table_names_differing_only_by_case_or_underscores_match() { + for (golden, candidate) in [("user", "User"), ("leased_job", "LeasedJob"), ("item_v_2", "item_v2")] { + let golden = tables_named(&[golden]); + let candidate = align_table_names(&golden, &tables_named(&[candidate])); + + assert!(diff_maps(&golden, &candidate).is_null()); + } + } + + #[test] + fn different_table_names_produce_a_schema_diff() { + let golden = tables_named(&["item_v_2"]); + let candidate = align_table_names(&golden, &tables_named(&["item_v3"])); + + assert!(!diff_maps(&golden, &candidate).is_null()); + } + + #[test] + fn extra_table_differing_only_by_case_produces_a_schema_diff() { + let golden = tables_named(&["user"]); + let candidate = align_table_names(&golden, &tables_named(&["User", "user"])); + + assert_eq!( + diff_maps(&golden, &candidate)["only_llm"]["User"]["id"], + r#"{"U64":[]}"# + ); + } + + #[test] + fn matched_table_still_compares_properties() { + let golden = extract_schema(¤t_schema(true)).tables; + let mut candidate = tables_named(&["ChildItem"]); + candidate.get_mut("ChildItem").unwrap().remove("@indexes"); + let candidate = align_table_names(&golden, &candidate); + + assert!(diff_maps(&golden, &candidate)["changed"]["child_item"].is_object()); + } + #[test] fn missing_default_value_produces_a_schema_diff() { let mut golden = current_schema(true); From c9b66225b78ea7a44fffbbb326b1291cc011ab0a Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Mon, 2 Nov 2026 09:17:16 -0500 Subject: [PATCH 12/41] Fail schema parity for the C# 1.x table form Name without Accessor is the 1.x form. The migration guide calls it no longer valid and Rust rejects it, but C# still compiles it, and the schema looks the same as a valid table that sets both. Check the source text so outdated code fails while valid 2.0 code passes. --- tools/xtask-llm-benchmark/README.md | 3 +- tools/xtask-llm-benchmark/src/eval/scorers.rs | 63 ++++++++++++++++++- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index b4d72ffb518..c6311d8fbe0 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -19,7 +19,8 @@ Cleanup errors block the batch, but completed scoring remains in the local recor Compiler errors in SDK-generated files score as normal failures. Saved batch records keep the evidence so suspected SDK defects can go to the core team. Schema parity treats table names that differ only by case or underscores as the -same table, because prompts do not fix a table's casing. +same table, because prompts do not fix a table's casing. It fails any C# table that +sets `Name` without `Accessor`, because that is the 1.x form. Each batch also saves an append-only JSONL record under `LLM_BENCHMARK_REPORT_DIR/batches`, or `tools/xtask-llm-benchmark/runs/batches` diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 598eff4279e..edc0a7e894e 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -1,9 +1,11 @@ use crate::bench::utils::debug_llm_verbose; use crate::eval::utils::{run_with_timeout, sql_exec_with_timeout}; use crate::eval::{normalize, spacetime_command, ScoreDetails}; +use regex::Regex; use serde_json::{json, Value}; use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; +use std::sync::LazyLock; use std::time::{Duration, Instant}; use std::{io, thread}; @@ -86,7 +88,7 @@ impl Scorer for SchemaParityScorer { self.id_str } - fn score(&self, _llm_output: &str) -> ScoreDetails { + fn score(&self, llm_output: &str) -> ScoreDetails { let golden = match describe_db(&self.server, &self.golden_db, self.timeout) { Ok(v) => v, Err(e) => return err_details("describe_golden", e), @@ -112,7 +114,12 @@ impl Scorer for SchemaParityScorer { let reducers_diff = diff_sets(&schema_a.reducers, &schema_b.reducers); let exports_diff = diff_sets(&schema_a.exports, &schema_b.exports); let rls_diff = diff_sets(&schema_a.row_level_security, &schema_b.row_level_security); - let pass = tables_diff.is_null() && reducers_diff.is_null() && exports_diff.is_null() && rls_diff.is_null(); + let name_without_accessor = has_table_name_without_accessor(llm_output); + let pass = tables_diff.is_null() + && reducers_diff.is_null() + && exports_diff.is_null() + && rls_diff.is_null() + && !name_without_accessor; ScoreDetails { pass, @@ -121,6 +128,7 @@ impl Scorer for SchemaParityScorer { "server": self.server, "golden_db": self.golden_db, "llm_db": self.llm_db, + "table_name_without_accessor": name_without_accessor, "tables_equal": tables_diff.is_null(), "reducers_equal": reducers_diff.is_null(), "exports_equal": exports_diff.is_null(), @@ -528,6 +536,23 @@ fn align_table_names( .collect() } +// shortcut: text match for the 1.x C# form, which the schema cannot show; delete once the C# SDK rejects it as Rust does. +fn has_table_name_without_accessor(source: &str) -> bool { + static COMMENT: LazyLock = LazyLock::new(|| Regex::new(r"(?s)/\*.*?\*/|//[^\n]*").unwrap()); + static TABLE: LazyLock = LazyLock::new(|| { + Regex::new(r"[\[,]\s*(?:global::)?(?:SpacetimeDB\.)?Table\s*\(([^()]*(?:\([^()]*\)[^()]*)*)\)").unwrap() + }); + + TABLE.captures_iter(&COMMENT.replace_all(source, "")).any(|table| { + let sets = |property: &str| { + table[1] + .split(',') + .any(|argument| argument.split('=').next().is_some_and(|key| key.trim() == property)) + }; + sets("Name") && !sets("Accessor") + }) +} + fn diff_maps(a: &BTreeMap>, b: &BTreeMap>) -> Value { let mut only_a = BTreeMap::new(); let mut only_b = BTreeMap::new(); @@ -1505,6 +1530,40 @@ mod tests { assert!(diff_maps(&golden, &candidate)["changed"]["child_item"].is_object()); } + #[test] + fn table_name_without_accessor_is_detected() { + for source in [ + r#"[Table(Name = "User", Public = true)]"#, + r#"[SpacetimeDB.Table(Name = "user")]"#, + r#"[global::SpacetimeDB.Table(Public = true, Name = "User")]"#, + r#"[Table(Name = "TickTimer", Scheduled = nameof(Tick))]"#, + "[Table(\n Name = \"Reminder\",\n Scheduled = nameof(SendReminder),\n ScheduledAt = nameof(Reminder.ScheduledAt))]", + "[Table(Accessor = \"A\")]\n[Table(Name = \"B\")]", + r#"[Table(Accessor = "A"), Table(Name = "B")]"#, + ] { + assert!(has_table_name_without_accessor(source), "{source}"); + } + } + + #[test] + fn valid_table_declarations_are_not_flagged() { + for source in [ + "[Table]", + "[Table(Public = true)]", + r#"[Table(Accessor = "User", Public = true)]"#, + r#"[SpacetimeDB.Table(Name = "Parent", Public = true, Accessor = "Parent")]"#, + r#"[Table(Accessor = "TickTimer", Scheduled = nameof(Tick))]"#, + r#"[SpacetimeDB.Index.BTree(Name = "ByOwner", Columns = new[] { nameof(Owner) })]"#, + r#"[View(Name = "ActiveUsers")]"#, + r#"// [Table(Name = "User")]"#, + "/* [Table(Name = \"User\")] */", + r#"#[table(accessor = user, name = "User")]"#, + r#"const user = table({ name: 'User' }, {});"#, + ] { + assert!(!has_table_name_without_accessor(source), "{source}"); + } + } + #[test] fn missing_default_value_produces_a_schema_diff() { let mut golden = current_schema(true); From e11e0b802937776faa519d0097e6ad4a87726806 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Mon, 2 Nov 2026 09:56:44 -0500 Subject: [PATCH 13/41] Score Rust compile errors and invalid modules as code failures The CLI's cargo wrapper line looked like a Rust diagnostic with no location, so every Rust compile error stopped the batch as an infrastructure failure. A module that builds but fails schema validation did the same. Both are wrong code, so score them normally. --- tools/xtask-llm-benchmark/README.md | 2 +- tools/xtask-llm-benchmark/src/bench/runner.rs | 21 +++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index c6311d8fbe0..361706082cb 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -43,7 +43,7 @@ Zero scored tasks means no score, not 0%. An incomplete batch stays off the webs Migration setup failures and publish failures without a recognized source-code diagnostic also block the batch, even when the server responds to a health check. This is conservative: an unknown code error can block a batch until classified. -A candidate whose schema the server rejects as a manual migration is scored normally. +A candidate whose schema fails validation or needs a manual migration is scored normally. Each finished attempt is saved before an error can stop the batch. If a task reports an infrastructure error, new tasks stop, but started tasks finish and save diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index a8cc0dae3e2..b5c4af108a6 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -243,8 +243,8 @@ fn provider_retry_delay(error: &anyhow::Error) -> Option { fn is_code_publish_failure(message: &str, source_root: &Path, lang: Lang) -> bool { // Unknown publish failures are unscored. A healthy ping does not prove a valid build environment. let message = message.replace('\\', "/").to_ascii_lowercase(); - // The candidate built, but its schema cannot migrate the task's existing data. - if message.contains("aborting publish due to required manual migration") { + // The candidate built, but its schema is invalid or cannot migrate the task's existing data. + if message.contains("errors occurred:") || message.contains("aborting publish due to required manual migration") { return true; } if [ @@ -292,8 +292,11 @@ fn is_code_publish_failure(message: &str, source_root: &Path, lang: Lang) -> boo for (index, line) in lines.iter().enumerate() { let rust_error = matches!(lang, Lang::Rust) && (line.starts_with("error[e") || line.starts_with("error:")); if rust_error { - // Cargo/rustc summaries add no source diagnostic of their own. - if line.starts_with("error: could not compile") || line.starts_with("error: aborting due to") { + // Cargo, rustc and CLI summaries add no source diagnostic of their own. + if line.starts_with("error: could not compile") + || line.starts_with("error: aborting due to") + || line.starts_with("error: command [\"cargo\"") + { continue; } let location = lines[index + 1..] @@ -1794,8 +1797,9 @@ mod build_cleanup_tests { "error: this file contains an unclosed delimiter", "error: unknown start of token: \\", ] { - let diagnostic = - format!("{diagnostic}\n --> src/lib.rs:10:5\nerror: could not compile `test` due to 1 previous error"); + let diagnostic = format!( + "{diagnostic}\n --> src/lib.rs:10:5\nerror: could not compile `test` due to 1 previous error\nError: command [\"cargo\", \"build\"] exited with code 101" + ); assert!(is_code_publish_failure(&diagnostic, root, Lang::Rust), "{diagnostic}"); } for (lang, diagnostic) in [ @@ -1819,8 +1823,10 @@ mod build_cleanup_tests { assert!(is_code_publish_failure(&diagnostic, root, Lang::CSharp), "{diagnostic}"); } let rejected_migration = "Errors occurred:\nAdding a column enabled to table widget requires a default value annotation\n\nAborting publish due to required manual migration."; + let invalid_schema = "Error: Local module schema inspection failed:\nError: Errors occurred:\nname `OrderLineDetail` is used for multiple types"; for lang in [Lang::Rust, Lang::CSharp, Lang::TypeScript] { assert!(is_code_publish_failure(rejected_migration, root, lang)); + assert!(is_code_publish_failure(invalid_schema, root, lang)); } for diagnostic in [ "HTTP 503 Service Unavailable", @@ -1830,7 +1836,10 @@ mod build_cleanup_tests { "command dotnet publish exited with code 1", "migration setup failed", "unknown publish error", + "Local module schema inspection timed out", "error: linking with `cc` failed", + "error: linking with `cc` failed\nError: command [\"cargo\", \"build\"] exited with code 101", + "Error: command [\"cargo\", \"build\"] exited with code 101", "error: could not compile `test` due to 1 previous error", "error[E0463]: can't find crate for `std`\n --> src/lib.rs:1:1", "error[E0425]: cannot find value\n --> /dependencies/src/lib.rs:1:1", From ad820d9a1f5ad963c8aebae328e6bfe7bf143722 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Mon, 2 Nov 2026 10:49:08 -0500 Subject: [PATCH 14/41] Score candidate publish failures unless the environment failed Model mistakes have no fixed list of messages, so treating every unrecognized failure as infrastructure kept stopping whole batches on code that was simply wrong. Score candidate failures by default, and stop only on a short list of environment errors or a failed server ping. A batch now completes only if known-good code still publishes at the end, so a build environment that breaks mid-run cannot upload misleading zeroes. --- tools/xtask-llm-benchmark/README.md | 11 +- tools/xtask-llm-benchmark/src/bench/runner.rs | 217 ++++++------------ 2 files changed, 80 insertions(+), 148 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index 361706082cb..806f925f238 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -40,10 +40,13 @@ task counts separate from scorer-check counts and always include coverage. For example, 99 passing tasks and one refusal means **99/99 scored tasks passed, with 99/100 tasks scored and one refusal**. It does not mean a complete 100% benchmark. Zero scored tasks means no score, not 0%. An incomplete batch stays off the website. -Migration setup failures and publish failures without a recognized source-code -diagnostic also block the batch, even when the server responds to a health check. -This is conservative: an unknown code error can block a batch until classified. -A candidate whose schema fails validation or needs a manual migration is scored normally. +Model mistakes have no fixed list of messages, so any candidate publish failure +scores normally unless it matches a short list of environment errors, such as a +full disk or a refused connection, or the server fails a health check afterward. +Migration setup failures also block the batch. A batch only completes if a known-good +module still publishes at the end, so a build environment that breaks mid-run cannot +upload misleading zeroes. A brief outage that matches none of these checks can score +one task as a failure; its error stays in the batch record. Each finished attempt is saved before an error can stop the batch. If a task reports an infrastructure error, new tasks stop, but started tasks finish and save diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index b5c4af108a6..ec305c986d9 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -240,83 +240,46 @@ fn provider_retry_delay(error: &anyhow::Error) -> Option { } } -fn is_code_publish_failure(message: &str, source_root: &Path, lang: Lang) -> bool { - // Unknown publish failures are unscored. A healthy ping does not prove a valid build environment. - let message = message.replace('\\', "/").to_ascii_lowercase(); - // The candidate built, but its schema is invalid or cannot migrate the task's existing data. - if message.contains("errors occurred:") || message.contains("aborting publish due to required manual migration") { - return true; - } - if [ - "error[e0463]", - "error[e0460]", - "error[e0514]", - "error cs0006", - "error cs0009", - "error cs0012", - "error cs0518", - "error cs1705", - ] - .iter() - .any(|code| message.contains(code)) - { - return false; - } - let relative = match lang { - Lang::Rust => "src/lib.rs", - Lang::CSharp => "lib.cs", - Lang::TypeScript => "src/index.ts", - }; - let root = source_root - .canonicalize() - .or_else(|_| std::path::absolute(source_root)) - .unwrap_or_else(|_| source_root.to_path_buf()) - .to_string_lossy() - .replace('\\', "/") - .to_ascii_lowercase(); - let root = root.trim_start_matches("//?/"); - let absolute = format!("{root}/{relative}"); - let sdk_output = format!("{root}/obj/"); - let is_generated_location = |location: &str| { - let location = location.trim().trim_start_matches("//?/").trim_start_matches("./"); - // SDK source generators compile the candidate's types from its own obj directory. - (matches!(lang, Lang::CSharp) && location.starts_with(&sdk_output)) - || [relative, absolute.as_str()].iter().any(|path| { - location - .strip_prefix(path) - .is_some_and(|rest| rest.starts_with(':') || rest.starts_with('(')) - }) +// Model mistakes have no fixed list of messages, so a candidate failure scores unless it matches one of these. +fn is_environment_failure(message: &str) -> bool { + const SIGNATURES: [&str; 11] = [ + "no space left on device", + "out of memory", + "cannot allocate memory", + "connection refused", + "connection reset", + "error sending request", + "failed to download", + "spurious network error", + "could not resolve host", + "unable to load the service index", + "service unavailable", + ]; + // rustc quotes source lines as `12 | code`, and a model's own strings can read like these errors. + let quotes_source = |line: &str| { + line.split_once('|') + .is_some_and(|(number, _)| number.trim().parse::().is_ok()) }; - let lines: Vec<_> = message.lines().map(str::trim).collect(); - let mut found = false; - for (index, line) in lines.iter().enumerate() { - let rust_error = matches!(lang, Lang::Rust) && (line.starts_with("error[e") || line.starts_with("error:")); - if rust_error { - // Cargo, rustc and CLI summaries add no source diagnostic of their own. - if line.starts_with("error: could not compile") - || line.starts_with("error: aborting due to") - || line.starts_with("error: command [\"cargo\"") - { - continue; - } - let location = lines[index + 1..] - .iter() - .take_while(|line| !line.starts_with("error") && !line.starts_with("warning")) - .find_map(|line| line.strip_prefix("--> ")); - if !location.is_some_and(is_generated_location) { - return false; - } - found = true; - } else if (matches!(lang, Lang::CSharp) && (line.contains(": error cs") || line.contains(": error stdb"))) - || (matches!(lang, Lang::TypeScript) && line.contains(": error ts")) - { - if !is_generated_location(line) { - return false; - } - found = true; - } - } - found + message.lines().filter(|line| !quotes_source(line)).any(|line| { + let line = line.to_ascii_lowercase(); + SIGNATURES.iter().any(|signature| line.contains(signature)) + }) +} + +// A ping cannot show that builds still work, so a batch only completes if known-good code still publishes. +async fn ensure_environment_healthy(cfg: &BenchRunContext<'_>) -> Result<()> { + ensure_server_healthy(cfg.host.as_deref()).await?; + let route_tag = run_scope_tag(cfg.mode, cfg.route.vendor.slug(), &cfg.route.api_model); + let control = build_goldens_only_for_lang( + cfg.host.clone(), + cfg.bench_root, + cfg.lang, + Some(&["t_000".to_string()]), + &format!("{route_tag}-control"), + ) + .await + .context("known-good control module failed to publish; refusing to upload results")?; + delete_databases(control).await } impl TaskRunner { @@ -581,10 +544,9 @@ impl TaskRunner { (Some(database), None) } Err(error) => { - let source_root = work_server_dir_scoped(&category, &task_id, cfg.lang_name, "llm", &route_tag); - if !is_code_publish_failure(&format!("{error:#}"), &source_root, cfg.lang) { + if is_environment_failure(&format!("{error:#}")) { return Err(RunOneError::Infrastructure { - msg: format!("publish failed without a source-code diagnostic: {error:#}"), + msg: format!("publish hit an environment failure: {error:#}"), llm_output: Some(llm_output), provider_error: None, completed_outcome: None, @@ -1019,9 +981,9 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu println!("[runner] completed batch: {} results", outcomes.len()); - // A dead local server turns later publish failures into misleading zeroes. - // Refuse to analyze or upload any batch that cannot pass a final health check. - ensure_server_healthy(cfg.host.as_deref()).await?; + // A broken server or build environment turns later publish failures into misleading zeroes. + // Refuse to analyze or upload any batch that cannot pass a final environment check. + ensure_environment_healthy(cfg).await?; record.complete()?; if cfg.dry_run { @@ -1200,7 +1162,7 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> }; let outcomes = finalize_batch(outcomes, pending)?; - ensure_server_healthy(cfg.host.as_deref()).await?; + ensure_environment_healthy(cfg).await?; record.complete()?; if cfg.dry_run { @@ -1785,72 +1747,39 @@ mod build_cleanup_tests { } #[test] - fn only_explicit_code_diagnostics_are_scored_as_publish_failures() { - let root = std::env::temp_dir().join("benchmark-module"); - let root = root.as_path(); - for diagnostic in [ - "error[E0425]: cannot find value `missing` in this scope", - "error: expected one of `!` or `::`, found `main`", - "error: unexpected closing delimiter: `}`", - "error: mismatched closing delimiter: `)`", - "error: invalid suffix `words` for number literal", - "error: this file contains an unclosed delimiter", - "error: unknown start of token: \\", - ] { - let diagnostic = format!( - "{diagnostic}\n --> src/lib.rs:10:5\nerror: could not compile `test` due to 1 previous error\nError: command [\"cargo\", \"build\"] exited with code 101" - ); - assert!(is_code_publish_failure(&diagnostic, root, Lang::Rust), "{diagnostic}"); - } - for (lang, diagnostic) in [ - ( - Lang::CSharp, - "Lib.cs(10,5): error CS0103: The name 'missing' does not exist", - ), - ( - Lang::TypeScript, - "src/index.ts(2,3): error TS2304: Cannot find name 'missing'", - ), + fn candidate_publish_failures_score_unless_the_environment_failed() { + for message in [ + "error[E0425]: cannot find value `missing` in this scope\n --> src/lib.rs:10:5\nerror: could not compile `test` due to 1 previous error\nError: command [\"cargo\", \"build\"] exited with code 101", + "error: Expected a string literal for `name`, but found an identifier.\n --> src/lib.rs:4:9", + "error[E0463]: can't find crate for `serde`\n --> src/lib.rs:1:1", + "Lib.cs(10,5): error CS0103: The name 'missing' does not exist", + "obj/Release/Shape.cs(27,33): error CS0144: abstract type", + "src/index.ts(2,3): error TS2304: Cannot find name 'missing'", + "Errors occurred:\nAdding a column enabled to table widget requires a default value annotation\n\nAborting publish due to required manual migration.", + "Error: Local module schema inspection failed:\nError: Errors occurred:\nname `OrderLineDetail` is used for multiple types", + "Error: It seems like you haven't exported your schema. You must `export default schema(...);`", + "Your module doesn't import the `spacetimedb/server` package at all", + "Error: Local module schema inspection failed:\nReferenceError: missing is not defined", + "command dotnet publish exited with code 1", + "error[E0599]: no method named `body`\n --> src/lib.rs:12:5\n |\n12 | let page = fetch(url).expect(\"failed to download\");\n | ^^^^^", + "a failure nobody has seen before", ] { - assert!(is_code_publish_failure(diagnostic, root, lang), "{diagnostic}"); - } - for root in [root, Path::new("target/llm-runs/test-relative-module")] { - let source = std::path::absolute(root).unwrap().join("Lib.cs"); - let diagnostic = format!("{}(10,5): error CS0103: missing name", source.display()); - assert!(is_code_publish_failure(&diagnostic, root, Lang::CSharp), "{diagnostic}"); - let generated = std::path::absolute(root).unwrap().join("obj/Release/Shape.cs"); - let diagnostic = format!("{}(27,33): error CS0144: abstract type", generated.display()); - assert!(is_code_publish_failure(&diagnostic, root, Lang::CSharp), "{diagnostic}"); - } - let rejected_migration = "Errors occurred:\nAdding a column enabled to table widget requires a default value annotation\n\nAborting publish due to required manual migration."; - let invalid_schema = "Error: Local module schema inspection failed:\nError: Errors occurred:\nname `OrderLineDetail` is used for multiple types"; - for lang in [Lang::Rust, Lang::CSharp, Lang::TypeScript] { - assert!(is_code_publish_failure(rejected_migration, root, lang)); - assert!(is_code_publish_failure(invalid_schema, root, lang)); + assert!(!is_environment_failure(message), "{message}"); } - for diagnostic in [ - "HTTP 503 Service Unavailable", - "authentication token expired", + for message in [ "No space left on device", + "fatal error: out of memory", + "cannot allocate memory", + "tcp connect error: Connection refused (os error 111)", + "Connection reset by peer", + "error sending request for url (http://127.0.0.1:3000/v1/database)", "failed to download package", - "command dotnet publish exited with code 1", - "migration setup failed", - "unknown publish error", - "Local module schema inspection timed out", - "error: linking with `cc` failed", - "error: linking with `cc` failed\nError: command [\"cargo\", \"build\"] exited with code 101", - "Error: command [\"cargo\", \"build\"] exited with code 101", - "error: could not compile `test` due to 1 previous error", - "error[E0463]: can't find crate for `std`\n --> src/lib.rs:1:1", - "error[E0425]: cannot find value\n --> /dependencies/src/lib.rs:1:1", - "Lib.cs(1,1): error CS0006: Metadata file could not be found", - "/dependencies/Lib.cs(1,1): error CS0103: missing name", - "/dependencies/obj/Shape.cs(1,1): error CS0144: abstract type", - "node_modules/package/index.ts(1,1): error TS2304: missing name", + "warning: spurious network error (3 tries remaining)", + "Could not resolve host: index.crates.io", + "error NU1301: Unable to load the service index for source https://api.nuget.org/v3/index.json", + "HTTP 503 Service Unavailable", ] { - for lang in [Lang::Rust, Lang::CSharp, Lang::TypeScript] { - assert!(!is_code_publish_failure(diagnostic, root, lang), "{diagnostic}"); - } + assert!(is_environment_failure(message), "{message}"); } } From 9f1c2315f583e697e9ebd6522372712cbd381dd7 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Mon, 2 Nov 2026 13:52:18 -0500 Subject: [PATCH 15/41] Allow 12 hours for periodic benchmark jobs Models run one at a time and a C# batch takes about 34 minutes, so a full model list would hit GitHub's 6-hour default and lose the remaining models. --- .github/workflows/llm-benchmark-periodic.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 3d3807599c8..5a14dcf1467 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -76,6 +76,8 @@ jobs: name: Run benchmarks (${{ matrix.lang }}) needs: prepare_matrix runs-on: spacetimedb-linux + # Models run one at a time, and a C# batch takes about 34 minutes. + timeout-minutes: 720 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target TARGET_TRIPLE: x86_64-unknown-linux-gnu From bd0e97dab8648cbfc235a8d47c3563c71ea1cce3 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Mon, 2 Nov 2026 17:05:42 -0500 Subject: [PATCH 16/41] Keep scorer CLI identities per benchmark server Each C# batch starts a fresh server, but scorers shared one saved CLI token, which the next server rejected with InvalidSignature. Only the first model in a C# run could finish. Give each server address its own CLI root. --- tools/xtask-llm-benchmark/src/eval/scorers.rs | 10 ++++---- tools/xtask-llm-benchmark/src/eval/utils.rs | 25 ++++++++++++++++++- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index edc0a7e894e..75e31afa8bc 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -1,6 +1,6 @@ use crate::bench::utils::debug_llm_verbose; use crate::eval::utils::{run_with_timeout, sql_exec_with_timeout}; -use crate::eval::{normalize, spacetime_command, ScoreDetails}; +use crate::eval::{normalize, scorer_command, ScoreDetails}; use regex::Regex; use serde_json::{json, Value}; use std::collections::{BTreeMap, BTreeSet}; @@ -613,7 +613,7 @@ fn call_reducer_json_out_with_timeout( host: Option<&str>, timeout: Duration, ) -> Result { - let mut cmd = spacetime_command(); + let mut cmd = scorer_command(host); cmd.arg("call"); if let Some(h) = host { cmd.arg("--server").arg(h); @@ -650,7 +650,7 @@ pub fn sql_raw(db: &str, query: &str, host: Option<&str>) -> Result, timeout: Duration) -> Result { - let mut cmd = spacetime_command(); + let mut cmd = scorer_command(host); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { cmd.arg("--server").arg(h); @@ -677,7 +677,7 @@ fn sql_raw_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Dura } fn sql_json_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { - let mut cmd = spacetime_command(); + let mut cmd = scorer_command(host); cmd.arg("sql").arg(db).arg(query).args(["--format", "json"]); if let Some(h) = host { cmd.arg("--server").arg(h); @@ -706,7 +706,7 @@ pub fn sql_count(db: &str, query: &str, host: Option<&str>) -> Result, timeout: Duration) -> Result { - let mut cmd = spacetime_command(); + let mut cmd = scorer_command(host); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { cmd.arg("--server").arg(h); diff --git a/tools/xtask-llm-benchmark/src/eval/utils.rs b/tools/xtask-llm-benchmark/src/eval/utils.rs index cf97051e960..1125e5d0b42 100644 --- a/tools/xtask-llm-benchmark/src/eval/utils.rs +++ b/tools/xtask-llm-benchmark/src/eval/utils.rs @@ -55,6 +55,20 @@ pub(crate) fn spacetime_command() -> Command { Command::new("spacetime") } +fn scorer_cli_root(host: Option<&str>) -> PathBuf { + let host = crate::bench::utils::sanitize_db_name(host.unwrap_or("local")); + env::temp_dir().join(format!("stdb-llm-scorer-{}-{host}", std::process::id())) +} + +// A saved token only works on the server that issued it, and each C# batch starts a new server. +pub(crate) fn scorer_command(host: Option<&str>) -> Command { + let root = scorer_cli_root(host); + let _ = std::fs::create_dir_all(&root); + let mut cmd = spacetime_command(); + cmd.arg("--root-dir").arg(root); + cmd +} + pub fn sql_exec(db: &str, query: &str, host: Option<&str>) -> Result<(), String> { sql_exec_with_timeout(db, query, host, Duration::from_secs(30)) } @@ -65,7 +79,7 @@ pub(crate) fn sql_exec_with_timeout( host: Option<&str>, timeout: Duration, ) -> Result<(), String> { - let mut cmd = spacetime_command(); + let mut cmd = scorer_command(host); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { cmd.arg("--server").arg(h); @@ -230,4 +244,13 @@ mod tests { assert_eq!(stdout.len(), expected_len); assert_eq!(stderr.len(), expected_len); } + + #[test] + fn each_server_gets_its_own_scorer_identity() { + let first = Some("http://127.0.0.1:3000"); + let second = Some("http://127.0.0.1:3001"); + + assert_eq!(scorer_cli_root(first), scorer_cli_root(first)); + assert_ne!(scorer_cli_root(first), scorer_cli_root(second)); + } } From db052ea7fa603c724258a18633e76db7a1806e7f Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Mon, 2 Nov 2026 18:21:30 -0500 Subject: [PATCH 17/41] Stop C# benchmark builds losing their SDK generators After a slow model left the C# compiler idle for about 20 minutes, every later build failed with CS8784 because the SDK generators would not load, including the references. Turn off the shared compiler, and treat CS8784 as an environment failure so it cannot score as bad code. --- tools/xtask-llm-benchmark/src/bench/publishers.rs | 2 ++ tools/xtask-llm-benchmark/src/bench/runner.rs | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index 94eb998f51e..3bd6e20f972 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -724,6 +724,8 @@ impl DotnetPublisher { // when running multiple dotnet builds in parallel. .env("MSBUILDDISABLENODEREUSE", "1") .env("DOTNET_CLI_USE_MSBUILD_SERVER", "0") + // A slow model can idle the shared compiler long enough that later builds lose the SDK generators. + .env("UseSharedCompilation", "false") } } diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index ec305c986d9..db40f08e53e 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -242,7 +242,8 @@ fn provider_retry_delay(error: &anyhow::Error) -> Option { // Model mistakes have no fixed list of messages, so a candidate failure scores unless it matches one of these. fn is_environment_failure(message: &str) -> bool { - const SIGNATURES: [&str; 11] = [ + const SIGNATURES: [&str; 12] = [ + "warning cs8784", "no space left on device", "out of memory", "cannot allocate memory", @@ -1778,6 +1779,7 @@ mod build_cleanup_tests { "Could not resolve host: index.crates.io", "error NU1301: Unable to load the service index for source https://api.nuget.org/v3/index.json", "HTTP 503 Service Unavailable", + "CSC : warning CS8784: Generator 'Module' failed to initialize.", ] { assert!(is_environment_failure(message), "{message}"); } From be59dd78264736a076a0670e398580fc0a8438da Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 3 Nov 2026 00:58:42 -0500 Subject: [PATCH 18/41] Check behaviour on 27 tasks and stop scoring valid code as wrong - Add behaviour scenarios to the 27 tasks that only checked schema, and reword the advanced prompts to match the others. - Schema parity matches lifecycle reducers by kind, matches view names regardless of case or underscores, and accepts extra indexes and extra view primary keys. - Prompts now state table visibility where they did not, say when a module has no tables, and state t_074's unit return. - Scorer CLI roots log in once before use, so the login prompt no longer lands in parsed output. - Checks that query a missing table retry with the module's own spelling when it differs only by case or underscores. - The direct Anthropic client defaults to 16000 output tokens, because thinking counts against the limit. --- tools/xtask-llm-benchmark/README.md | 10 +- .../src/bench/publishers.rs | 15 ++ .../t_083_atomic_reservation/tasks/csharp.txt | 2 +- .../t_083_atomic_reservation/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_084_stable_pagination/tasks/csharp.txt | 2 +- .../t_084_stable_pagination/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_085_online_backfill/tasks/csharp.txt | 2 +- .../t_085_online_backfill/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_086_incremental_totals/tasks/csharp.txt | 2 +- .../t_086_incremental_totals/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_087_webhook_ordering/tasks/csharp.txt | 2 +- .../t_087_webhook_ordering/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_088_lease_fencing/tasks/csharp.txt | 2 +- .../t_088_lease_fencing/tasks/rust.txt | 2 +- .../t_088_lease_fencing/tasks/typescript.txt | 2 +- .../t_089_cancel_reschedule/tasks/csharp.txt | 2 +- .../t_089_cancel_reschedule/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 2 +- .../t_090_transactional_quota/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_091_scheduled_queue/tasks/csharp.txt | 2 +- .../t_091_scheduled_queue/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_092_procedure_cache/tasks/csharp.txt | 2 +- .../t_092_procedure_cache/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 2 +- .../tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 2 +- .../tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../auth/t_026_auth_identity_check/spec.rs | 19 +- .../tasks/csharp.txt | 1 + .../t_027_private_vs_public_table/spec.rs | 21 +- .../auth/t_041_registered_user_gate/spec.rs | 20 +- .../auth/t_042_admin_bootstrap/spec.rs | 18 +- .../auth/t_043_role_based_access/spec.rs | 32 ++- .../benchmarks/auth/t_044_ban_list/spec.rs | 25 +- .../benchmarks/auth/t_045_rate_limit/spec.rs | 27 +- .../auth/t_046_shared_document/spec.rs | 25 +- .../basics/t_002_scheduled_table/spec.rs | 16 +- .../t_002_scheduled_table/tasks/csharp.txt | 2 +- .../t_002_scheduled_table/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_003_struct_in_table/tasks/csharp.txt | 2 +- .../t_003_struct_in_table/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_008_index_lookup/tasks/csharp.txt | 2 +- .../basics/t_008_index_lookup/tasks/rust.txt | 2 +- .../t_008_index_lookup/tasks/typescript.txt | 2 +- .../benchmarks/basics/t_010_connect/spec.rs | 26 +- .../basics/t_038_schedule_at_time/spec.rs | 20 +- .../t_038_schedule_at_time/tasks/csharp.txt | 2 +- .../t_038_schedule_at_time/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../basics/t_039_cancel_schedule/spec.rs | 18 +- .../t_039_cancel_schedule/tasks/csharp.txt | 2 +- .../t_039_cancel_schedule/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../basics/t_040_lifecycle_player/spec.rs | 15 +- .../data_modeling/t_024_event_table/spec.rs | 14 +- .../t_025_optional_fields/spec.rs | 18 +- .../t_028_cascade_delete/spec.rs | 19 +- .../t_029_filter_and_aggregate/spec.rs | 20 +- .../t_030_two_table_join/spec.rs | 22 +- .../t_031_unique_constraint/spec.rs | 20 +- .../t_072_procedure_return/tasks/csharp.txt | 2 + .../t_072_procedure_return/tasks/rust.txt | 2 + .../tasks/typescript.txt | 2 + .../t_074_fetch_and_store/tasks/csharp.txt | 2 +- .../t_074_fetch_and_store/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_076_http_handler/tasks/csharp.txt | 2 + .../t_076_http_handler/tasks/rust.txt | 2 + .../t_076_http_handler/tasks/typescript.txt | 2 + .../t_077_http_router/tasks/csharp.txt | 2 + .../t_077_http_router/tasks/rust.txt | 2 + .../t_077_http_router/tasks/typescript.txt | 2 + .../queries/t_022_view_basic/spec.rs | 20 +- .../queries/t_023_view_per_user/spec.rs | 21 +- .../queries/t_032_range_query/spec.rs | 21 +- .../queries/t_033_sort_and_limit/spec.rs | 21 +- .../queries/t_034_find_first/spec.rs | 18 +- .../queries/t_035_select_distinct/spec.rs | 17 +- .../t_036_count_without_collect/spec.rs | 17 +- .../queries/t_037_multi_column_filter/spec.rs | 17 +- .../t_019_many_to_many/tasks/csharp.txt | 4 +- .../schema/t_019_many_to_many/tasks/rust.txt | 4 +- .../t_019_many_to_many/tasks/typescript.txt | 4 +- .../schema/t_020_ecs/tasks/csharp.txt | 4 +- .../schema/t_020_ecs/tasks/rust.txt | 4 +- .../schema/t_020_ecs/tasks/typescript.txt | 4 +- .../xtask-llm-benchmark/src/eval/scenario.rs | 113 +++++++++ tools/xtask-llm-benchmark/src/eval/scorers.rs | 236 ++++++++++++++++-- tools/xtask-llm-benchmark/src/eval/utils.rs | 41 ++- .../src/llm/clients/anthropic.rs | 7 +- 103 files changed, 961 insertions(+), 159 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index 806f925f238..5cda492c3cf 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -18,9 +18,13 @@ starting the next task. Rust and TypeScript keep their batched reference builds. Cleanup errors block the batch, but completed scoring remains in the local record. Compiler errors in SDK-generated files score as normal failures. Saved batch records keep the evidence so suspected SDK defects can go to the core team. -Schema parity treats table names that differ only by case or underscores as the -same table, because prompts do not fix a table's casing. It fails any C# table that -sets `Name` without `Accessor`, because that is the 1.x form. +Schema parity treats table and view names that differ only by case or underscores +as the same, because prompts do not fix their casing. It matches lifecycle reducers +by kind, not by name, because clients cannot call them. It accepts extra indexes +and extra view primary keys, but still fails a missing one. It fails any C# table +that sets `Name` without `Accessor`, because that is the 1.x form. Behaviour checks +query a table or view by the module's own spelling when it differs only by case or +underscores. Each batch also saves an append-only JSONL record under `LLM_BENCHMARK_REPORT_DIR/batches`, or `tools/xtask-llm-benchmark/runs/batches` diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index 3bd6e20f972..33d6e386384 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -367,6 +367,21 @@ fn database_cli_root(module_name: &str, remove_on_drop: bool) -> Result Result { + let db = sanitize_db_name(module_name); + let path = env::temp_dir() + .join(format!("stdb-llm-cli-{}-{db}", CLI_SESSION_TAG.as_str())) + .join("config/cli.toml"); + let config = fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; + config + .lines() + .find_map(|line| line.trim().strip_prefix("spacetimedb_token")) + .and_then(|value| value.trim_start().strip_prefix('=')) + .map(|value| value.trim().trim_matches('"').to_owned()) + .with_context(|| format!("no token in {}", path.display())) +} + /// Check if the process was killed by a signal (e.g., SIGSEGV = 11) #[cfg(unix)] fn signal_killed_by(status: &std::process::ExitStatus) -> Option { diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt index 68ecc340d00..22a6d331fe8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that reserves two products atomically. Return only the module source. Reserve two distinct products atomically. A request ID identifies the entire payload. Exact replay is a no-op; reuse with different payload is an error. Validate quantities and products before accepting. No overselling, partial deductions, or failed-request receipts are allowed. Use the exact error text stated below. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt index ce99fbbfdff..f5350514b3c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that reserves two products atomically. Return only the module source. Reserve two distinct products atomically. A request ID identifies the entire payload. Exact replay is a no-op; reuse with different payload is an error. Validate quantities and products before accepting. No overselling, partial deductions, or failed-request receipts are allowed. Use the exact error text stated below. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt index 6b51c56691e..9abea3d5ac9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that reserves two products atomically. Return only the module source. Reserve two distinct products atomically. A request ID identifies the entire payload. Exact replay is a no-op; reuse with different payload is an error. Validate quantities and products before accepting. No overselling, partial deductions, or failed-request receipts are allowed. Use the exact error text stated below. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt index adfc06b771c..6e13d97c1f2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that serves stable paginated audit logs. Return only the module source. Implement a tenant-scoped audit-log page. The ordering key is (occurred_at, id), ascending. The exclusive cursor is that pair; never use timestamp alone. Only rows for the requested tenant with visible=true can enter a page. Apply filtering BEFORE the limit. The page table is a test adapter: replace it on each read_page call; it is not a per-client cache. No native pagination API or scan-cost guarantee is assumed. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt index 8866e8ec33c..e6f78579bf8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that serves stable paginated audit logs. Return only the module source. Implement a tenant-scoped audit-log page. The ordering key is (occurred_at, id), ascending. The exclusive cursor is that pair; never use timestamp alone. Only rows for the requested tenant with visible=true can enter a page. Apply filtering BEFORE the limit. The page table is a test adapter: replace it on each read_page call; it is not a per-client cache. No native pagination API or scan-cost guarantee is assumed. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt index 12f9667c5c9..f5f1aaeee58 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that serves stable paginated audit logs. Return only the module source. Implement a tenant-scoped audit-log page. The ordering key is (occurred_at, id), ascending. The exclusive cursor is that pair; never use timestamp alone. Only rows for the requested tenant with visible=true can enter a page. Apply filtering BEFORE the limit. The page table is a test adapter: replace it on each read_page call; it is not a per-client cache. No native pagination API or scan-cost guarantee is assumed. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt index 5955ce364e0..892bf22e214 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that runs a resumable online backfill. Return only the module source. Implement a resumable online backfill while writes and deletes continue. The legacy table is authoritative. Capture and apply are separate transactions, so a captured snapshot can become stale. Keep tombstones in legacy_item; never resurrect a deleted item. This task tests the data migration algorithm inside one module, not deployment schema conversion. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt index 79508d705bb..8e0cb0834a4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that runs a resumable online backfill. Return only the module source. Implement a resumable online backfill while writes and deletes continue. The legacy table is authoritative. Capture and apply are separate transactions, so a captured snapshot can become stale. Keep tombstones in legacy_item; never resurrect a deleted item. This task tests the data migration algorithm inside one module, not deployment schema conversion. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt index d76f57cd84a..42bbe669102 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that runs a resumable online backfill. Return only the module source. Implement a resumable online backfill while writes and deletes continue. The legacy table is authoritative. Capture and apply are separate transactions, so a captured snapshot can become stale. Keep tombstones in legacy_item; never resurrect a deleted item. This task tests the data migration algorithm inside one module, not deployment schema conversion. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt index 04f9f4d16b5..340f5c187e7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that maintains category totals incrementally. Return only the module source. Maintain category totals after arbitrary sale inserts, updates, moves and deletes. A category row exists exactly when its sale_count is nonzero, even if total_amount is zero. Negative and zero sale amounts are valid. All source and aggregate changes must be atomic. This is a correctness test, not a scan-cost benchmark. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt index 5f81bcb792a..fdb26f6f6c5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that maintains category totals incrementally. Return only the module source. Maintain category totals after arbitrary sale inserts, updates, moves and deletes. A category row exists exactly when its sale_count is nonzero, even if total_amount is zero. Negative and zero sale amounts are valid. All source and aggregate changes must be atomic. This is a correctness test, not a scan-cost benchmark. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt index 8f395605b8b..79f999bdcd9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that maintains category totals incrementally. Return only the module source. Maintain category totals after arbitrary sale inserts, updates, moves and deletes. A category row exists exactly when its sale_count is nonzero, even if total_amount is zero. Negative and zero sale amounts are valid. All source and aggregate changes must be atomic. This is a correctness test, not a scan-cost benchmark. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/csharp.txt index e9581523d31..b73b0172416 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that handles out-of-order webhook events. Return only the module source. Implement POST /webhook with plain-text body account|event_id|sequence|value (exactly four fields). Account, event_id and value must be nonempty. Sequence must contain only ASCII decimal digits, fit u64, and be positive. Malformed input returns HTTP 400 body 'invalid' without any writes. Valid input returns HTTP 200. In one transaction, check the event ID globally: exact replay returns 'duplicate'; reuse with a different account, sequence or value returns 'conflict', without changes. For a new event, insert a receipt even if its sequence is stale. If sequence is greater than that account's last sequence, update state and return 'applied'; otherwise return 'stale'. Accounts advance independently. Response bodies must contain exactly those words. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/rust.txt index 9461cca31f8..24346f4c176 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that handles out-of-order webhook events. Return only the module source. Implement POST /webhook with plain-text body account|event_id|sequence|value (exactly four fields). Account, event_id and value must be nonempty. Sequence must contain only ASCII decimal digits, fit u64, and be positive. Malformed input returns HTTP 400 body 'invalid' without any writes. Valid input returns HTTP 200. In one transaction, check the event ID globally: exact replay returns 'duplicate'; reuse with a different account, sequence or value returns 'conflict', without changes. For a new event, insert a receipt even if its sequence is stale. If sequence is greater than that account's last sequence, update state and return 'applied'; otherwise return 'stale'. Accounts advance independently. Response bodies must contain exactly those words. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/typescript.txt index cba4e57ee1e..73b77ea18f1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that handles out-of-order webhook events. Return only the module source. Implement POST /webhook with plain-text body account|event_id|sequence|value (exactly four fields). Account, event_id and value must be nonempty. Sequence must contain only ASCII decimal digits, fit u64, and be positive. Malformed input returns HTTP 400 body 'invalid' without any writes. Valid input returns HTTP 200. In one transaction, check the event ID globally: exact replay returns 'duplicate'; reuse with a different account, sequence or value returns 'conflict', without changes. For a new event, insert a receipt even if its sequence is stale. If sequence is greater than that account's last sequence, update state and return 'applied'; otherwise return 'stale'. Accounts advance independently. Response bodies must contain exactly those words. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt index edd10eb26e2..ca6bc9f2bc4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that leases jobs with fencing tokens. Return only the module source. Implement a leased job with monotonically increasing fencing tokens. Use the server transaction timestamp, in microseconds, for all expiry checks. A completion must match the current worker and token AND arrive before expiry. Reclaiming an expired lease increments the token. No caller-supplied current time. Workers are opaque labels in this test; authentication is tested separately. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/rust.txt index 2e87adf3310..cc83a25fb66 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that leases jobs with fencing tokens. Return only the module source. Implement a leased job with monotonically increasing fencing tokens. Use the server transaction timestamp, in microseconds, for all expiry checks. A completion must match the current worker and token AND arrive before expiry. Reclaiming an expired lease increments the token. No caller-supplied current time. Workers are opaque labels in this test; authentication is tested separately. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/typescript.txt index a8845c08d97..7d49f421382 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that leases jobs with fencing tokens. Return only the module source. Implement a leased job with monotonically increasing fencing tokens. Use the server transaction timestamp, in microseconds, for all expiry checks. A completion must match the current worker and token AND arrive before expiry. Reclaiming an expired lease increments the token. No caller-supplied current time. Workers are opaque labels in this test; authentication is tested separately. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/csharp.txt index 07e40848c7e..b38f8263bec 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that manages renewable reservations with scheduled expiry. Return only the module source. Implement renewable reservations with scheduled expiry. An old timer must never expire a newer generation. Cancellation is durable and must also invalidate old timers. Do not remove old timers when renewing or cancelling: let them execute and record their outcome, which makes stale-callback behavior observable. Only the scheduler may invoke expire_reservation; external calls must be rejected. Native private-callback rejection is valid; an explicit identity guard must report 'scheduler only'. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/rust.txt index b663ad5dd0a..f1fd1271e08 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that manages renewable reservations with scheduled expiry. Return only the module source. Implement renewable reservations with scheduled expiry. An old timer must never expire a newer generation. Cancellation is durable and must also invalidate old timers. Do not remove old timers when renewing or cancelling: let them execute and record their outcome, which makes stale-callback behavior observable. Only the scheduler may invoke expire_reservation; external calls must be rejected. Native private-callback rejection is valid; an explicit identity guard must report 'scheduler only'. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/typescript.txt index 09fc822a122..e9a81504fcd 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_089_cancel_reschedule/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that manages renewable reservations with scheduled expiry. Return only the module source. Implement renewable reservations with scheduled expiry. An old timer must never expire a newer generation. Cancellation is durable and must also invalidate old timers. Do not remove old timers when renewing or cancelling: let them execute and record their outcome, which makes stale-callback behavior observable. Only the scheduler may invoke expire_reservation; external calls must be rejected. Native private-callback rejection is valid; an explicit identity guard must report 'scheduler only'. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/csharp.txt index 5ada543268d..34e5a27e362 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that enforces a per-identity transactional quota. Return only the module source. Implement a per-identity lifetime quota of 3 accepted units. The sender, not a caller-supplied identity, owns each request. This is a transaction/quota test, not a timed-window rate limiter. Invalid requests and failed transactions must not consume quota or reserve request IDs. Exact replay is a no-op; changing owner or amount for an existing request ID is a conflict. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt index 4431a42ed6d..5192a2ea0ad 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that enforces a per-identity transactional quota. Return only the module source. Implement a per-identity lifetime quota of 3 accepted units. The sender, not a caller-supplied identity, owns each request. This is a transaction/quota test, not a timed-window rate limiter. Invalid requests and failed transactions must not consume quota or reserve request IDs. Exact replay is a no-op; changing owner or amount for an existing request ID is a conflict. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt index 74c73ddfe82..5606c8ea595 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that enforces a per-identity transactional quota. Return only the module source. Implement a per-identity lifetime quota of 3 accepted units. The sender, not a caller-supplied identity, owns each request. This is a transaction/quota test, not a timed-window rate limiter. Invalid requests and failed transactions must not consume quota or reserve request IDs. Exact replay is a no-op; changing owner or amount for an existing request ID is a conflict. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt index 4d64e0c1c86..7966b01e630 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that runs a scheduled job queue with durable completion state. Return only the module source. Implement a scheduled job queue with durable completion state. A failed job must not stop other jobs. A business failure (negative input) is recorded as failed; it must not throw and lose its status. Duplicate submissions must not run a job twice. The scheduled callback must reject direct external calls. Native private-callback rejection is valid; an explicit identity guard must report 'scheduler only'. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt index 7465e5601ad..0cd2131f220 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that runs a scheduled job queue with durable completion state. Return only the module source. Implement a scheduled job queue with durable completion state. A failed job must not stop other jobs. A business failure (negative input) is recorded as failed; it must not throw and lose its status. Duplicate submissions must not run a job twice. The scheduled callback must reject direct external calls. Native private-callback rejection is valid; an explicit identity guard must report 'scheduler only'. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt index 48950d72a1d..16e4ee96b31 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that runs a scheduled job queue with durable completion state. Return only the module source. Implement a scheduled job queue with durable completion state. A failed job must not stop other jobs. A business failure (negative input) is recorded as failed; it must not throw and lose its status. Duplicate submissions must not run a job twice. The scheduled callback must reject direct external calls. Native private-callback rejection is valid; an explicit identity guard must report 'scheduler only'. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/csharp.txt index ef1646c99f4..e7cec592bc2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that caches an external service in a procedure. Return only the module source. Implement a persistent cache for an external text-generating service. Cache identity is the exact (product, language) pair, NOT TTL or URL, and must not collide when either string contains punctuation. Procedure fetch_cached(product:string, language:string, ttl_ms:u64, url:string) returns a string. If product/language is empty or ttl_ms>60000, return 'invalid input' without HTTP or writes. For ttl_ms>0, return a nonexpired cached value without HTTP. Expired/missing entries cause one HTTP GET to url outside a transaction. On HTTP/network failure return 'upstream error' without changing the cache. On HTTP 200, cache the complete response text with expiry server_now+ttl_ms*1000, and return it. ttl_ms=0 always fetches but never reads or changes cache rows. Use the server clock. Requests in this test are sequential; request coalescing is not required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/rust.txt index be907540ae5..0468f189008 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that caches an external service in a procedure. Return only the module source. Implement a persistent cache for an external text-generating service. Cache identity is the exact (product, language) pair, NOT TTL or URL, and must not collide when either string contains punctuation. Procedure fetch_cached(product:string, language:string, ttl_ms:u64, url:string) returns a string. If product/language is empty or ttl_ms>60000, return 'invalid input' without HTTP or writes. For ttl_ms>0, return a nonexpired cached value without HTTP. Expired/missing entries cause one HTTP GET to url outside a transaction. On HTTP/network failure return 'upstream error' without changing the cache. On HTTP 200, cache the complete response text with expiry server_now+ttl_ms*1000, and return it. ttl_ms=0 always fetches but never reads or changes cache rows. Use the server clock. Requests in this test are sequential; request coalescing is not required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/typescript.txt index 1acc0ee6f97..41285a8b6f1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_092_procedure_cache/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that caches an external service in a procedure. Return only the module source. Implement a persistent cache for an external text-generating service. Cache identity is the exact (product, language) pair, NOT TTL or URL, and must not collide when either string contains punctuation. Procedure fetch_cached(product:string, language:string, ttl_ms:u64, url:string) returns a string. If product/language is empty or ttl_ms>60000, return 'invalid input' without HTTP or writes. For ttl_ms>0, return a nonexpired cached value without HTTP. Expired/missing entries cause one HTTP GET to url outside a transaction. On HTTP/network failure return 'upstream error' without changing the cache. On HTTP 200, cache the complete response text with expiry server_now+ttl_ms*1000, and return it. ttl_ms=0 always fetches but never reads or changes cache rows. Use the server clock. Requests in this test are sequential; request coalescing is not required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt index 2d0483e5bd3..800244f4766 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that shares a private document with revocable permissions. Return only the module source. Implement one shared private document (ID 1). Its first writer becomes its permanent owner. Only that owner can change its text or reader permissions. Authorized users can query and subscribe to a safe projection. Revocation must remove the document from an existing reader subscription and block future content updates. Do not use row-level security. Keep both source tables private and never expose secret_body in the public view. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt index a127d251322..23246a4ca5c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that shares a private document with revocable permissions. Return only the module source. Implement one shared private document (ID 1). Its first writer becomes its permanent owner. Only that owner can change its text or reader permissions. Authorized users can query and subscribe to a safe projection. Revocation must remove the document from an existing reader subscription and block future content updates. Do not use row-level security. Keep both source tables private and never expose secret_body in the public view. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt index 24009b9c522..ce534744d57 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that shares a private document with revocable permissions. Return only the module source. Implement one shared private document (ID 1). Its first writer becomes its permanent owner. Only that owner can change its text or reader permissions. Authorized users can query and subscribe to a safe projection. Revocation must remove the document from an existing reader subscription and block future content updates. Do not use row-level security. Keep both source tables private and never expose secret_body in the public view. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/csharp.txt index 6048a875f48..480617fafaa 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/csharp.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in csharp. Return only the module source. +Write a SpacetimeDB backend module in C# that tracks presence across multiple sessions. Return only the module source. Track real connection lifecycle events. One identity may have multiple simultaneous connections. Disconnecting one connection must not mark its other connections offline. Reconnect must rebuild presence without leaving stale sessions. Do not add seed or exercise reducers. The grader opens actual WebSockets with shared and distinct credentials. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/rust.txt index 17e4e86875d..23cae172dd9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/rust.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in rust. Return only the module source. +Write a SpacetimeDB backend module in Rust that tracks presence across multiple sessions. Return only the module source. Track real connection lifecycle events. One identity may have multiple simultaneous connections. Disconnecting one connection must not mark its other connections offline. Reconnect must rebuild presence without leaving stale sessions. Do not add seed or exercise reducers. The grader opens actual WebSockets with shared and distinct credentials. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/typescript.txt index 960e6fa57b1..cdd97ae1db5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/typescript.txt @@ -1,4 +1,4 @@ -Implement this SpacetimeDB module in typescript. Return only the module source. +Write a SpacetimeDB backend module in TypeScript that tracks presence across multiple sessions. Return only the module source. Track real connection lifecycle events. One identity may have multiple simultaneous connections. Disconnecting one connection must not mark its other connections offline. Reconnect must rebuild presence without leaving stale sessions. Do not add seed or exercise reducers. The grader opens actual WebSockets with shared and distinct credentials. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/spec.rs index e5466e2ccd7..4aca77e8f99 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/spec.rs @@ -1,8 +1,23 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let (owner, owner_id) = s.new_user()?; + let (other, _) = s.new_user()?; + s.expect_call_as(&owner, "create_message", json!(["hello"]), None)?; + s.rows("message", &["id", "owner", "text"], json!([[1, owner_id, "hello"]]))?; + s.expect_rejected_as(&other, "delete_message", json!([99]), "not found")?; + s.expect_rejected_as(&other, "delete_message", json!([1]), "unauthorized")?; + s.rows("message", &["id", "owner", "text"], json!([[1, owner_id, "hello"]]))?; + s.expect_call_as(&owner, "delete_message", json!([1]), None)?; + s.rows("message", &["id"], json!([]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt index d1bbef5075b..24db1ea5662 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt @@ -3,6 +3,7 @@ Write a SpacetimeDB backend module in C# that defines a table of messages owned TABLES - Message - Struct: Message + - public table - Fields: - Id: ulong (primary key, AutoInc) - Owner: Identity (index BTree) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/spec.rs index e5466e2ccd7..06830d0b675 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/spec.rs @@ -1,8 +1,25 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.call("register_user", json!(["Ann", "ann@example.com", "secret-hash"]))?; + s.rows("user_public", &["id", "name"], json!([[1, "Ann"]]))?; + s.owner_rows( + "user_internal", + &["id", "name", "email", "password_hash"], + json!([[1, "Ann", "ann@example.com", "secret-hash"]]), + )?; + let (stranger, _) = s.new_user()?; + let query = format!("SELECT * FROM \"{}\"", crate::eval::table_name("user_internal", s.lang)); + let (status, _) = s.sql_as(&stranger, &query)?; + anyhow::ensure!((400..500).contains(&status), "private table exposed: HTTP {status}"); + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs index e5466e2ccd7..2cfc82b2863 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs @@ -1,8 +1,24 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + // C# uses RegisterUser because Register collides with a generated member. + let register = if s.lang == crate::eval::Lang::CSharp { "register_user" } else { "register" }; + let (user, user_id) = s.new_user()?; + s.expect_rejected_as(&user, "post_message", json!(["early"]), "not registered")?; + s.rows("message", &["id"], json!([]))?; + s.expect_call_as(&user, register, json!(["Ann"]), None)?; + s.expect_rejected_as(&user, register, json!(["Ann again"]), "already registered")?; + s.expect_call_as(&user, "post_message", json!(["hello"]), None)?; + s.rows("message", &["id", "sender", "text"], json!([[1, user_id, "hello"]]))?; + s.owner_rows("user", &["identity", "name"], json!([[user_id, "Ann"]]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/spec.rs index e5466e2ccd7..6f74fe27a8f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/spec.rs @@ -1,8 +1,22 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let (admin, _) = s.new_user()?; + let (other, _) = s.new_user()?; + s.expect_rejected_as(&admin, "admin_action", json!(["early"]), "not bootstrapped")?; + s.expect_call_as(&admin, "bootstrap_admin", json!([]), None)?; + s.expect_rejected_as(&other, "bootstrap_admin", json!([]), "already bootstrapped")?; + s.expect_rejected_as(&other, "admin_action", json!(["stolen"]), "not admin")?; + s.expect_call_as(&admin, "admin_action", json!(["allowed"]), None)?; + s.rows("admin_log", &["id", "action"], json!([[1, "allowed"]]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs index e5466e2ccd7..8e87785c3cb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs @@ -1,8 +1,36 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + // C# uses RegisterUser because Register collides with a generated member. + let register = if s.lang == crate::eval::Lang::CSharp { "register_user" } else { "register" }; + let (admin, admin_id) = s.new_user()?; + let (member, member_id) = s.new_user()?; + let (outsider, outsider_id) = s.new_user()?; + s.seed("user", &["identity", "role"], json!([[admin_id, "admin"]]))?; + s.expect_rejected_as(&outsider, "member_action", json!([]), "not registered")?; + s.expect_rejected_as(&outsider, "admin_action", json!([]), "not registered")?; + s.expect_rejected_as(&outsider, "promote", json!([member_id]), "not registered")?; + s.expect_call_as(&member, register, json!([]), None)?; + s.expect_rejected_as(&member, register, json!([]), "already registered")?; + s.expect_call_as(&member, "member_action", json!([]), None)?; + s.expect_rejected_as(&member, "admin_action", json!([]), "not admin")?; + s.expect_rejected_as(&member, "promote", json!([member_id]), "not admin")?; + s.expect_rejected_as(&admin, "promote", json!([outsider_id]), "target not registered")?; + s.expect_call_as(&admin, "promote", json!([member_id]), None)?; + s.expect_call_as(&member, "admin_action", json!([]), None)?; + s.owner_rows( + "user", + &["identity", "role"], + json!([[admin_id, "admin"], [member_id, "admin"]]), + )?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/spec.rs index e5466e2ccd7..051c6e01d36 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/spec.rs @@ -1,8 +1,29 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let (admin, admin_id) = s.new_user()?; + let (player, player_id) = s.new_user()?; + let (other, other_id) = s.new_user()?; + s.seed("admin", &["identity"], json!([[admin_id]]))?; + s.expect_rejected_as(&player, "add_admin", json!([player_id]), "not admin")?; + s.expect_rejected_as(&player, "ban_player", json!([other_id]), "not admin")?; + s.expect_call_as(&admin, "add_admin", json!([other_id]), None)?; + s.expect_call_as(&admin, "add_admin", json!([other_id]), None)?; + s.expect_call_as(&player, "join_game", json!(["Ann"]), None)?; + s.expect_rejected_as(&player, "join_game", json!(["Ann again"]), "already in game")?; + s.rows("player", &["identity", "name"], json!([[player_id, "Ann"]]))?; + s.expect_call_as(&admin, "ban_player", json!([player_id]), None)?; + s.rows("player", &["identity"], json!([]))?; + s.expect_rejected_as(&player, "join_game", json!(["Ann"]), "banned")?; + s.expect_call_as(&other, "ban_player", json!([admin_id]), None)?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/spec.rs index e5466e2ccd7..64a4eec30c1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/spec.rs @@ -1,8 +1,31 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let (first, first_id) = s.new_user()?; + let (second, second_id) = s.new_user()?; + s.expect_call_as(&first, "limited_action", json!(["one"]), None)?; + s.expect_rejected_as(&first, "limited_action", json!(["blocked"]), "rate limited")?; + s.expect_call_as(&second, "limited_action", json!(["other caller"]), None)?; + s.rows( + "action_log", + &["identity", "payload"], + json!([[first_id, "one"], [second_id, "other caller"]]), + )?; + std::thread::sleep(std::time::Duration::from_millis(1100)); + s.expect_call_as(&first, "limited_action", json!(["two"]), None)?; + s.rows( + "action_log", + &["identity", "payload"], + json!([[first_id, "one"], [second_id, "other caller"], [first_id, "two"]]), + )?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs index e5466e2ccd7..297da04aea8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs @@ -1,8 +1,29 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let (owner, owner_id) = s.new_user()?; + let (friend, friend_id) = s.new_user()?; + let (stranger, _) = s.new_user()?; + s.expect_call_as(&owner, "create_document", json!(["draft"]), None)?; + s.rows("document", &["id", "owner", "title"], json!([[1, owner_id, "draft"]]))?; + s.expect_rejected_as(&owner, "share_document", json!([9, friend_id]), "not found")?; + s.expect_rejected_as(&stranger, "share_document", json!([1, friend_id]), "not owner")?; + s.expect_rejected_as(&friend, "edit_document", json!([9, "x"]), "not found")?; + s.expect_rejected_as(&friend, "edit_document", json!([1, "early"]), "unauthorized")?; + s.expect_call_as(&owner, "share_document", json!([1, friend_id]), None)?; + s.expect_call_as(&friend, "edit_document", json!([1, "shared edit"]), None)?; + s.expect_rejected_as(&stranger, "edit_document", json!([1, "stolen"]), "unauthorized")?; + s.rows("document", &["id", "owner", "title"], json!([[1, owner_id, "shared edit"]]))?; + s.expect_call_as(&owner, "edit_document", json!([1, "owner edit"]), None)?; + s.rows("document", &["id", "owner", "title"], json!([[1, owner_id, "owner edit"]]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/spec.rs index e5466e2ccd7..8b444b2d0da 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/spec.rs @@ -1,8 +1,20 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let query = format!("SELECT * FROM \"{}\"", crate::eval::table_name("tick_timer", s.lang)); + let rows = s.owner_sql(&query)?; + anyhow::ensure!(rows.as_array().map(Vec::len) == Some(1), "expected one timer, got {rows}"); + anyhow::ensure!(rows.to_string().contains("50000"), "expected a 50ms interval, got {rows}"); + std::thread::sleep(std::time::Duration::from_millis(300)); + // An interval row must survive its ticks. + anyhow::ensure!(s.owner_sql(&query)? == rows, "interval row changed"); + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/csharp.txt index f01ce023168..cf4dc7f4a85 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines a scheduled table and a scheduled reducer. TABLE -- TickTimer +- TickTimer (private) - Struct: TickTimer - Fields: - ScheduledId: ulong (primary key, auto-increment) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/rust.txt index 332b8dc133f..20e75450ea9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines a scheduled table and a scheduled reducer. TABLE -- tick_timer +- tick_timer (private) - Struct: TickTimer - Fields: - scheduled_id: u64 (primary key, auto-increment) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/typescript.txt index c39a29ab8a5..1b1c10ae5be 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a scheduled table and a scheduled reducer. TABLE -- tick_timer +- tick_timer (private) - Fields: - scheduledId: bigint (u64, primary key, auto-increment) - scheduledAt: ScheduleAt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/csharp.txt index 44ca63669b9..e3cb21944aa 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/csharp.txt @@ -7,7 +7,7 @@ TYPES - Y: int TABLE -- Entity +- Entity (private) - Struct: Entity - Fields: - Id: int (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/rust.txt index e6d8f118c88..ad73e34ca7d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/rust.txt @@ -7,7 +7,7 @@ TYPES - y: i32 TABLE -- entity +- entity (private) - Struct: Entity - Fields: - id: i32 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/typescript.txt index ddf6f46f5c8..68a1c93adbb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_003_struct_in_table/tasks/typescript.txt @@ -6,7 +6,7 @@ TYPES - y: number (i32) TABLE -- entity +- entity (private) - Fields: - id: number (i32, primary key) - pos: Position \ No newline at end of file diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/csharp.txt index 000ee078289..32127a018dd 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines two tables and two reducers: one that inserts a user row, and one that looks up a row by primary-key index and writes a projection to another table. TABLES -- User +- User (private) - Struct: User - Fields: - Id: ulong (primary key, auto-increment) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt index 077a00eb6af..a3c53890d86 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines two tables and two reducers: one that inserts a user row, and one that looks up a row by primary-key index and writes a projection to another table. TABLES -- user +- user (private) - Struct: User - Fields: - id: u64 (primary key, auto_inc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt index 0ad5d7cb66e..de9ed908263 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines two tables and two reducers: one that inserts a user row, and one that looks up a row by primary-key index and writes a projection to another table. TABLES -- user +- user (private) - Fields: - id: number (u64, primary key, autoInc) - name: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/spec.rs index e5466e2ccd7..e29dfb866df 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/spec.rs @@ -1,8 +1,30 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let (user, _) = s.new_user()?; + drop(s.connect(&user)?); + let count = |kind: &str| -> anyhow::Result { + let table = crate::eval::table_name("event", s.lang); + let column = crate::eval::ident("kind", crate::eval::casing_for_lang(s.lang)); + let query = format!("SELECT COUNT(*) AS n FROM \"{table}\" WHERE \"{column}\" = '{kind}'"); + Ok(s.owner_sql(&query)?[0][0].as_u64().unwrap_or_default()) + }; + s.eventually(|| { + let (connected, disconnected) = (count("connected")?, count("disconnected")?); + // HTTP SQL runs the same hooks, so the reading itself can be one event ahead. + anyhow::ensure!( + connected >= 1 && disconnected >= 1 && connected.abs_diff(disconnected) <= 1, + "connected {connected}, disconnected {disconnected}" + ); + Ok(()) + })?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs index e5466e2ccd7..7cf3dc5256c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs @@ -1,8 +1,24 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let table = crate::eval::table_name("reminder", s.lang); + let column = crate::eval::ident("scheduled_at", crate::eval::casing_for_lang(s.lang)); + let rows = s.owner_sql(&format!("SELECT \"{column}\" FROM \"{table}\""))?; + anyhow::ensure!(rows.as_array().map(Vec::len) == Some(1), "expected one reminder, got {rows}"); + // Tag 1 is ScheduleAt::Time, about 60 seconds after publishing, not an interval. + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_micros() as i64; + let due = rows[0][0][1][0].as_i64().unwrap_or_default(); + anyhow::ensure!( + rows[0][0][0] == 1 && due > now && due <= now + 61_000_000, + "expected a time about 60 seconds ahead, got {rows}" + ); + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt index d4fec2a3d61..0773d06ea68 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that schedules a one-shot reducer to fire at a specific future time. TABLE -- Reminder +- Reminder (private) - Fields: - ScheduledId: ulong (primary key, auto-increment) - ScheduledAt: ScheduleAt (the schedule column) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt index 8e601abb547..f2d79b2773e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that schedules a one-shot reducer to fire at a specific future time. TABLE -- reminder +- reminder (private) - Struct: Reminder - Fields: - scheduled_id: u64 (primary key, auto-increment) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt index 7396a8784d9..88d00e86ec0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that schedules a one-shot reducer to fire at a specific future time. TABLE -- reminder +- reminder (private) - Fields: - scheduledId: u64 (primary key, auto-increment) - scheduledAt: scheduleAt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs index e5466e2ccd7..4587c529c87 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs @@ -1,8 +1,22 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let table = crate::eval::table_name("cleanup_job", s.lang); + let query = format!("SELECT * FROM \"{table}\""); + let rows = s.owner_sql(&query)?; + anyhow::ensure!(rows.as_array().map(Vec::len) == Some(1), "expected one job, got {rows}"); + let id = rows[0][0].clone(); + s.call("cancel_cleanup", json!([id]))?; + let rows = s.owner_sql(&query)?; + anyhow::ensure!(rows.as_array().map(Vec::is_empty) == Some(true), "job not cancelled: {rows}"); + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt index d775e92fe0f..94e183745b9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that schedules a reducer and allows canceling it before it fires. TABLE -- CleanupJob +- CleanupJob (private) - Fields: - ScheduledId: ulong (primary key, auto-increment) - ScheduledAt: ScheduleAt (the schedule column) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt index 949b98b99f7..9497de9be70 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that schedules a reducer and allows canceling it before it fires. TABLE -- cleanup_job +- cleanup_job (private) - Struct: CleanupJob - Fields: - scheduled_id: u64 (primary key, auto-increment) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt index f7bc2aba746..ab0404bfd89 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that schedules a reducer and allows canceling it before it fires. TABLE -- cleanup_job +- cleanup_job (private) - Fields: - scheduledId: u64 (primary key, auto-increment) - scheduledAt: scheduleAt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/spec.rs index e5466e2ccd7..264ef8c4c7d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/spec.rs @@ -1,8 +1,19 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let (user, user_id) = s.new_user()?; + let live = s.connect(&user)?; + s.presence_rows("online_player", "identity", &["identity"], json!([[user_id]]))?; + drop(live); + s.presence_rows("online_player", "identity", &["identity"], json!([]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs index e5466e2ccd7..992369c04ca 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs @@ -1,8 +1,18 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let query = format!("SELECT * FROM \"{}\"", crate::eval::table_name("damage_event", s.lang)); + let update = s.watch_update(&query, || s.call("deal_damage", json!([7, 12, "fire"])))?; + anyhow::ensure!(update.contains("fire") && update.contains("12"), "missing event: {update}"); + s.rows("damage_event", &["entity_id", "damage", "source"], json!([]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/spec.rs index e5466e2ccd7..fe81e5b7101 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/spec.rs @@ -1,8 +1,22 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.call("create_player", json!(["Ann", {"some": "Ace"}, {"some": 10}]))?; + s.call("create_player", json!(["Bo", {"none": []}, {"none": []}]))?; + let table = crate::eval::table_name("player", s.lang); + let rows = s.owner_sql(&format!("SELECT * FROM \"{table}\""))?; + anyhow::ensure!( + rows == json!([[1, "Ann", [0, "Ace"], [0, 10]], [2, "Bo", [1, []], [1, []]]]), + "unexpected players: {rows}" + ); + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/spec.rs index e5466e2ccd7..2a8948b8ade 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/spec.rs @@ -1,8 +1,23 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.seed("author", &["id", "name"], json!([[1, "gone"], [2, "kept"]]))?; + s.seed( + "post", + &["id", "author_id", "title"], + json!([[1, 1, "a"], [2, 1, "b"], [3, 2, "c"]]), + )?; + s.call("delete_author", json!([1]))?; + s.owner_rows("author", &["id", "name"], json!([[2, "kept"]]))?; + s.owner_rows("post", &["id", "author_id", "title"], json!([[3, 2, "c"]]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs index e5466e2ccd7..1fb5df8a9cf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs @@ -1,8 +1,24 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.seed( + "order", + &["id", "category", "amount", "fulfilled"], + json!([[1, "tools", 10, true], [2, "tools", 25, false], [3, "toys", 99, true]]), + )?; + s.call("compute_stats", json!(["tools"]))?; + s.owner_rows("category_stats", &["category", "total_amount", "order_count"], json!([["tools", 35, 2]]))?; + s.seed("order", &["id", "category", "amount", "fulfilled"], json!([[4, "tools", 5, true]]))?; + s.call("compute_stats", json!(["tools"]))?; + s.owner_rows("category_stats", &["category", "total_amount", "order_count"], json!([["tools", 40, 3]]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/spec.rs index e5466e2ccd7..ab8d26d53c2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/spec.rs @@ -1,8 +1,26 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.seed("customer", &["id", "name"], json!([[1, "Ann"], [2, "Bo"]]))?; + s.seed( + "order", + &["id", "customer_id", "product", "amount"], + json!([[10, 1, "lamp", 3], [11, 2, "desk", 1], [12, 99, "orphan", 7]]), + )?; + s.call("build_order_details", json!([]))?; + s.owner_rows( + "order_detail", + &["order_id", "customer_name", "product", "amount"], + json!([[10, "Ann", "lamp", 3], [11, "Bo", "desk", 1]]), + )?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/spec.rs index e5466e2ccd7..75bda0a897c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/spec.rs @@ -1,8 +1,24 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let (user, _) = s.new_user()?; + s.expect_call_as(&user, "create_account", json!(["a@example.com", "first"]), None)?; + let (status, body) = s.call_as(&user, "create_account", json!(["a@example.com", "second"]))?; + anyhow::ensure!(status == 530, "duplicate email accepted: HTTP {status}: {body}"); + s.expect_call_as(&user, "create_account", json!(["b@example.com", "third"]), None)?; + s.owner_rows( + "account", + &["email", "display_name"], + json!([["a@example.com", "first"], ["b@example.com", "third"]]), + )?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt index cdbb236e4e9..27abf405d4f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in C# that defines a procedure with a typed return value. +The module defines no tables. + TYPES - Summary - Total: uint diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt index f2df9647087..09a0ae0e9bd 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines a procedure with a typed return value. +The module defines no tables. + TYPES - Summary - total: u32 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt index 48ee3ce58f6..5d981911361 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a procedure with a typed return value. +The module defines no tables. + TYPES - Summary - total: u32 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt index e74391b2da4..31d5a74ebbd 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt @@ -10,7 +10,7 @@ TABLE - ValidBody: bool PROCEDURE -- FetchAndStore(url: string) +- FetchAndStore(url: string) -> void - Perform an outbound GET to the supplied public URL before opening a transaction - Inside a short transaction, store Id 1 and the response status - Set ValidBody based on whether the response body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt index 67d18b777af..0baada33970 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt @@ -10,7 +10,7 @@ TABLE - valid_body: bool PROCEDURE -- fetch_and_store(url: String) +- fetch_and_store(url: String) -> () - Perform an outbound GET to the supplied public URL before opening a transaction - Inside a short transaction, store id 1 and the response status - Set valid_body based on whether the response body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt index 087c8bcce0d..b2bf4f40170 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt @@ -10,7 +10,7 @@ TABLE - validBody: bool PROCEDURE -- fetch_and_store(url: string) +- fetch_and_store(url: string) -> unit - Perform an outbound GET to the supplied public URL before opening a transaction - Inside a short transaction, store id 1 and the response status - Set validBody based on whether the response body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt index 519ad043afc..7c9f8699b0a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in C# that defines and routes an inbound HTTP handler. +The module defines no tables. + HANDLER - POST /echo - Read the request body diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt index 628b58a290b..2d1d23466a2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines and routes an inbound HTTP handler. +The module defines no tables. + HANDLER - POST /echo - Read the request body diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt index 9595ae843f7..2dac719f87f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines and routes an inbound HTTP handler. +The module defines no tables. + HANDLER - POST /echo - Read the request body diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt index e1dcc494794..b9dde3c0800 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in C# that routes two HTTP methods on the same path. +The module defines no tables. + HANDLERS - GET /items - Return status 200 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt index ed040b322f9..bfc70c38c61 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in Rust that routes two HTTP methods on the same path. +The module defines no tables. + HANDLERS - GET /items - Return status 200 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt index cfa7421bee6..97f2cb126ca 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that routes two HTTP methods on the same path. +The module defines no tables. + HANDLERS - GET /items - Return status 200 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/spec.rs index e5466e2ccd7..93c82923e29 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_022_view_basic/spec.rs @@ -1,8 +1,24 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.seed( + "announcement", + &["id", "message", "active"], + json!([[1, "shown", true], [2, "hidden", false], [3, "also shown", true]]), + )?; + s.rows( + "active_announcements", + &["id", "message", "active"], + json!([[1, "shown", true], [3, "also shown", true]]), + )?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs index e5466e2ccd7..1beb87ed548 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs @@ -1,8 +1,25 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let (first, first_id) = s.new_user()?; + let (second, second_id) = s.new_user()?; + let (stranger, _) = s.new_user()?; + s.seed( + "profile", + &["id", "identity", "name", "bio"], + json!([[1, first_id, "Ann", "first"], [2, second_id, "Bo", "second"]]), + )?; + s.rows_as(&first, "my_profile", &["id", "name", "bio"], json!([[1, "Ann", "first"]]))?; + s.rows_as(&second, "my_profile", &["id", "name", "bio"], json!([[2, "Bo", "second"]]))?; + s.rows_as(&stranger, "my_profile", &["id", "name", "bio"], json!([]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs index e5466e2ccd7..3269ff9fc0a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs @@ -1,8 +1,25 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.seed( + "product", + &["id", "name", "price"], + json!([[1, "low", 9], [2, "min", 10], [3, "mid", 15], [4, "max", 20], [5, "high", 21]]), + )?; + s.call("find_in_price_range", json!([10, 20]))?; + s.owner_rows( + "price_range_result", + &["product_id", "name", "price"], + json!([[2, "min", 10], [3, "mid", 15], [4, "max", 20]]), + )?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/spec.rs index e5466e2ccd7..778a47b0d47 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/spec.rs @@ -1,8 +1,25 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.seed( + "player", + &["id", "name", "score"], + json!([[1, "low", 5], [2, "top", 50], [3, "mid", 20], [4, "second", 30]]), + )?; + s.call("build_leaderboard", json!([2]))?; + s.owner_rows( + "leaderboard", + &["rank", "player_name", "score"], + json!([[1, "top", 50], [2, "second", 30]]), + )?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/spec.rs index e5466e2ccd7..7dd35176792 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/spec.rs @@ -1,8 +1,22 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + // Only one incomplete task, because the prompt does not define an iteration order. + s.seed( + "task", + &["id", "title", "completed"], + json!([[1, "done", true], [2, "open", false], [3, "also done", true]]), + )?; + s.call("find_first_incomplete", json!([]))?; + s.owner_rows("first_incomplete", &["task_id", "title"], json!([[2, "open"]]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs index e5466e2ccd7..02a3d803fd9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs @@ -1,8 +1,21 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.seed( + "order", + &["id", "category", "amount"], + json!([[1, "tools", 1], [2, "toys", 2], [3, "tools", 3], [4, "books", 4]]), + )?; + s.call("collect_distinct_categories", json!([]))?; + s.owner_rows("distinct_category", &["category"], json!([["books"], ["tools"], ["toys"]]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/spec.rs index e5466e2ccd7..2df6ee6f68f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/spec.rs @@ -1,8 +1,21 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.seed( + "user", + &["id", "name", "active"], + json!([[1, "a", true], [2, "b", false], [3, "c", true]]), + )?; + s.call("compute_user_counts", json!([]))?; + s.owner_rows("user_stats", &["key", "count"], json!([["active", 2], ["total", 3]]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs index e5466e2ccd7..23eb918d611 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs @@ -1,8 +1,21 @@ use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; use crate::eval::BenchmarkSpec; +use serde_json::json; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { - default_schema_parity_scorers(host_url, file!(), route_tag) + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + s.seed( + "event_log", + &["id", "category", "severity", "message"], + json!([[1, "db", 2, "match"], [2, "db", 3, "wrong severity"], [3, "net", 2, "wrong category"], [4, "db", 2, "also match"]]), + )?; + s.call("filter_events", json!(["db", 2]))?; + s.owner_rows("filtered_event", &["event_id", "message"], json!([[1, "match"], [4, "also match"]]))?; + Ok(()) + })); + scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/csharp.txt index 6f286f438d2..283a20398f1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/csharp.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in C# that defines three tables modeling a many-to-many relationship and seeds rows. TABLES -- User +- User (private) - Struct: User - Fields: - UserId: ulong (primary key, auto-increment) - Name: string -- Group +- Group (private) - Struct: Group - Fields: - GroupId: ulong (primary key, auto-increment) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/rust.txt index 1f70f9c4864..104222701a7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/rust.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in Rust that defines three tables modeling a many-to-many relationship and seeds rows. TABLES -- user +- user (private) - Struct: User - Fields: - user_id: u64 (primary key, auto_inc) - name: String -- group +- group (private) - Struct: Group - Fields: - group_id: u64 (primary key, auto_inc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt index 82a6457926d..59e1a1563d3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt @@ -1,12 +1,12 @@ Write a SpacetimeDB backend module in TypeScript that defines three tables modeling a many-to-many relationship and seeds rows. TABLES -- user +- user (private) - Fields: - userId: number (u64, primary key, autoInc) - name: string -- group +- group (private) - Fields: - groupId: number (u64, primary key, autoInc) - title: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/csharp.txt index 41cb11e201b..f15de58c4ad 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that models a minimal ECS and computes next positions. TABLES -- Entity +- Entity (private) - Struct: Entity - Fields: - Id: ulong (primary key, auto-increment) @@ -13,7 +13,7 @@ TABLES - X: int - Y: int -- Velocity +- Velocity (private) - Struct: Velocity - Fields: - EntityId: int (primary key) — NOT auto-increment, this is a foreign key to Entity diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/rust.txt index 1d14c4d02c8..30ee432dc85 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that models a minimal ECS and computes next positions. TABLES -- entity +- entity (private) - Struct: Entity - Fields: - id: u64 (primary key, auto_inc) @@ -13,7 +13,7 @@ TABLES - x: i32 - y: i32 -- velocity +- velocity (private) - Struct: Velocity - Fields: - entity_id: i32 (primary key) — NOT auto_inc, this is a foreign key to entity diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt index cceed5f55af..225703a9966 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that models a minimal ECS and computes next positions. TABLES -- entity +- entity (private) - Fields: - id: number (u64, primary key, autoInc) @@ -11,7 +11,7 @@ TABLES - x: number (i32) - y: number (i32) -- velocity +- velocity (private) - Fields: - entityId: number (i32, primary key) — NOT autoInc, this is a foreign key to entity - vx: number (i32) diff --git a/tools/xtask-llm-benchmark/src/eval/scenario.rs b/tools/xtask-llm-benchmark/src/eval/scenario.rs index 6363b295b3a..328ccc8c369 100644 --- a/tools/xtask-llm-benchmark/src/eval/scenario.rs +++ b/tools/xtask-llm-benchmark/src/eval/scenario.rs @@ -137,6 +137,17 @@ impl Scenario { } pub fn sql_as(&self, token: &str, query: &str) -> Result<(u16, String)> { + let result = + super::scorers::with_actual_relation_names(&self.db, query, Some(&self.server), |query| { + match self.sql_once(token, query) { + Ok((400, body)) if body.contains("no such table") => Err(body), + other => Ok(other), + } + }); + result.unwrap_or_else(|body| Ok((400, body))) + } + + fn sql_once(&self, token: &str, query: &str) -> Result<(u16, String)> { let mut request = reqwest::blocking::Client::builder() .timeout(Duration::from_secs(15)) .build()? @@ -196,6 +207,98 @@ impl Scenario { call_reducer_json_out(&self.db, reducer, args, Some(&self.server)) } + pub fn owner_token(&self) -> Result { + crate::bench::publishers::database_owner_token(&self.db) + } + + pub fn owner_rows(&self, table: &str, columns: &[&str], expected: Value) -> Result<()> { + self.rows_as(&self.owner_token()?, table, columns, expected) + } + + pub fn owner_sql(&self, query: &str) -> Result { + let (status, body) = self.sql_as(&self.owner_token()?, query)?; + ensure!(status == 200, "{query}: HTTP {status}: {body}"); + let result: Value = serde_json::from_str(&body)?; + Ok(result[0]["rows"].clone()) + } + + /// Inserts rows as the owner, for tasks with private tables and no reducer that adds rows. + pub fn seed(&self, table: &str, columns: &[&str], rows: Value) -> Result<()> { + let literal = |value: &Value| match value { + Value::String(text) => format!("'{}'", text.replace('\'', "''")), + Value::Object(_) => value["__identity__"].as_str().unwrap_or_default().to_owned(), + other => other.to_string(), + }; + let columns = columns + .iter() + .map(|c| format!("\"{}\"", ident(c, casing_for_lang(self.lang)))) + .collect::>() + .join(", "); + for row in rows.as_array().ok_or_else(|| anyhow!("rows must be an array"))? { + let values = row + .as_array() + .ok_or_else(|| anyhow!("row must be an array"))? + .iter() + .map(literal) + .collect::>() + .join(", "); + let table = table_name(table, self.lang); + self.owner_sql(&format!("INSERT INTO \"{table}\" ({columns}) VALUES ({values})"))?; + } + Ok(()) + } + + /// A panic or plain exception reaches the caller as a generic fatal error, without its message. + pub fn expect_rejected_as(&self, token: &str, reducer: &str, args: Value, expected: &str) -> Result<()> { + let (status, body) = self.call_as(token, reducer, args)?; + ensure!( + status == 530 && (body.contains(expected) || body.contains("fatal error")), + "{reducer}: expected rejection {expected:?}, got HTTP {status}: {body}" + ); + Ok(()) + } + + /// Event tables need a binary WebSocket protocol, so this subscribes through the CLI. + pub fn watch_update(&self, query: &str, action: impl FnOnce() -> Result<()>) -> Result { + use std::io::{BufRead, BufReader, Read}; + let mut child = super::scorer_command(Some(&self.server)) + .args([ + "subscribe", + "--print-initial-update", + "--num-updates", + "1", + "--timeout", + "15", + ]) + .arg("--server") + .arg(&self.server) + .arg(&self.db) + .arg(query) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn()?; + let mut stdout = BufReader::new(child.stdout.take().ok_or_else(|| anyhow!("no subscribe output"))?); + let mut initial = String::new(); + stdout.read_line(&mut initial)?; + let result = action(); + let mut update = String::new(); + stdout.read_to_string(&mut update)?; + let _ = child.wait(); + result?; + Ok(update) + } + + pub fn eventually(&self, check: impl Fn() -> Result<()>) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + match check() { + Ok(()) => return Ok(()), + Err(error) if Instant::now() >= deadline => return Err(error), + Err(_) => std::thread::sleep(Duration::from_millis(100)), + } + } + } + /// Require the contract's error text. Transport errors and missing reducers must not pass. pub fn reject(&self, reducer: &str, args: Value, expected: &str) -> Result<()> { match self.call_result(reducer, &args) { @@ -272,7 +375,17 @@ impl Scenario { } } +/// SQL rows print an identity as a one-element array, unlike its JSON argument form. +fn identity_cells(value: &Value) -> Value { + match value { + Value::Array(items) => Value::Array(items.iter().map(identity_cells).collect()), + Value::Object(fields) if fields.contains_key("__identity__") => json!([fields["__identity__"]]), + other => other.clone(), + } +} + fn compare_rows(actual: &Value, expected: &Value) -> Result<()> { + let expected = &identity_cells(expected); let sorted = |v: &Value| -> Result> { let mut rows = v .as_array() diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 75e31afa8bc..9ab9d7be61d 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -108,9 +108,15 @@ impl Scorer for SchemaParityScorer { } let schema_a = extract_schema(&golden); - let schema_b = extract_schema(&llm); + let mut schema_b = extract_schema(&llm); + // Extra view primary keys only help clients match rows; behaviour checks catch a wrong one. + schema_b + .exports + .retain(|export| !export.starts_with("view_primary_key:") || schema_a.exports.contains(export)); + let candidate_tables = + accept_extra_indexes(&schema_a.tables, align_table_names(&schema_a.tables, &schema_b.tables)); - let tables_diff = diff_maps(&schema_a.tables, &align_table_names(&schema_a.tables, &schema_b.tables)); + let tables_diff = diff_maps(&schema_a.tables, &candidate_tables); let reducers_diff = diff_sets(&schema_a.reducers, &schema_b.reducers); let exports_diff = diff_sets(&schema_a.exports, &schema_b.exports); let rls_diff = diff_sets(&schema_a.row_level_security, &schema_b.row_level_security); @@ -250,9 +256,30 @@ fn extract_schema(v: &Value) -> SchemaSnapshot { } } + // Clients cannot call lifecycle reducers, so they match by lifecycle kind, not by name. + let mut lifecycle_kinds = BTreeMap::new(); + for lifecycle in schema_array(v, "life_cycle_reducers", "LifeCycleReducers") + .into_iter() + .flatten() + { + let spec = lifecycle.get("lifecycle_spec"); + let kind = spec + .and_then(Value::as_object) + .and_then(|spec| spec.keys().next().cloned()) + .unwrap_or_else(|| canonical_value(spec)); + lifecycle_kinds.insert(schema_name(lifecycle.get("function_name")), kind.clone()); + schema.exports.insert(format!("lifecycle:{kind}")); + } + if let Some(rs) = schema_array(v, "reducers", "Reducers") { for r in rs { - let name = names.function(schema_name(r.get("source_name").or_else(|| r.get("name")))); + let source = schema_name(r.get("source_name").or_else(|| r.get("name"))); + let name = names.function(source.clone()); + let name = lifecycle_kinds + .get(&name) + .or_else(|| lifecycle_kinds.get(&source)) + .map(|kind| format!("<{kind}>")) + .unwrap_or(name); let mut sig = format!("{}({})", name, parameter_types(r, types).join(",")); append_field(&mut sig, "visibility", r.get("visibility")); append_type(&mut sig, "ok", r.get("ok_return_type"), types); @@ -285,21 +312,19 @@ fn extract_schema(v: &Value) -> SchemaSnapshot { schema.exports.insert(view_export(view, types, &names)); } insert_canonical_exports(&mut schema.exports, v, "schedules", "Schedules", "schedule"); - insert_canonical_exports( - &mut schema.exports, - v, - "life_cycle_reducers", - "LifeCycleReducers", - "lifecycle", - ); insert_http_routes(&mut schema.exports, v); - insert_canonical_exports( - &mut schema.exports, - v, - "view_primary_keys", - "ViewPrimaryKeys", - "view_primary_key", - ); + for key in schema_array(v, "view_primary_keys", "ViewPrimaryKeys") + .into_iter() + .flatten() + { + let mut key = key.clone(); + if let Some(view) = key.get_mut("view_source_name") { + *view = Value::String(loose_name(&schema_name(Some(view)))); + } + schema + .exports + .insert(format!("view_primary_key:{}", canonical_value(Some(&key)))); + } for rule in schema_array(v, "row_level_security", "RowLevelSecurity") .into_iter() @@ -361,14 +386,20 @@ fn function_export(kind: &str, function: &Value, types: &[Value], names: &Schema let name = names.function(schema_name( function.get("source_name").or_else(|| function.get("name")), )); + named_function_export(kind, &name, function, types) +} + +fn named_function_export(kind: &str, name: &str, function: &Value, types: &[Value]) -> String { let mut signature = format!("{kind}:{name}({})", parameter_types(function, types).join(",")); append_type(&mut signature, "return", function.get("return_type"), types); append_field(&mut signature, "visibility", function.get("visibility")); signature } +// Prompts do not fix a view's casing, and TypeScript modules often camelCase it. fn view_export(view: &Value, types: &[Value], names: &SchemaNames) -> String { - let mut signature = function_export("view", view, types, names); + let name = names.function(schema_name(view.get("source_name").or_else(|| view.get("name")))); + let mut signature = named_function_export("view", &loose_name(&name), view, types); append_field(&mut signature, "public", view.get("is_public")); append_field(&mut signature, "anonymous", view.get("is_anonymous")); signature @@ -524,18 +555,47 @@ fn align_table_names( golden: &BTreeMap>, candidate: &BTreeMap>, ) -> BTreeMap> { - let key = |name: &str| name.to_lowercase().replace('_', ""); candidate .iter() .map(|(name, table)| { - let golden_name = golden - .keys() - .find(|golden_name| !candidate.contains_key(*golden_name) && key(golden_name) == key(name)); + let golden_name = golden.keys().find(|golden_name| { + !candidate.contains_key(*golden_name) && loose_name(golden_name) == loose_name(name) + }); (golden_name.unwrap_or(name).clone(), table.clone()) }) .collect() } +fn loose_name(name: &str) -> String { + name.to_lowercase().replace('_', "") +} + +// Extra indexes only speed up lookups; a missing golden index still fails. +fn accept_extra_indexes( + golden: &BTreeMap>, + mut candidate: BTreeMap>, +) -> BTreeMap> { + for (name, table) in &mut candidate { + let Some(golden_table) = golden.get(name) else { + continue; + }; + let expected = golden_table.get("@indexes").map(String::as_str).unwrap_or(""); + let actual = table.get("@indexes").map(String::as_str).unwrap_or(""); + let actual: BTreeSet<&str> = actual.split(';').collect(); + if expected + .split(';') + .filter(|index| !index.is_empty()) + .all(|index| actual.contains(index)) + { + match golden_table.get("@indexes") { + Some(expected) => table.insert("@indexes".to_owned(), expected.clone()), + None => table.remove("@indexes"), + }; + } + } + candidate +} + // shortcut: text match for the 1.x C# form, which the schema cannot show; delete once the C# SDK rejects it as Rust does. fn has_table_name_without_accessor(source: &str) -> bool { static COMMENT: LazyLock = LazyLock::new(|| Regex::new(r"(?s)/\*.*?\*/|//[^\n]*").unwrap()); @@ -649,7 +709,66 @@ pub fn sql_raw(db: &str, query: &str, host: Option<&str>) -> Result( + db: &str, + query: &str, + host: Option<&str>, + run: impl Fn(&str) -> Result, +) -> Result { + static MISSING: LazyLock = LazyLock::new(|| Regex::new(r"no such table: `([^`]+)`").unwrap()); + let mut query = query.to_owned(); + for _ in 0..4 { + let error = match run(&query) { + Ok(value) => return Ok(value), + Err(error) => error, + }; + let renamed = MISSING.captures(&error).and_then(|missing| { + let actual = actual_relation_name(host?, db, &missing[1])?; + Some(rename_relation(&query, &missing[1], &actual)) + }); + match renamed { + Some(renamed) if renamed != query => query = renamed, + _ => return Err(error), + } + } + run(&query) +} + +fn rename_relation(query: &str, missing: &str, actual: &str) -> String { + let pattern = Regex::new(&format!(r#""{0}"|\b{0}\b"#, regex::escape(missing))).unwrap(); + pattern + .replace_all(query, format!("\"{actual}\"").as_str()) + .into_owned() +} + +fn actual_relation_name(server: &str, db: &str, missing: &str) -> Option { + let schema = describe_db(server, db, Duration::from_secs(15)).ok()?; + let names = SchemaNames::from_schema(&schema); + ["Tables", "Views"] + .into_iter() + .flat_map(|section| { + schema_array(&schema, §ion.to_lowercase(), section) + .into_iter() + .flatten() + }) + .map(|relation| { + let source = schema_name(relation.get("source_name").or_else(|| relation.get("name"))); + names + .tables + .get(&source) + .or_else(|| names.functions.get(&source)) + .cloned() + .unwrap_or(source) + }) + .find(|name| name != missing && loose_name(name) == loose_name(missing)) +} + fn sql_raw_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { + with_actual_relation_names(db, query, host, |query| sql_raw_once(db, query, host, timeout)) +} + +fn sql_raw_once(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { let mut cmd = scorer_command(host); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { @@ -677,6 +796,10 @@ fn sql_raw_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Dura } fn sql_json_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { + with_actual_relation_names(db, query, host, |query| sql_json_once(db, query, host, timeout)) +} + +fn sql_json_once(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { let mut cmd = scorer_command(host); cmd.arg("sql").arg(db).arg(query).args(["--format", "json"]); if let Some(h) = host { @@ -706,6 +829,10 @@ pub fn sql_count(db: &str, query: &str, host: Option<&str>) -> Result, timeout: Duration) -> Result { + with_actual_relation_names(db, query, host, |query| sql_count_once(db, query, host, timeout)) +} + +fn sql_count_once(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { let mut cmd = scorer_command(host); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { @@ -1481,6 +1608,71 @@ mod tests { let candidate = extract_schema(¤t_schema(false)); assert!(!diff_maps(&golden.tables, &candidate.tables).is_null()); + assert!(!diff_maps(&golden.tables, &accept_extra_indexes(&golden.tables, candidate.tables)).is_null()); + } + + #[test] + fn missing_relation_is_renamed_quoted_or_bare() { + assert_eq!( + rename_relation(r#"SELECT * FROM "OnlineUser" WHERE x = 1"#, "OnlineUser", "online_user"), + r#"SELECT * FROM "online_user" WHERE x = 1"# + ); + assert_eq!( + rename_relation("SELECT COUNT(*) FROM OnlineUser", "OnlineUser", "online_user"), + r#"SELECT COUNT(*) FROM "online_user""# + ); + assert_eq!( + rename_relation("SELECT * FROM OnlineUsers", "OnlineUser", "online_user"), + "SELECT * FROM OnlineUsers" + ); + } + + #[test] + fn extra_index_is_accepted() { + let golden = extract_schema(¤t_schema(false)).tables; + let candidate = extract_schema(¤t_schema(true)).tables; + + assert!(diff_maps(&golden, &accept_extra_indexes(&golden, candidate)).is_null()); + } + + fn lifecycle_schema(function: &str, kind: &str) -> Value { + json!({ + "sections": [ + { "Typespace": { "types": [] } }, + { "Reducers": [{ "source_name": function, "params": { "elements": [] } }] }, + { "LifeCycleReducers": [{ "lifecycle_spec": { kind: [] }, "function_name": function }] } + ] + }) + } + + #[test] + fn lifecycle_reducers_match_by_kind_not_name() { + let golden = extract_schema(&lifecycle_schema("client_connected", "OnConnect")); + + assert_eq!(golden, extract_schema(&lifecycle_schema("on_connect", "OnConnect"))); + assert_ne!(golden, extract_schema(&lifecycle_schema("client_connected", "Init"))); + } + + #[test] + fn view_names_differing_only_by_case_or_underscores_match() { + let schema = |view: &str| { + json!({ + "sections": [ + { "Typespace": { "types": [] } }, + { "Views": [{ "source_name": view, "params": { "elements": [] }, "is_public": true }] }, + { "ViewPrimaryKeys": [{ "view_source_name": view, "columns": ["id"] }] } + ] + }) + }; + + assert_eq!( + extract_schema(&schema("active_announcements")), + extract_schema(&schema("activeAnnouncements")) + ); + assert_ne!( + extract_schema(&schema("active_announcements")), + extract_schema(&schema("announcements")) + ); } fn tables_named(names: &[&str]) -> BTreeMap> { diff --git a/tools/xtask-llm-benchmark/src/eval/utils.rs b/tools/xtask-llm-benchmark/src/eval/utils.rs index 1125e5d0b42..2e7c052a100 100644 --- a/tools/xtask-llm-benchmark/src/eval/utils.rs +++ b/tools/xtask-llm-benchmark/src/eval/utils.rs @@ -1,7 +1,9 @@ +use std::collections::BTreeSet; use std::env; use std::io::{self, Read}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +use std::sync::Mutex; use std::thread; use std::time::{Duration, Instant}; @@ -62,8 +64,23 @@ fn scorer_cli_root(host: Option<&str>) -> PathBuf { // A saved token only works on the server that issued it, and each C# batch starts a new server. pub(crate) fn scorer_command(host: Option<&str>) -> Command { + static LOGGED_IN: Mutex> = Mutex::new(BTreeSet::new()); let root = scorer_cli_root(host); let _ = std::fs::create_dir_all(&root); + if let Some(host) = host { + // Otherwise the first command on a fresh root prints a login prompt into the stdout scorers parse. + let mut logged_in = LOGGED_IN.lock().unwrap_or_else(|e| e.into_inner()); + if !logged_in.contains(&root) { + let mut login = spacetime_command(); + login + .arg("--root-dir") + .arg(&root) + .args(["login", "--no-browser", "--server-issued-login", host]); + if run_with_timeout(login, Path::new("."), Duration::from_secs(30)).is_ok_and(|(code, _, _)| code == 0) { + logged_in.insert(root.clone()); + } + } + } let mut cmd = spacetime_command(); cmd.arg("--root-dir").arg(root); cmd @@ -79,17 +96,19 @@ pub(crate) fn sql_exec_with_timeout( host: Option<&str>, timeout: Duration, ) -> Result<(), String> { - let mut cmd = scorer_command(host); - cmd.arg("sql").arg(db).arg(query); - if let Some(h) = host { - cmd.arg("--server").arg(h); - } - let (code, _, stderr) = run_with_timeout(cmd, Path::new("."), timeout) - .map_err(|e| format!("spacetime sql failed or timed out: {e}"))?; - if code != 0 { - return Err(format!("spacetime sql failed:\n{}", String::from_utf8_lossy(&stderr))); - } - Ok(()) + super::scorers::with_actual_relation_names(db, query, host, |query| { + let mut cmd = scorer_command(host); + cmd.arg("sql").arg(db).arg(query); + if let Some(h) = host { + cmd.arg("--server").arg(h); + } + let (code, _, stderr) = run_with_timeout(cmd, Path::new("."), timeout) + .map_err(|e| format!("spacetime sql failed or timed out: {e}"))?; + if code != 0 { + return Err(format!("spacetime sql failed:\n{}", String::from_utf8_lossy(&stderr))); + } + Ok(()) + }) } pub(crate) fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) -> io::Result<(i32, Vec, Vec)> { diff --git a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs index 4bcc0426dc1..7dbb64e7452 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs @@ -3,8 +3,8 @@ use super::oa_compat::ProviderRefusal; use crate::bench::utils::debug_llm_verbose; use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ - anthropic_ctx_limit_tokens, build_anthropic_messages, desired_output_tokens, deterministic_trim_prefix, - estimate_tokens, headroom_tokens_env, non_context_reserve_tokens_env, + anthropic_ctx_limit_tokens, build_anthropic_messages, deterministic_trim_prefix, estimate_tokens, + headroom_tokens_env, non_context_reserve_tokens_env, output_token_limit_env, }; use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; use anyhow::{anyhow, Context, Result}; @@ -252,8 +252,9 @@ fn compute_backoff(status: StatusCode, headers: &HeaderMap, attempt: u32) -> std std::time::Duration::from_millis(base_ms.min(cap_ms) + jitter) } +// Thinking counts against max_tokens, so the shared 4096 default truncates long answers. fn anthropic_max_output_tokens() -> u32 { - desired_output_tokens().max(1) as u32 + output_token_limit_env().unwrap_or(16_000).max(1) as u32 } pub fn normalize_anthropic_model(id: &str) -> &str { From 2da7c5ad50ca9af8d308b2f4ff32e860040369d6 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 3 Nov 2026 01:47:05 -0500 Subject: [PATCH 19/41] Resolve subscribed table names and use C# names in the t_093 prompt - Subscriptions cannot retry a missing name, so t_024 and t_093 resolve the module's own spelling of the table first. - The C# t_093 prompt named SafeDocument's fields id and title, which the check reads as Id and Title. --- .../advanced/t_093_permission_revocation/spec.rs | 5 +---- .../t_093_permission_revocation/tasks/csharp.txt | 4 ++-- .../data_modeling/t_024_event_table/spec.rs | 2 +- tools/xtask-llm-benchmark/src/eval/scenario.rs | 6 ++++++ tools/xtask-llm-benchmark/src/eval/scorers.rs | 11 ++++++++--- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/spec.rs index b674afc88a3..49e2ffee752 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/spec.rs @@ -22,10 +22,7 @@ pub fn spec() -> BenchmarkSpec { ), )?; anyhow::ensure!((400..500).contains(&code), "private source exposed: HTTP {code}"); - let query = format!( - "SELECT * FROM \"{}\"", - crate::eval::table_name("visible_document", s.lang) - ); + let query = format!("SELECT * FROM \"{}\"", s.relation("visible_document")); let mut live = s.connect(&reader)?; live.subscribe(1, &query)?; live.expect_update("SubscribeApplied", json!([]), json!([]))?; diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt index 800244f4766..26f6f83d27c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt @@ -1,6 +1,6 @@ Write a SpacetimeDB backend module in C# that shares a private document with revocable permissions. Return only the module source. -Implement one shared private document (ID 1). Its first writer becomes its permanent owner. Only that owner can change its text or reader permissions. Authorized users can query and subscribe to a safe projection. Revocation must remove the document from an existing reader subscription and block future content updates. Do not use row-level security. Keep both source tables private and never expose secret_body in the public view. +Implement one shared private document (ID 1). Its first writer becomes its permanent owner. Only that owner can change its text or reader permissions. Authorized users can query and subscribe to a safe projection. Revocation must remove the document from an existing reader subscription and block future content updates. Do not use row-level security. Keep both source tables private and never expose SecretBody in the public view. Tables (the first field is the primary key): - private PrivateDocument: Id: ulong, Owner: Identity, Title: string, SecretBody: string @@ -10,5 +10,5 @@ Reducers (argument order is part of the contract): - set_document(Title: string, SecretBody: string): Create document ID 1 with sender as owner, or update it if sender is owner. Any other sender fails 'owner only'. Preserve ownership. - set_access(Reader: Identity, Enabled: bool): Require an existing document whose owner is sender ('owner only'). Insert or update the reader's permission. Disabling a reader does not remove the owner's own access. -Define SafeDocument with id:u64 and title:string only. Public sender-aware view visible_document returns an array of SafeDocument: the single document when sender is owner or read_access[sender].enabled is true, otherwise empty. Use indexed lookups so permission changes are tracked as view dependencies. Do not add public source tables or diagnostic reducers. +Define SafeDocument with Id: ulong and Title: string only. Public sender-aware view visible_document returns an array of SafeDocument: the single document when sender is owner or ReadAccess[sender].Enabled is true, otherwise empty. Use indexed lookups so permission changes are tracked as view dependencies. Do not add public source tables or diagnostic reducers. Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs index 992369c04ca..c3460e27e0d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs @@ -7,7 +7,7 @@ pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { let mut scorers = default_schema_parity_scorers(host, file!(), route); scorers.push(scenario(file!(), route, host, lang, |s| { - let query = format!("SELECT * FROM \"{}\"", crate::eval::table_name("damage_event", s.lang)); + let query = format!("SELECT * FROM \"{}\"", s.relation("damage_event")); let update = s.watch_update(&query, || s.call("deal_damage", json!([7, 12, "fire"])))?; anyhow::ensure!(update.contains("fire") && update.contains("12"), "missing event: {update}"); s.rows("damage_event", &["entity_id", "damage", "source"], json!([]))?; diff --git a/tools/xtask-llm-benchmark/src/eval/scenario.rs b/tools/xtask-llm-benchmark/src/eval/scenario.rs index 328ccc8c369..45955c35b4b 100644 --- a/tools/xtask-llm-benchmark/src/eval/scenario.rs +++ b/tools/xtask-llm-benchmark/src/eval/scenario.rs @@ -307,6 +307,12 @@ impl Scenario { } } + /// Subscriptions cannot retry a missing name, so resolve the module's spelling first. + pub fn relation(&self, name: &str) -> String { + let guess = table_name(name, self.lang); + super::scorers::actual_relation_name(&self.server, &self.db, &guess).unwrap_or(guess) + } + pub fn query(&self, table: &str, columns: &[&str]) -> String { let columns = columns .iter() diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 9ab9d7be61d..99e4f90acdb 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -742,10 +742,11 @@ fn rename_relation(query: &str, missing: &str, actual: &str) -> String { .into_owned() } -fn actual_relation_name(server: &str, db: &str, missing: &str) -> Option { +/// The module's spelling of `wanted`: an exact match first, else one differing only by case or underscores. +pub(crate) fn actual_relation_name(server: &str, db: &str, wanted: &str) -> Option { let schema = describe_db(server, db, Duration::from_secs(15)).ok()?; let names = SchemaNames::from_schema(&schema); - ["Tables", "Views"] + let relations: Vec = ["Tables", "Views"] .into_iter() .flat_map(|section| { schema_array(&schema, §ion.to_lowercase(), section) @@ -761,7 +762,11 @@ fn actual_relation_name(server: &str, db: &str, missing: &str) -> Option .cloned() .unwrap_or(source) }) - .find(|name| name != missing && loose_name(name) == loose_name(missing)) + .collect(); + let exact = relations.iter().find(|name| *name == wanted); + exact + .or_else(|| relations.iter().find(|name| loose_name(name) == loose_name(wanted))) + .cloned() } fn sql_raw_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { From 3e0062ab310011cb802e1188b1dc45c8823c05ac Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 3 Nov 2026 07:34:24 -0500 Subject: [PATCH 20/41] Use each language's declared names in the advanced prompts The C# and TypeScript prompts still named tables and columns in Rust's snake_case in their prose, and two C# prompts used Rust member names. --- .../t_084_stable_pagination/tasks/csharp.txt | 4 ++-- .../t_084_stable_pagination/tasks/typescript.txt | 2 +- .../advanced/t_085_online_backfill/tasks/csharp.txt | 12 ++++++------ .../t_086_incremental_totals/tasks/csharp.txt | 2 +- .../t_086_incremental_totals/tasks/typescript.txt | 2 +- .../advanced/t_087_webhook_ordering/tasks/csharp.txt | 2 +- .../t_087_webhook_ordering/tasks/typescript.txt | 2 +- .../advanced/t_088_lease_fencing/tasks/csharp.txt | 4 ++-- .../t_088_lease_fencing/tasks/typescript.txt | 4 ++-- .../t_089_cancel_reschedule/tasks/csharp.txt | 4 ++-- .../advanced/t_091_scheduled_queue/tasks/csharp.txt | 4 ++-- .../advanced/t_092_procedure_cache/tasks/csharp.txt | 2 +- .../t_093_permission_revocation/tasks/typescript.txt | 2 +- .../t_094_multi_session_presence/tasks/csharp.txt | 2 +- .../tasks/typescript.txt | 2 +- 15 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt index 6e13d97c1f2..fbfc75ceb03 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt @@ -1,6 +1,6 @@ Write a SpacetimeDB backend module in C# that serves stable paginated audit logs. Return only the module source. -Implement a tenant-scoped audit-log page. The ordering key is (occurred_at, id), ascending. The exclusive cursor is that pair; never use timestamp alone. Only rows for the requested tenant with visible=true can enter a page. Apply filtering BEFORE the limit. The page table is a test adapter: replace it on each read_page call; it is not a per-client cache. No native pagination API or scan-cost guarantee is assumed. +Implement a tenant-scoped audit-log page. The ordering key is (OccurredAt, id), ascending. The exclusive cursor is that pair; never use timestamp alone. Only rows for the requested tenant with visible=true can enter a page. Apply filtering BEFORE the limit. The page table is a test adapter: replace it on each read_page call; it is not a per-client cache. No native pagination API or scan-cost guarantee is assumed. Tables (the first field is the primary key): - public AuditEntry: Id: ulong, Tenant: string, OccurredAt: ulong, Visible: bool @@ -8,6 +8,6 @@ Tables (the first field is the primary key): Reducers (argument order is part of the contract): - add_entry(Id: ulong, Tenant: string, OccurredAt: ulong, Visible: bool): Insert the supplied audit entry. -- read_page(Tenant: string, AfterTime: ulong, AfterId: ulong, Limit: ulong): Reject limits outside 1..=10 with 'invalid limit', preserving the previous page. Otherwise replace page_entry with matching entries strictly after the cursor, sorted by the pair, at most limit rows. Positions start at 0 and are consecutive. Empty pages clear old results. Audit rows are never changed. +- read_page(Tenant: string, AfterTime: ulong, AfterId: ulong, Limit: ulong): Reject limits outside 1..=10 with 'invalid limit', preserving the previous page. Otherwise replace PageEntry with matching entries strictly after the cursor, sorted by the pair, at most limit rows. Positions start at 0 and are consecutive. Empty pages clear old results. Audit rows are never changed. Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt index f5f1aaeee58..7936c85a041 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt @@ -1,6 +1,6 @@ Write a SpacetimeDB backend module in TypeScript that serves stable paginated audit logs. Return only the module source. -Implement a tenant-scoped audit-log page. The ordering key is (occurred_at, id), ascending. The exclusive cursor is that pair; never use timestamp alone. Only rows for the requested tenant with visible=true can enter a page. Apply filtering BEFORE the limit. The page table is a test adapter: replace it on each read_page call; it is not a per-client cache. No native pagination API or scan-cost guarantee is assumed. +Implement a tenant-scoped audit-log page. The ordering key is (occurredAt, id), ascending. The exclusive cursor is that pair; never use timestamp alone. Only rows for the requested tenant with visible=true can enter a page. Apply filtering BEFORE the limit. The page table is a test adapter: replace it on each read_page call; it is not a per-client cache. No native pagination API or scan-cost guarantee is assumed. Tables (the first field is the primary key): - public audit_entry: id: u64 (bigint), tenant: string, occurredAt: u64 (bigint), visible: bool diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt index 892bf22e214..fad6c482a06 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt @@ -1,6 +1,6 @@ Write a SpacetimeDB backend module in C# that runs a resumable online backfill. Return only the module source. -Implement a resumable online backfill while writes and deletes continue. The legacy table is authoritative. Capture and apply are separate transactions, so a captured snapshot can become stale. Keep tombstones in legacy_item; never resurrect a deleted item. This task tests the data migration algorithm inside one module, not deployment schema conversion. +Implement a resumable online backfill while writes and deletes continue. The legacy table is authoritative. Capture and apply are separate transactions, so a captured snapshot can become stale. Keep tombstones in LegacyItem; never resurrect a deleted item. This task tests the data migration algorithm inside one module, not deployment schema conversion. Tables (the first field is the primary key): - public LegacyItem: Id: ulong, Value: string, Revision: ulong, Deleted: bool @@ -8,10 +8,10 @@ Tables (the first field is the primary key): - public ItemV2: Id: ulong, Value: string, Revision: ulong Reducers (argument order is part of the contract): -- write_item(Id: ulong, Value: string): Insert or update the legacy row, incrementing its revision (start at 1) and clearing deleted. Dual-write the same value and revision to item_v2. IDs may be reused after deletion. -- seed_legacy(Id: ulong, Value: string): Insert a legacy-only row at revision 1 with deleted=false. Used only to prepare pre-migration data. Do not write item_v2. -- delete_item(Id: ulong): If legacy exists and is not deleted, increment revision, mark deleted and remove item_v2. Repeated deletion or missing ID is a no-op. -- capture_batch(AfterId: ulong, Limit: ulong): Reject limit outside 1..=10 with 'invalid limit'. Replace backfill_snapshot with at most limit legacy rows with id>after_id, ordered by id. Capture tombstones too. Do not modify source or destination. -- apply_batch(): For each captured ID, reconcile item_v2 with the CURRENT authoritative legacy row, not the stale captured contents. Delete destination tombstones; upsert current live rows. Leave snapshots intact so retries are safe. Do not touch IDs outside the captured batch. +- write_item(Id: ulong, Value: string): Insert or update the legacy row, incrementing its revision (start at 1) and clearing deleted. Dual-write the same value and revision to ItemV2. IDs may be reused after deletion. +- seed_legacy(Id: ulong, Value: string): Insert a legacy-only row at revision 1 with deleted=false. Used only to prepare pre-migration data. Do not write ItemV2. +- delete_item(Id: ulong): If legacy exists and is not deleted, increment revision, mark deleted and remove ItemV2. Repeated deletion or missing ID is a no-op. +- capture_batch(AfterId: ulong, Limit: ulong): Reject limit outside 1..=10 with 'invalid limit'. Replace BackfillSnapshot with at most limit legacy rows with id>after_id, ordered by id. Capture tombstones too. Do not modify source or destination. +- apply_batch(): For each captured ID, reconcile ItemV2 with the CURRENT authoritative legacy row, not the stale captured contents. Delete destination tombstones; upsert current live rows. Leave snapshots intact so retries are safe. Do not touch IDs outside the captured batch. Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt index 340f5c187e7..0b56a37b989 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt @@ -1,6 +1,6 @@ Write a SpacetimeDB backend module in C# that maintains category totals incrementally. Return only the module source. -Maintain category totals after arbitrary sale inserts, updates, moves and deletes. A category row exists exactly when its sale_count is nonzero, even if total_amount is zero. Negative and zero sale amounts are valid. All source and aggregate changes must be atomic. This is a correctness test, not a scan-cost benchmark. +Maintain category totals after arbitrary sale inserts, updates, moves and deletes. A category row exists exactly when its SaleCount is nonzero, even if TotalAmount is zero. Negative and zero sale amounts are valid. All source and aggregate changes must be atomic. This is a correctness test, not a scan-cost benchmark. Tables (the first field is the primary key): - public Sale: Id: ulong, Category: string, Amount: long diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt index 79f999bdcd9..8e8a6c0fe13 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt @@ -1,6 +1,6 @@ Write a SpacetimeDB backend module in TypeScript that maintains category totals incrementally. Return only the module source. -Maintain category totals after arbitrary sale inserts, updates, moves and deletes. A category row exists exactly when its sale_count is nonzero, even if total_amount is zero. Negative and zero sale amounts are valid. All source and aggregate changes must be atomic. This is a correctness test, not a scan-cost benchmark. +Maintain category totals after arbitrary sale inserts, updates, moves and deletes. A category row exists exactly when its saleCount is nonzero, even if totalAmount is zero. Negative and zero sale amounts are valid. All source and aggregate changes must be atomic. This is a correctness test, not a scan-cost benchmark. Tables (the first field is the primary key): - public sale: id: u64 (bigint), category: string, amount: i64 (bigint) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/csharp.txt index b73b0172416..a6849423869 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/csharp.txt @@ -1,6 +1,6 @@ Write a SpacetimeDB backend module in C# that handles out-of-order webhook events. Return only the module source. -Implement POST /webhook with plain-text body account|event_id|sequence|value (exactly four fields). Account, event_id and value must be nonempty. Sequence must contain only ASCII decimal digits, fit u64, and be positive. Malformed input returns HTTP 400 body 'invalid' without any writes. Valid input returns HTTP 200. In one transaction, check the event ID globally: exact replay returns 'duplicate'; reuse with a different account, sequence or value returns 'conflict', without changes. For a new event, insert a receipt even if its sequence is stale. If sequence is greater than that account's last sequence, update state and return 'applied'; otherwise return 'stale'. Accounts advance independently. Response bodies must contain exactly those words. +Implement POST /webhook with plain-text body account|EventId|sequence|value (exactly four fields). Account, EventId and value must be nonempty. Sequence must contain only ASCII decimal digits, fit u64, and be positive. Malformed input returns HTTP 400 body 'invalid' without any writes. Valid input returns HTTP 200. In one transaction, check the event ID globally: exact replay returns 'duplicate'; reuse with a different account, sequence or value returns 'conflict', without changes. For a new event, insert a receipt even if its sequence is stale. If sequence is greater than that account's last sequence, update state and return 'applied'; otherwise return 'stale'. Accounts advance independently. Response bodies must contain exactly those words. Tables (the first field is the primary key): - public WebhookReceipt: EventId: string, Account: string, Sequence: ulong, Value: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/typescript.txt index 73b77ea18f1..14dc65ef591 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_087_webhook_ordering/tasks/typescript.txt @@ -1,6 +1,6 @@ Write a SpacetimeDB backend module in TypeScript that handles out-of-order webhook events. Return only the module source. -Implement POST /webhook with plain-text body account|event_id|sequence|value (exactly four fields). Account, event_id and value must be nonempty. Sequence must contain only ASCII decimal digits, fit u64, and be positive. Malformed input returns HTTP 400 body 'invalid' without any writes. Valid input returns HTTP 200. In one transaction, check the event ID globally: exact replay returns 'duplicate'; reuse with a different account, sequence or value returns 'conflict', without changes. For a new event, insert a receipt even if its sequence is stale. If sequence is greater than that account's last sequence, update state and return 'applied'; otherwise return 'stale'. Accounts advance independently. Response bodies must contain exactly those words. +Implement POST /webhook with plain-text body account|eventId|sequence|value (exactly four fields). Account, eventId and value must be nonempty. Sequence must contain only ASCII decimal digits, fit u64, and be positive. Malformed input returns HTTP 400 body 'invalid' without any writes. Valid input returns HTTP 200. In one transaction, check the event ID globally: exact replay returns 'duplicate'; reuse with a different account, sequence or value returns 'conflict', without changes. For a new event, insert a receipt even if its sequence is stale. If sequence is greater than that account's last sequence, update state and return 'applied'; otherwise return 'stale'. Accounts advance independently. Response bodies must contain exactly those words. Tables (the first field is the primary key): - public webhook_receipt: eventId: string, account: string, sequence: u64 (bigint), value: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt index ca6bc9f2bc4..323bcdf52e4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt @@ -7,7 +7,7 @@ Tables (the first field is the primary key): Reducers (argument order is part of the contract): - create_job(Id: ulong): Insert an unclaimed job: worker/result empty, generation/expiry=0, done=false. -- claim(Id: ulong, Worker: string, LeaseMs: ulong): Require nonempty worker and lease_ms in 1..=60000 ('invalid lease'), existing job ('missing job'), not done ('already done'), and unclaimed or expired ('lease busy'). Increment generation, set worker and expires_at=server_now+lease_ms*1000. -- complete(Id: ulong, Worker: string, Generation: ulong, Result: string): Require existing job ('missing job'). Exact replay of an already successful completion is a no-op, even after expiry. Any other completion of a done job fails ('already done'). For pending jobs, require nonempty matching worker, positive matching generation and server_now Date: Tue, 3 Nov 2026 08:58:22 -0500 Subject: [PATCH 21/41] Fix C# examples that do not compile in the server skill and lifecycle docs - Lifecycle reducers named OnConnect, OnDisconnect and OnInit fail with STDB0010, because reducer names may not start with On, on or __. - The HTTP example passed an undeclared uri that models read as System.Uri, though the API takes a string. --- .../00200-functions/00300-reducers/00500-lifecycle.md | 4 ++-- skills/csharp-server/SKILL.md | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md index 9758bf53663..c419d31b8e3 100644 --- a/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md +++ b/docs/docs/00200-core-concepts/00200-functions/00300-reducers/00500-lifecycle.md @@ -203,7 +203,7 @@ export const onConnect = spacetimedb.clientConnected((ctx) => { ```csharp [SpacetimeDB.Reducer(ReducerKind.ClientConnected)] -public static void OnConnect(ReducerContext ctx) +public static void ClientConnected(ReducerContext ctx) { Log.Info($"Client connected: {ctx.Sender}"); @@ -318,7 +318,7 @@ export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { ```csharp [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)] -public static void OnDisconnect(ReducerContext ctx) +public static void ClientDisconnected(ReducerContext ctx) { Log.Info($"Client disconnected: {ctx.Sender}"); diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index 1d08ba89fe7..b2befa6dd81 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -153,15 +153,17 @@ The pattern is `ctx.Db.{Accessor}.{ColumnName}.{Method}(value)` for all indexed ```csharp [SpacetimeDB.Reducer(ReducerKind.Init)] -public static void OnInit(ReducerContext ctx) { ... } +public static void Init(ReducerContext ctx) { ... } [SpacetimeDB.Reducer(ReducerKind.ClientConnected)] -public static void OnConnect(ReducerContext ctx) { ... } +public static void ClientConnected(ReducerContext ctx) { ... } [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)] -public static void OnDisconnect(ReducerContext ctx) { ... } +public static void ClientDisconnected(ReducerContext ctx) { ... } ``` +Reducer and procedure method names must not start with `On`, `on`, or `__`; the code generator rejects them (STDB0010), so name lifecycle reducers `ClientConnected`, not `OnConnect`. + `ctx.ConnectionId` is `ConnectionId?`, including in connection lifecycle reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. ## Views @@ -285,6 +287,7 @@ public static ResultValue Inspect(ProcedureContext ctx, string input) => Outbound HTTP is available through `ctx.Http`; handle its success/error result before using the response. Open short database transactions with `ctx.WithTx`. Perform network I/O before opening the transaction, and keep only database work inside its callback. ```csharp +string uri = "https://example.com/data"; // the C# HTTP API takes the URI as a string var result = ctx.Http.Get(uri); var text = result.Match( response => response.Body.ToStringUtf8Lossy(), From 7ead14b0830d2fe59618fb1560bcd23981cf5287 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 3 Nov 2026 10:44:20 -0500 Subject: [PATCH 22/41] Make the delete checks fail a module that does nothing t_006 and t_058 only counted rows after deleting, so a module whose reducers did nothing passed. Each now confirms the rows were inserted and that rows outside the deletion survive. --- .../benchmarks/basics/t_006_delete/spec.rs | 16 +++++++++++++ .../reducers/t_058_batched_delete/spec.rs | 23 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs index c2e9ead6f8d..2c391201a02 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs @@ -40,6 +40,22 @@ pub fn spec() -> BenchmarkSpec { }, )); + // The count check alone passes when insert does nothing, so prove the row existed and others survive. + v.push(crate::eval::scenario::scenario(file!(), route_tag, host_url, lang, |s| { + s.call("insert_user", serde_json::json!(["Ada", 36, true]))?; + s.call("insert_user", serde_json::json!(["Bob", 25, false]))?; + let columns = ["name", "age", "active"]; + s.rows("user", &columns, serde_json::json!([["Ada", 36, true], ["Bob", 25, false]]))?; + let ids = s.owner_sql(&s.query("user", &["id", "name"]))?; + let ada = ids + .as_array() + .and_then(|rows| rows.iter().find(|row| row[1] == "Ada")) + .map(|row| row[0].clone()) + .ok_or_else(|| anyhow::anyhow!("no id for Ada in {ids}"))?; + s.call("delete_user", serde_json::json!([ada]))?; + s.rows("user", &columns, serde_json::json!([["Bob", 25, false]])) + })); + v }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs index 1bde4f8755c..005f65fe766 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs @@ -2,7 +2,7 @@ use crate::eval::defaults::{ default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer, }; use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; -use serde_json::json; +use serde_json::{json, Value}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { @@ -35,6 +35,27 @@ pub fn spec() -> BenchmarkSpec { "scheduled_batches_finish", Duration::from_secs(10), )); + // The count check alone passes when seeding does nothing, so prove rows existed and other groups survive. + scorers.push(crate::eval::scenario::scenario(file!(), route_tag, host_url, lang, |s| { + let count = |group: u64| -> anyhow::Result { + let query = format!( + "SELECT COUNT(*) AS n FROM \"{}\" WHERE \"{}\" = {group}", + s.relation("work_item"), + ident("group_id", casing_for_lang(s.lang)) + ); + Ok(s.owner_sql(&query)?[0][0].clone()) + }; + s.call("seed_group", json!([8, 3]))?; + s.call("seed_group", json!([9, 5]))?; + anyhow::ensure!(count(8)? == json!(3) && count(9)? == json!(5), "seed_group did not insert the rows"); + s.call("request_delete", json!([9]))?; + s.eventually(|| { + anyhow::ensure!(count(9)? == json!(0), "group 9 still has rows"); + Ok(()) + })?; + anyhow::ensure!(count(8)? == json!(3), "deleting group 9 touched group 8"); + Ok(()) + })); scorers }) } From 4d670cb26cf0d2ad6da1aea4ef46ccbe77d7b70e Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 3 Nov 2026 20:42:42 -0500 Subject: [PATCH 23/41] State requirements in the prompts instead of naming APIs, and fix what CI exposed Prompts: - Describe the caller's identity, the current time and lifecycle events instead of naming the API that returns them, so the prompts no longer give away syntax the benchmark measures. Some of those names were wrong: Rust's sender as a field, TypeScript's as a call, and a C# time method that does not exist. - Call 64-bit TypeScript integers bigint rather than number. - State what t_055 counts as invalid, that t_023's C# table is public, t_011's TypeScript argument types, and that callers must receive the quoted text when an advanced operation fails. Checks: - Compare view primary keys only on t_067, the task that asks for one. - Accept extra HTTP routes and any 4xx rejection on t_077, since the prompt says reject and not how. - Ignore a table's exact spelling when comparing schedules. Skills: - Say how a callback confirms the scheduler called it, that interval rows stay after running, and where C# sum types must be declared. CI: - Raise C# concurrency to 4 for a dry run that compares it with 2. --- .github/workflows/llm-benchmark-periodic.yml | 2 +- skills/csharp-server/SKILL.md | 6 +- skills/rust-server/SKILL.md | 4 +- skills/typescript-server/SKILL.md | 4 +- tools/xtask-llm-benchmark/README.md | 7 +- .../t_083_atomic_reservation/tasks/csharp.txt | 2 +- .../t_083_atomic_reservation/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_084_stable_pagination/tasks/csharp.txt | 2 +- .../t_084_stable_pagination/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_085_online_backfill/tasks/csharp.txt | 2 +- .../t_085_online_backfill/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_086_incremental_totals/tasks/csharp.txt | 2 +- .../t_086_incremental_totals/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_088_lease_fencing/tasks/csharp.txt | 2 +- .../t_088_lease_fencing/tasks/rust.txt | 2 +- .../t_088_lease_fencing/tasks/typescript.txt | 2 +- .../t_089_cancel_reschedule/tasks/csharp.txt | 2 +- .../t_089_cancel_reschedule/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 2 +- .../t_090_transactional_quota/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_091_scheduled_queue/tasks/csharp.txt | 2 +- .../t_091_scheduled_queue/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 2 +- .../tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 2 +- .../tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 6 +- .../t_026_auth_identity_check/tasks/rust.txt | 6 +- .../tasks/typescript.txt | 10 +- .../tasks/typescript.txt | 4 +- .../tasks/csharp.txt | 8 +- .../t_041_registered_user_gate/tasks/rust.txt | 8 +- .../tasks/typescript.txt | 10 +- .../t_042_admin_bootstrap/tasks/csharp.txt | 4 +- .../auth/t_042_admin_bootstrap/tasks/rust.txt | 4 +- .../tasks/typescript.txt | 6 +- .../t_043_role_based_access/tasks/csharp.txt | 10 +- .../t_043_role_based_access/tasks/rust.txt | 10 +- .../tasks/typescript.txt | 10 +- .../auth/t_044_ban_list/tasks/csharp.txt | 8 +- .../auth/t_044_ban_list/tasks/rust.txt | 8 +- .../auth/t_044_ban_list/tasks/typescript.txt | 10 +- .../auth/t_045_rate_limit/tasks/csharp.txt | 6 +- .../auth/t_045_rate_limit/tasks/rust.txt | 6 +- .../t_045_rate_limit/tasks/typescript.txt | 10 +- .../t_046_shared_document/tasks/csharp.txt | 6 +- .../auth/t_046_shared_document/tasks/rust.txt | 6 +- .../tasks/typescript.txt | 14 +-- .../basics/t_005_update/tasks/typescript.txt | 4 +- .../basics/t_006_delete/tasks/typescript.txt | 4 +- .../basics/t_007_crud/tasks/typescript.txt | 2 +- .../t_008_index_lookup/tasks/typescript.txt | 6 +- .../basics/t_009_init/tasks/typescript.txt | 2 +- .../basics/t_010_connect/tasks/typescript.txt | 6 +- .../tasks/typescript.txt | 2 +- .../t_038_schedule_at_time/tasks/csharp.txt | 2 +- .../t_038_schedule_at_time/tasks/rust.txt | 1 - .../tasks/typescript.txt | 1 - .../t_039_cancel_schedule/tasks/csharp.txt | 2 +- .../t_040_lifecycle_player/tasks/csharp.txt | 4 +- .../t_040_lifecycle_player/tasks/rust.txt | 4 +- .../tasks/typescript.txt | 4 +- .../t_024_event_table/tasks/typescript.txt | 4 +- .../tasks/typescript.txt | 2 +- .../t_028_cascade_delete/tasks/typescript.txt | 8 +- .../tasks/typescript.txt | 6 +- .../t_030_two_table_join/tasks/typescript.txt | 8 +- .../tasks/typescript.txt | 2 +- .../procedures/t_077_http_router/spec.rs | 12 +- .../t_023_view_per_user/tasks/csharp.txt | 5 +- .../t_023_view_per_user/tasks/rust.txt | 4 +- .../t_023_view_per_user/tasks/typescript.txt | 8 +- .../t_032_range_query/tasks/typescript.txt | 4 +- .../t_033_sort_and_limit/tasks/typescript.txt | 6 +- .../t_034_find_first/tasks/typescript.txt | 4 +- .../tasks/typescript.txt | 2 +- .../tasks/typescript.txt | 4 +- .../tasks/typescript.txt | 4 +- .../tasks/csharp.txt | 2 +- .../tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/csharp.txt | 4 +- .../tasks/typescript.txt | 4 +- .../tasks/typescript.txt | 2 +- .../tasks/typescript.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_018_constraints/tasks/typescript.txt | 2 +- .../t_019_many_to_many/tasks/typescript.txt | 10 +- .../schema/t_020_ecs/tasks/typescript.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_049_binary_storage/tasks/typescript.txt | 2 +- .../views/t_067_view_primary_key/spec.rs | 4 +- .../xtask-llm-benchmark/src/eval/defaults.rs | 14 +++ .../xtask-llm-benchmark/src/eval/scenario.rs | 6 +- tools/xtask-llm-benchmark/src/eval/scorers.rs | 116 +++++++++++++++--- 104 files changed, 334 insertions(+), 224 deletions(-) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 5a14dcf1467..ea2e68f2271 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -184,7 +184,7 @@ jobs: # Bound in-flight provider credit reservations across the three language jobs. LLM_BENCH_CONCURRENCY: "8" LLM_BENCH_RUST_CONCURRENCY: "4" - LLM_BENCH_CSHARP_CONCURRENCY: "2" + LLM_BENCH_CSHARP_CONCURRENCY: "4" LLM_BENCH_ROUTE_CONCURRENCY: "1" LLM_BENCHMARK_REPORT_DIR: ${{ runner.temp }}/llm-benchmark-reports LLM_BENCHMARK_DIAGNOSTICS_DIR: ${{ runner.temp }}/llm-benchmark-diagnostics diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index b2befa6dd81..d92444afa0a 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -256,7 +256,7 @@ public partial struct TickTimer [SpacetimeDB.Reducer] public static void Tick(ReducerContext ctx, TickTimer timer) { - // timer row is auto-deleted after this reducer runs + // a one-time row is deleted after this runs; an interval row stays } // One-time: fires once at a specific time @@ -267,7 +267,7 @@ var at = new ScheduleAt.Interval(TimeSpan.FromSeconds(5)); ctx.Db.TickTimer.Insert(new TickTimer { ScheduledId = 0, ScheduledAt = at }); ``` -Scheduled reducer callbacks use the ordinary `[SpacetimeDB.Reducer]` attribute. There is no `ReducerKind.Scheduled`; the table's `Scheduled` option associates the callback. +Scheduled reducer callbacks use the ordinary `[SpacetimeDB.Reducer]` attribute. There is no `ReducerKind.Scheduled`; the table's `Scheduled` option associates the callback. To let only the scheduler run a callback, reject calls where `ctx.Sender != ctx.DatabaseIdentity`. To construct a `ConnectionId` from a 128-bit numeric representation, encode it as exactly 16 little-endian bytes and call `ConnectionId.From(bytes)`. The result is nullable and must be checked before use. @@ -359,3 +359,5 @@ public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle R var a = new Shape.Circle(new Circle { Radius = 10 }); var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 }); ``` + +Declare `TaggedEnum` records inside your `Module` class. At file scope, a variant named after its own type, such as `Circle Circle`, fails with CS8910. diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index 0e282242263..af0649ef5b1 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -244,7 +244,7 @@ pub struct TickTimer { #[spacetimedb::reducer] pub fn tick(ctx: &ReducerContext, timer: TickTimer) { - // timer row is auto-deleted after this reducer runs + // a one-time row is deleted after this runs; an interval row stays } // One-time: fires once at a specific time @@ -255,6 +255,8 @@ let at = ScheduleAt::Interval(std::time::Duration::from_secs(5).into()); ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, scheduled_at: at }); ``` +To let only the scheduler run a callback, reject calls where `ctx.sender() != ctx.database_identity()`. + Construct a connection ID from a numeric representation with `ConnectionId::from_u128(value)`. ## Procedures and HTTP diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 8563a5e9808..eed1867dfc3 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -246,7 +246,7 @@ const tickTimer = table({ export const tick = spacetimedb.reducer( { onSchedule: tickTimer }, { timer: tickTimer.rowType }, - (ctx, { timer }) => { /* timer row auto-deleted after this runs */ } + (ctx, { timer }) => { /* a one-time row is deleted after this runs; an interval row stays */ } ); // `onSchedule` also works for scheduled procedures whose return type is `t.unit()`. @@ -259,6 +259,8 @@ export const tick = spacetimedb.reducer( // const micros = at.tag === 'time' ? at.value : at.value.microsSinceUnixEpoch; // bigint ``` +To let only the scheduler run a callback, reject calls where `!ctx.sender.isEqual(ctx.databaseIdentity)`. + ## Custom Types ```typescript diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index 5cda492c3cf..b1fdd4eb960 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -20,9 +20,10 @@ Compiler errors in SDK-generated files score as normal failures. Saved batch records keep the evidence so suspected SDK defects can go to the core team. Schema parity treats table and view names that differ only by case or underscores as the same, because prompts do not fix their casing. It matches lifecycle reducers -by kind, not by name, because clients cannot call them. It accepts extra indexes -and extra view primary keys, but still fails a missing one. It fails any C# table -that sets `Name` without `Accessor`, because that is the 1.x form. Behaviour checks +by kind, not by name, because clients cannot call them. It accepts extra indexes and +HTTP routes but fails a missing one. It ignores view primary keys except on the task that asks for +one. It fails any C# table that sets `Name` without `Accessor`, because that is the +1.x form. Behaviour checks query a table or view by the module's own spelling when it differs only by case or underscores. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt index 22a6d331fe8..91cfb596583 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_stock(Id: ulong, Available: long): Insert a new product. Reject a negative quantity with 'invalid stock'. - reserve(RequestId: string, FirstId: ulong, FirstQty: long, SecondId: ulong, SecondQty: long): Exact replay returns success without changes. Changed replay: 'request conflict'. New request requires a nonempty ID, distinct products and positive quantities ('invalid reservation'); both products must exist ('missing product') and have enough stock ('insufficient stock'). Deduct both quantities and record the payload in one transaction. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt index f5350514b3c..f41eb6db735 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_stock(id: u64, available: i64): Insert a new product. Reject a negative quantity with 'invalid stock'. - reserve(request_id: String, first_id: u64, first_qty: i64, second_id: u64, second_qty: i64): Exact replay returns success without changes. Changed replay: 'request conflict'. New request requires a nonempty ID, distinct products and positive quantities ('invalid reservation'); both products must exist ('missing product') and have enough stock ('insufficient stock'). Deduct both quantities and record the payload in one transaction. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt index 9abea3d5ac9..97dcc448456 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_stock(id: u64 (bigint), available: i64 (bigint)): Insert a new product. Reject a negative quantity with 'invalid stock'. - reserve(requestId: string, firstId: u64 (bigint), firstQty: i64 (bigint), secondId: u64 (bigint), secondQty: i64 (bigint)): Exact replay returns success without changes. Changed replay: 'request conflict'. New request requires a nonempty ID, distinct products and positive quantities ('invalid reservation'); both products must exist ('missing product') and have enough stock ('insufficient stock'). Deduct both quantities and record the payload in one transaction. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt index fbfc75ceb03..3513a9e48fb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_entry(Id: ulong, Tenant: string, OccurredAt: ulong, Visible: bool): Insert the supplied audit entry. - read_page(Tenant: string, AfterTime: ulong, AfterId: ulong, Limit: ulong): Reject limits outside 1..=10 with 'invalid limit', preserving the previous page. Otherwise replace PageEntry with matching entries strictly after the cursor, sorted by the pair, at most limit rows. Positions start at 0 and are consecutive. Empty pages clear old results. Audit rows are never changed. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt index e6f78579bf8..daf77a2b877 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_entry(id: u64, tenant: String, occurred_at: u64, visible: bool): Insert the supplied audit entry. - read_page(tenant: String, after_time: u64, after_id: u64, limit: u64): Reject limits outside 1..=10 with 'invalid limit', preserving the previous page. Otherwise replace page_entry with matching entries strictly after the cursor, sorted by the pair, at most limit rows. Positions start at 0 and are consecutive. Empty pages clear old results. Audit rows are never changed. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt index 7936c85a041..89b69b174b1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_entry(id: u64 (bigint), tenant: string, occurredAt: u64 (bigint), visible: bool): Insert the supplied audit entry. - read_page(tenant: string, afterTime: u64 (bigint), afterId: u64 (bigint), limit: u64 (bigint)): Reject limits outside 1..=10 with 'invalid limit', preserving the previous page. Otherwise replace page_entry with matching entries strictly after the cursor, sorted by the pair, at most limit rows. Positions start at 0 and are consecutive. Empty pages clear old results. Audit rows are never changed. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt index fad6c482a06..4160b1f737a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt @@ -14,4 +14,4 @@ Reducers (argument order is part of the contract): - capture_batch(AfterId: ulong, Limit: ulong): Reject limit outside 1..=10 with 'invalid limit'. Replace BackfillSnapshot with at most limit legacy rows with id>after_id, ordered by id. Capture tombstones too. Do not modify source or destination. - apply_batch(): For each captured ID, reconcile ItemV2 with the CURRENT authoritative legacy row, not the stale captured contents. Delete destination tombstones; upsert current live rows. Leave snapshots intact so retries are safe. Do not touch IDs outside the captured batch. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt index 8e0cb0834a4..177c7a75b9c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt @@ -14,4 +14,4 @@ Reducers (argument order is part of the contract): - capture_batch(after_id: u64, limit: u64): Reject limit outside 1..=10 with 'invalid limit'. Replace backfill_snapshot with at most limit legacy rows with id>after_id, ordered by id. Capture tombstones too. Do not modify source or destination. - apply_batch(): For each captured ID, reconcile item_v2 with the CURRENT authoritative legacy row, not the stale captured contents. Delete destination tombstones; upsert current live rows. Leave snapshots intact so retries are safe. Do not touch IDs outside the captured batch. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt index 42bbe669102..2b86252bc69 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt @@ -14,4 +14,4 @@ Reducers (argument order is part of the contract): - capture_batch(afterId: u64 (bigint), limit: u64 (bigint)): Reject limit outside 1..=10 with 'invalid limit'. Replace backfill_snapshot with at most limit legacy rows with id>after_id, ordered by id. Capture tombstones too. Do not modify source or destination. - apply_batch(): For each captured ID, reconcile item_v2 with the CURRENT authoritative legacy row, not the stale captured contents. Delete destination tombstones; upsert current live rows. Leave snapshots intact so retries are safe. Do not touch IDs outside the captured batch. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt index 0b56a37b989..53bcf78e13b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - set_sale(Id: ulong, Category: string, Amount: long): Insert or replace a sale. Remove its old contribution and add its new contribution. Reject empty category with 'invalid category'. - remove_sale(Id: ulong): Delete a sale and its contribution. Missing IDs are a no-op. Remove a category only when its last sale is deleted. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt index fdb26f6f6c5..3fd70554a3a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - set_sale(id: u64, category: String, amount: i64): Insert or replace a sale. Remove its old contribution and add its new contribution. Reject empty category with 'invalid category'. - remove_sale(id: u64): Delete a sale and its contribution. Missing IDs are a no-op. Remove a category only when its last sale is deleted. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt index 8e8a6c0fe13..a094c936a44 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - set_sale(id: u64 (bigint), category: string, amount: i64 (bigint)): Insert or replace a sale. Remove its old contribution and add its new contribution. Reject empty category with 'invalid category'. - remove_sale(id: u64 (bigint)): Delete a sale and its contribution. Missing IDs are a no-op. Remove a category only when its last sale is deleted. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt index 323bcdf52e4..b015403cb17 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - claim(Id: ulong, Worker: string, LeaseMs: ulong): Require nonempty worker and lease_ms in 1..=60000 ('invalid lease'), existing job ('missing job'), not done ('already done'), and unclaimed or expired ('lease busy'). Increment generation, set worker and ExpiresAt=server_now+lease_ms*1000. - complete(Id: ulong, Worker: string, Generation: ulong, Result: string): Require existing job ('missing job'). Exact replay of an already successful completion is a no-op, even after expiry. Any other completion of a done job fails ('already done'). For pending jobs, require nonempty matching worker, positive matching generation and server_now0 ('invalid request'). Require used+units<=3 ('quota exceeded'), starting at used=0. Insert request and update quota atomically. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt index 5192a2ea0ad..ff89c27d8d9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt @@ -9,4 +9,4 @@ Tables (the first field is the primary key): Reducers (argument order is part of the contract): - submit(request_id: String, units: i64): If request ID exists, identical owner+units returns success; otherwise fail 'request conflict'. For new IDs, require nonempty request ID and units>0 ('invalid request'). Require used+units<=3 ('quota exceeded'), starting at used=0. Insert request and update quota atomically. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt index 5606c8ea595..7af9b80cf7a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt @@ -9,4 +9,4 @@ Tables (the first field is the primary key): Reducers (argument order is part of the contract): - submit(requestId: string, units: i64 (bigint)): If request ID exists, identical owner+units returns success; otherwise fail 'request conflict'. For new IDs, require nonempty request ID and units>0 ('invalid request'). Require used+units<=3 ('quota exceeded'), starting at used=0. Insert request and update quota atomically. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt index f71955b6c46..3c027df5190 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - enqueue(Id: ulong, Input: long): Exact existing id+input is a no-op even after completion or failure. Changed input fails 'request conflict'. New input must be within -1000000..=1000000 ('invalid input'). Insert QueuedWork with status='queued', attempts=0, and one timer due after 1ms. WorkTimer uses an auto-increment primary key and schedules execute_work(job: WorkTimer). For a queued job only, increment attempts once; negative input sets status='failed' without an effect. Otherwise insert work_effect(id,input*2) and set status='complete' in the same transaction. Missing or already terminal jobs are no-ops. Do not delete QueuedWork rows. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt index 0cd2131f220..f4d8b5956a2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - enqueue(id: u64, input: i64): Exact existing id+input is a no-op even after completion or failure. Changed input fails 'request conflict'. New input must be within -1000000..=1000000 ('invalid input'). Insert queued_work with status='queued', attempts=0, and one timer due after 1ms. work_timer uses an auto-increment primary key and schedules execute_work(job: WorkTimer). For a queued job only, increment attempts once; negative input sets status='failed' without an effect. Otherwise insert work_effect(id,input*2) and set status='complete' in the same transaction. Missing or already terminal jobs are no-ops. Do not delete queued_work rows. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt index 16e4ee96b31..9ce21289ec4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - enqueue(id: u64 (bigint), input: i64 (bigint)): Exact existing id+input is a no-op even after completion or failure. Changed input fails 'request conflict'. New input must be within -1000000..=1000000 ('invalid input'). Insert queued_work with status='queued', attempts=0, and one timer due after 1ms. work_timer uses an auto-increment primary key and schedules execute_work(job: WorkTimer). For a queued job only, increment attempts once; negative input sets status='failed' without an effect. Otherwise insert work_effect(id,input*2) and set status='complete' in the same transaction. Missing or already terminal jobs are no-ops. Do not delete queued_work rows. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt index 26f6f83d27c..ba5197c80d3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - set_access(Reader: Identity, Enabled: bool): Require an existing document whose owner is sender ('owner only'). Insert or update the reader's permission. Disabling a reader does not remove the owner's own access. Define SafeDocument with Id: ulong and Title: string only. Public sender-aware view visible_document returns an array of SafeDocument: the single document when sender is owner or ReadAccess[sender].Enabled is true, otherwise empty. Use indexed lookups so permission changes are tracked as view dependencies. Do not add public source tables or diagnostic reducers. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt index 23246a4ca5c..e56311bf5d0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - set_access(reader: spacetimedb::Identity, enabled: bool): Require an existing document whose owner is sender ('owner only'). Insert or update the reader's permission. Disabling a reader does not remove the owner's own access. Define SafeDocument with id:u64 and title:string only. Public sender-aware view visible_document returns an array of SafeDocument: the single document when sender is owner or read_access[sender].enabled is true, otherwise empty. Use indexed lookups so permission changes are tracked as view dependencies. Do not add public source tables or diagnostic reducers. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt index 632b5af5fa1..6b1b5730610 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - set_access(reader: Identity, enabled: bool): Require an existing document whose owner is sender ('owner only'). Insert or update the reader's permission. Disabling a reader does not remove the owner's own access. Define SafeDocument with id:u64 and title:string only. Public sender-aware view visible_document returns an array of SafeDocument: the single document when sender is owner or read_access[sender].enabled is true, otherwise empty. Use indexed lookups so permission changes are tracked as view dependencies. Do not add public source tables or diagnostic reducers. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/csharp.txt index 7f5061abb05..8983d1017b9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/csharp.txt @@ -6,5 +6,5 @@ Tables (the first field is the primary key): - public LiveSession: ConnectionId: ConnectionId, Owner: Identity - public OnlineUser: Owner: Identity, Connections: ulong -On client_connected, insert a LiveSession using ctx.ConnectionId and ctx.Sender, and increment that identity's OnlineUser count (insert at 1). On client_disconnected, remove exactly that connection if it exists, decrement its owner's count, and delete OnlineUser only when the count reaches zero. A missing session is a no-op. Both changes are transactional. +When a client connects, insert a LiveSession using its connection ID and identity, and increment that identity's OnlineUser count (insert at 1). When a client disconnects, remove exactly that connection if it exists, decrement its owner's count, and delete OnlineUser only when the count reaches zero. A missing session is a no-op. Both changes are transactional. Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/rust.txt index 23cae172dd9..5c9a0443462 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/rust.txt @@ -6,5 +6,5 @@ Tables (the first field is the primary key): - public live_session: connection_id: spacetimedb::ConnectionId, owner: spacetimedb::Identity - public online_user: owner: spacetimedb::Identity, connections: u64 -On client_connected, insert a live_session using ctx.connection_id and ctx.sender, and increment that identity's online_user count (insert at 1). On client_disconnected, remove exactly that connection if it exists, decrement its owner's count, and delete online_user only when the count reaches zero. A missing session is a no-op. Both changes are transactional. +When a client connects, insert a live_session using its connection ID and identity, and increment that identity's online_user count (insert at 1). When a client disconnects, remove exactly that connection if it exists, decrement its owner's count, and delete online_user only when the count reaches zero. A missing session is a no-op. Both changes are transactional. Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/typescript.txt index 9567f28d3d1..d4dc41a190d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_094_multi_session_presence/tasks/typescript.txt @@ -6,5 +6,5 @@ Tables (the first field is the primary key): - public live_session: connectionId: ConnectionId, owner: Identity - public online_user: owner: Identity, connections: u64 (bigint) -On client_connected, insert a live_session using ctx.connectionId and ctx.sender, and increment that identity's online_user count (insert at 1). On client_disconnected, remove exactly that connection if it exists, decrement its owner's count, and delete online_user only when the count reaches zero. A missing session is a no-op. Both changes are transactional. +When a client connects, insert a live_session using its connection ID and identity, and increment that identity's online_user count (insert at 1). When a client disconnects, remove exactly that connection if it exists, decrement its owner's count, and delete online_user only when the count reaches zero. A missing session is a no-op. Both changes are transactional. Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt index 24db1ea5662..4daf926f242 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt @@ -1,4 +1,4 @@ -Write a SpacetimeDB backend module in C# that defines a table of messages owned by users, and reducers that enforce ownership using ctx.Sender. +Write a SpacetimeDB backend module in C# that defines a table of messages owned by users, and reducers that enforce ownership using the caller's identity. TABLES - Message @@ -11,11 +11,11 @@ TABLES REDUCERS - CreateMessage(ctx, string text) - - Insert a new Message with Owner set to ctx.Sender + - Insert a new Message with Owner set to the caller's identity - Use Id: 0 for AutoInc - DeleteMessage(ctx, ulong id) - Find the message by Id - If not found, throw exception with "not found" - - If the message's Owner does not equal ctx.Sender, throw exception with "unauthorized" + - If the message's Owner does not equal the caller's identity, throw exception with "unauthorized" - Otherwise, delete the message diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt index 58689ccd927..41a4594d4c9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt @@ -1,4 +1,4 @@ -Write a SpacetimeDB backend module in Rust that defines a table of messages owned by users, and reducers that enforce ownership using ctx.sender. +Write a SpacetimeDB backend module in Rust that defines a table of messages owned by users, and reducers that enforce ownership using the caller's identity. TABLES - message @@ -11,11 +11,11 @@ TABLES REDUCERS - create_message(ctx, text: String) - - Insert a new Message with owner set to ctx.sender + - Insert a new Message with owner set to the caller's identity - Use id: 0 for auto_inc - delete_message(ctx, id: u64) - Find the message by id - If not found, panic with "not found" - - If the message's owner does not equal ctx.sender, panic with "unauthorized" + - If the message's owner does not equal the caller's identity, panic with "unauthorized" - Otherwise, delete the message diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt index 3eb1729d5af..625f8760c9a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt @@ -1,20 +1,20 @@ -Write a SpacetimeDB backend module in TypeScript that defines a table of messages owned by users, and reducers that enforce ownership using ctx.sender. +Write a SpacetimeDB backend module in TypeScript that defines a table of messages owned by users, and reducers that enforce ownership using the caller's identity. TABLES - message - public table - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - owner: Identity (index btree) - text: string REDUCERS - create_message(ctx, { text: string }) - - Insert a new message with owner set to ctx.sender() + - Insert a new message with owner set to the caller's identity - Use id: 0n for autoInc -- delete_message(ctx, { id: number (u64) }) +- delete_message(ctx, { id: bigint (u64) }) - Find the message by id - If not found, throw "not found" - - If the message's owner does not equal ctx.sender(), throw "unauthorized" + - If the message's owner does not equal the caller's identity, throw "unauthorized" - Otherwise, delete the message diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt index 46d2ab143e9..e2a2cdfd33e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt @@ -4,7 +4,7 @@ TABLES - user_internal - private table (do not set public: true) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - email: string - passwordHash: string @@ -12,7 +12,7 @@ TABLES - user_public - public table (set public: true) - Fields: - - id: number (u64, primary key) + - id: bigint (u64, primary key) - name: string REDUCERS diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt index 2f2d5a38577..c4cae4988db 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt @@ -18,9 +18,9 @@ TABLES REDUCERS - RegisterUser(ctx, string name) - - If a User row already exists for ctx.Sender, throw exception with "already registered" - - Otherwise insert a new User with Identity = ctx.Sender and the given Name + - If a User row already exists for the caller's identity, throw exception with "already registered" + - Otherwise insert a new User with Identity = the caller's identity and the given Name - PostMessage(ctx, string text) - - If no User row exists for ctx.Sender, throw exception with "not registered" - - Otherwise insert a Message with Sender = ctx.Sender and the given Text, Id = 0 for AutoInc + - If no User row exists for the caller's identity, throw exception with "not registered" + - Otherwise insert a Message with Sender = the caller's identity and the given Text, Id = 0 for AutoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt index 16e386b3009..a30093f935b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt @@ -18,9 +18,9 @@ TABLES REDUCERS - register(ctx, name: String) - - If a User row already exists for ctx.sender, panic with "already registered" - - Otherwise insert a new User with identity = ctx.sender and the given name + - If a User row already exists for the caller's identity, panic with "already registered" + - Otherwise insert a new User with identity = the caller's identity and the given name - post_message(ctx, text: String) - - If no User row exists for ctx.sender, panic with "not registered" - - Otherwise insert a Message with sender = ctx.sender and the given text, id = 0 for auto_inc + - If no User row exists for the caller's identity, panic with "not registered" + - Otherwise insert a Message with sender = the caller's identity and the given text, id = 0 for auto_inc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt index a4e2d6e8340..ed96e6112e0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt @@ -10,15 +10,15 @@ TABLES - message - public table - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - sender: Identity (index btree) - text: string REDUCERS - register(ctx, { name: string }) - - If a user row already exists for ctx.sender, throw "already registered" - - Otherwise insert a new user with identity = ctx.sender and the given name + - If a user row already exists for the caller's identity, throw "already registered" + - Otherwise insert a new user with identity = the caller's identity and the given name - post_message(ctx, { text: string }) - - If no user row exists for ctx.sender, throw "not registered" - - Otherwise insert a message with sender = ctx.sender and the given text, id = 0n for autoInc + - If no user row exists for the caller's identity, throw "not registered" + - Otherwise insert a message with sender = the caller's identity and the given text, id = 0n for autoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt index b8443e4e736..e09bceb4f06 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt @@ -18,9 +18,9 @@ TABLES REDUCERS - BootstrapAdmin(ctx) - If a Config row already exists (Id = 0), throw exception with "already bootstrapped" - - Otherwise insert Config { Id = 0, Admin = ctx.Sender } + - Otherwise insert Config { Id = 0, Admin = the caller's identity } - AdminAction(ctx, string action) - Load the Config row with Id = 0; if missing, throw exception with "not bootstrapped" - - If ctx.Sender does not equal config.Admin, throw exception with "not admin" + - If the caller's identity does not equal config.Admin, throw exception with "not admin" - Insert an AdminLog row with the given Action, Id = 0 for AutoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt index 61ab5cf6b29..5ee666aeb45 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt @@ -18,9 +18,9 @@ TABLES REDUCERS - bootstrap_admin(ctx) - If a Config row already exists (id = 0), panic with "already bootstrapped" - - Otherwise insert Config { id: 0, admin: ctx.sender } + - Otherwise insert Config { id: 0, admin: the caller's identity } - admin_action(ctx, action: String) - Load the Config row with id = 0; if missing, panic with "not bootstrapped" - - If ctx.sender does not equal config.admin, panic with "not admin" + - If the caller's identity does not equal config.admin, panic with "not admin" - Insert an AdminLog row with the given action, id = 0 for auto_inc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt index 4bfd7a4bb89..5ad23f5e119 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt @@ -10,15 +10,15 @@ TABLES - admin_log - public table - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - action: string REDUCERS - bootstrap_admin(ctx) - If a config row already exists (id = 0), throw "already bootstrapped" - - Otherwise insert config { id: 0, admin: ctx.sender } + - Otherwise insert config { id: 0, admin: the caller's identity } - admin_action(ctx, { action: string }) - Load the config row with id = 0; if missing, throw "not bootstrapped" - - If ctx.sender does not equal config.admin, throw "not admin" + - If the caller's identity does not equal config.admin, throw "not admin" - Insert an admin_log row with the given action, id = 0n for autoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt index 67df07c65b8..6f686fe2475 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt @@ -10,18 +10,18 @@ TABLES REDUCERS - RegisterUser(ctx) - - If a User row already exists for ctx.Sender, throw exception with "already registered" - - Insert User { Identity = ctx.Sender, Role = "member" } + - If a User row already exists for the caller's identity, throw exception with "already registered" + - Insert User { Identity = the caller's identity, Role = "member" } - Promote(ctx, Identity target) - - Load the User row for ctx.Sender; if missing, throw exception with "not registered" + - Load the User row for the caller's identity; if missing, throw exception with "not registered" - If the caller's Role is not "admin", throw exception with "not admin" - Load the User row for target; if missing, throw exception with "target not registered" - Update that row's Role to "admin" - MemberAction(ctx) - - Load the User row for ctx.Sender; if missing, throw exception with "not registered" + - Load the User row for the caller's identity; if missing, throw exception with "not registered" - AdminAction(ctx) - - Load the User row for ctx.Sender; if missing, throw exception with "not registered" + - Load the User row for the caller's identity; if missing, throw exception with "not registered" - If the Role is not "admin", throw exception with "not admin" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt index 73b4a876bfb..6776ba27e10 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt @@ -10,20 +10,20 @@ TABLES REDUCERS - register(ctx) - - If a User row already exists for ctx.sender, panic with "already registered" - - Insert User { identity: ctx.sender, role: "member" } + - If a User row already exists for the caller's identity, panic with "already registered" + - Insert User { identity: the caller's identity, role: "member" } - promote(ctx, target: Identity) - - Load the User row for ctx.sender; if missing, panic with "not registered" + - Load the User row for the caller's identity; if missing, panic with "not registered" - If the caller's role is not "admin", panic with "not admin" - Load the User row for target; if missing, panic with "target not registered" - Update that row's role to "admin" - member_action(ctx) - - Load the User row for ctx.sender; if missing, panic with "not registered" + - Load the User row for the caller's identity; if missing, panic with "not registered" - (No further logic needed; presence of the row is sufficient) - admin_action(ctx) - - Load the User row for ctx.sender; if missing, panic with "not registered" + - Load the User row for the caller's identity; if missing, panic with "not registered" - If the role is not "admin", panic with "not admin" - (No further logic needed beyond the guard) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt index 60cb6e06bbf..11f17965b9f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt @@ -9,18 +9,18 @@ TABLES REDUCERS - register(ctx) - - If a user row already exists for ctx.sender, throw "already registered" - - Insert user { identity: ctx.sender, role: "member" } + - If a user row already exists for the caller's identity, throw "already registered" + - Insert user { identity: the caller's identity, role: "member" } - promote(ctx, { target: Identity }) - - Load the user row for ctx.sender; if missing, throw "not registered" + - Load the user row for the caller's identity; if missing, throw "not registered" - If the caller's role is not "admin", throw "not admin" - Load the user row for target; if missing, throw "target not registered" - Update that row's role to "admin" - member_action(ctx) - - Load the user row for ctx.sender; if missing, throw "not registered" + - Load the user row for the caller's identity; if missing, throw "not registered" - admin_action(ctx) - - Load the user row for ctx.sender; if missing, throw "not registered" + - Load the user row for the caller's identity; if missing, throw "not registered" - If the role is not "admin", throw "not admin" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt index 5337d4c112e..0a1a9eab365 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt @@ -22,14 +22,14 @@ TABLES REDUCERS - AddAdmin(ctx, Identity target) - - If ctx.Sender is not in the Admin table, throw exception with "not admin" + - If the caller's identity is not in the Admin table, throw exception with "not admin" - Insert Admin { Identity = target } (ignore if already present) - BanPlayer(ctx, Identity target) - - If ctx.Sender is not in the Admin table, throw exception with "not admin" + - If the caller's identity is not in the Admin table, throw exception with "not admin" - Insert Banned { Identity = target } - Remove the Player row for target if one exists - JoinGame(ctx, string name) - - If ctx.Sender is in the Banned table, throw exception with "banned" - - Insert Player { Identity = ctx.Sender, Name = name }; if already present, throw exception with "already in game" + - If the caller's identity is in the Banned table, throw exception with "banned" + - Insert Player { Identity = the caller's identity, Name = name }; if already present, throw exception with "already in game" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt index 21717530997..6ffc16f7b19 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt @@ -22,14 +22,14 @@ TABLES REDUCERS - add_admin(ctx, target: Identity) - - If ctx.sender is not in the admin table, panic with "not admin" + - If the caller's identity is not in the admin table, panic with "not admin" - Insert Admin { identity: target } (ignore if already present) - ban_player(ctx, target: Identity) - - If ctx.sender is not in the admin table, panic with "not admin" + - If the caller's identity is not in the admin table, panic with "not admin" - Insert Banned { identity: target } - Remove the Player row for target if one exists - join_game(ctx, name: String) - - If ctx.sender is in the banned table, panic with "banned" - - Insert Player { identity: ctx.sender, name }; if already present, panic with "already in game" + - If the caller's identity is in the banned table, panic with "banned" + - Insert Player { identity: the caller's identity, name }; if already present, panic with "already in game" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt index 7400f7f0374..731a70fa2bf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt @@ -19,15 +19,15 @@ TABLES REDUCERS - add_admin(ctx, { target: Identity }) - - If ctx.sender is not in the admin table, throw "not admin" + - If the caller's identity is not in the admin table, throw "not admin" - Insert { identity: target } into admin (ignore if already present) - ban_player(ctx, { target: Identity }) - - If ctx.sender is not in the admin table, throw "not admin" + - If the caller's identity is not in the admin table, throw "not admin" - Insert { identity: target } into banned - Remove the player row for target if one exists - join_game(ctx, { name: string }) - - If ctx.sender is in the banned table, throw "banned" - - If a player row already exists for ctx.sender, throw "already in game" - - Insert player { identity: ctx.sender, name } + - If the caller's identity is in the banned table, throw "banned" + - If a player row already exists for the caller's identity, throw "already in game" + - Insert player { identity: the caller's identity, name } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt index 8fa3e429326..5ad4fa99e83 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt @@ -18,8 +18,8 @@ TABLES REDUCERS - LimitedAction(ctx, string payload) - - Get the current time in microseconds: ctx.Timestamp.ToUnixTimeMilliseconds() * 1000 - - Look up the RateLimit row for ctx.Sender + - Get the current time in microseconds + - Look up the RateLimit row for the caller's identity - If a row exists and (now - LastCallUs) < 1_000_000, throw exception with "rate limited" - Insert or update the RateLimit row with LastCallUs = now - - Insert an ActionLog row with Identity = ctx.Sender and the given Payload, Id = 0 for AutoInc + - Insert an ActionLog row with Identity = the caller's identity and the given Payload, Id = 0 for AutoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt index db178fe9cfc..27d223ef742 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt @@ -18,8 +18,8 @@ TABLES REDUCERS - limited_action(ctx, payload: String) - - Get the current time in microseconds as i64: ctx.timestamp.to_micros_since_unix_epoch() - - Look up the RateLimit row for ctx.sender + - Get the current time in microseconds + - Look up the RateLimit row for the caller's identity - If a row exists and (now - last_call_us) < 1_000_000, panic with "rate limited" - Insert or update the RateLimit row with last_call_us = now - - Insert an ActionLog row with identity = ctx.sender and the given payload, id = 0 for auto_inc + - Insert an ActionLog row with identity = the caller's identity and the given payload, id = 0 for auto_inc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt index 2af01a1dc16..8927ce1e370 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt @@ -5,19 +5,19 @@ TABLES - private table - Fields: - identity: Identity (primary key) - - last_call_us: number (u64, microsecond timestamp of the last allowed call) + - last_call_us: bigint (u64, microsecond timestamp of the last allowed call) - action_log - public table - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - identity: Identity - payload: string REDUCERS - limited_action(ctx, { payload: string }) - - Get the current time in microseconds: ctx.timestamp.microsSinceUnixEpoch (a BigInt) - - Look up the rate_limit row for ctx.sender + - Get the current time in microseconds + - Look up the rate_limit row for the caller's identity - If a row exists and (now - last_call_us) < 1_000_000n, throw "rate limited" - Insert or update the rate_limit row with last_call_us = now - - Insert an action_log row with identity = ctx.sender and the given payload, id = 0n for autoInc + - Insert an action_log row with identity = the caller's identity and the given payload, id = 0n for autoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt index cf07439c658..391b8302a57 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt @@ -18,16 +18,16 @@ TABLES REDUCERS - CreateDocument(ctx, string title) - - Insert a Document with Owner = ctx.Sender and the given Title, Id = 0 for AutoInc + - Insert a Document with Owner = the caller's identity and the given Title, Id = 0 for AutoInc - ShareDocument(ctx, ulong documentId, Identity target) - Load the Document by Id; if missing, throw exception with "not found" - - If Document.Owner != ctx.Sender, throw exception with "not owner" + - If Document.Owner != the caller's identity, throw exception with "not owner" - Insert a DocumentShare { DocumentId = documentId, SharedWith = target } - EditDocument(ctx, ulong documentId, string newTitle) - Load the Document by Id; if missing, throw exception with "not found" - The caller is authorized if they are the Owner OR if a DocumentShare row exists - with matching DocumentId and SharedWith == ctx.Sender + with matching DocumentId and SharedWith == the caller's identity - If neither condition is true, throw exception with "unauthorized" - Update the Document's Title to newTitle diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt index 791afb14297..a96af7a5b7d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt @@ -18,16 +18,16 @@ TABLES REDUCERS - create_document(ctx, title: String) - - Insert a Document with owner = ctx.sender and the given title, id = 0 for auto_inc + - Insert a Document with owner = the caller's identity and the given title, id = 0 for auto_inc - share_document(ctx, document_id: u64, target: Identity) - Load the Document by id; if missing, panic with "not found" - - If document.owner != ctx.sender, panic with "not owner" + - If document.owner != the caller's identity, panic with "not owner" - Insert a DocumentShare { document_id, shared_with: target } - edit_document(ctx, document_id: u64, new_title: String) - Load the Document by id; if missing, panic with "not found" - The caller is authorized if they are the owner OR if a DocumentShare row exists - with matching document_id and shared_with == ctx.sender + with matching document_id and shared_with == the caller's identity - If neither condition is true, panic with "unauthorized" - Update the Document's title to new_title diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt index 2d7c257b147..d14050ef853 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt @@ -4,28 +4,28 @@ TABLES - document - public table - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - owner: Identity (index btree) - title: string - document_share - public table - Fields: - - document_id: number (u64, index btree) + - document_id: bigint (u64, index btree) - shared_with: Identity (index btree) REDUCERS - create_document(ctx, { title: string }) - - Insert a document with owner = ctx.sender and the given title, id = 0n for autoInc + - Insert a document with owner = the caller's identity and the given title, id = 0n for autoInc -- share_document(ctx, { document_id: number (u64), target: Identity }) +- share_document(ctx, { document_id: bigint (u64), target: Identity }) - Load the document by id; if missing, throw "not found" - - If document.owner != ctx.sender, throw "not owner" + - If document.owner != the caller's identity, throw "not owner" - Insert a document_share { document_id, shared_with: target } -- edit_document(ctx, { document_id: number (u64), new_title: string }) +- edit_document(ctx, { document_id: bigint (u64), new_title: string }) - Load the document by id; if missing, throw "not found" - The caller is authorized if they are the owner OR if a document_share row exists - with matching document_id and shared_with == ctx.sender + with matching document_id and shared_with == the caller's identity - If neither condition is true, throw "unauthorized" - Update the document's title to new_title diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/typescript.txt index cabece62c5c..2efe0688258 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_005_update/tasks/typescript.txt @@ -3,12 +3,12 @@ Write a SpacetimeDB backend module in TypeScript that defines one table and two TABLE - user (public) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - age: number (i32) - active: boolean REDUCERS - insertUser: given name:string, age:number, active:boolean, insert a new row (id is auto-assigned) -- updateUser: given id:number, name:string, age:number, active:boolean, update the row with matching id +- updateUser: given id:bigint, name:string, age:number, active:boolean, update the row with matching id - (id=id, name=name, age=age, active=active) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/typescript.txt index 3caa5c891ff..e792d48e90a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/tasks/typescript.txt @@ -3,11 +3,11 @@ Write a SpacetimeDB backend module in TypeScript that defines one table and two TABLE - user (public) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - age: number (i32) - active: boolean REDUCERS - insertUser: given name:string, age:number, active:boolean, insert a new row (id is auto-assigned) -- deleteUser: given id:number, delete the row with matching id +- deleteUser: given id:bigint, delete the row with matching id diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/typescript.txt index b644e2d5876..90750f29fd1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_007_crud/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that defines one table and a re TABLE - user (public) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - age: number (i32) - active: boolean diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt index de9ed908263..6a235aed64c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_008_index_lookup/tasks/typescript.txt @@ -3,16 +3,16 @@ Write a SpacetimeDB backend module in TypeScript that defines two tables and two TABLES - user (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - age: number (i32) - active: boolean - result (public) - Fields: - - id: number (u64, primary key) + - id: bigint (u64, primary key) - name: string REDUCERS - insertUser: given name:string, age:number, active:boolean, insert a new row into user (id is auto-assigned) -- lookupUserName: given id:number, find user by id, insert projection (id, name) into result table \ No newline at end of file +- lookupUserName: given id:bigint, find user by id, insert projection (id, name) into result table \ No newline at end of file diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/typescript.txt index 326bd15c9e0..e770cea8b4a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_009_init/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that defines one table and an i TABLE - user (public) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - age: number (i32) - active: boolean diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt index a9a80b3415d..96389250d65 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt @@ -6,6 +6,6 @@ TABLE - id: bigint (u64, primary key, auto-increment) - kind: string -LIFECYCLE HOOKS (use spacetimedb.clientConnected / spacetimedb.clientDisconnected) -- onConnect: insert event with kind="connected" -- onDisconnect: insert event with kind="disconnected" +LIFECYCLE HOOKS +- onConnect: when a client connects, insert event with kind="connected" +- onDisconnect: when a client disconnects, insert event with kind="disconnected" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_011_helper_function/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_011_helper_function/tasks/typescript.txt index b0b99765a60..3a21295e132 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_011_helper_function/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_011_helper_function/tasks/typescript.txt @@ -10,4 +10,4 @@ HELPER FUNCTION - add(a: number, b: number): number - returns a + b REDUCERS -- computeSum: given id:number, a:number, b:number, insert row with sum=add(a,b) \ No newline at end of file +- computeSum: given id:number (i32), a:number (i32), b:number (i32), insert row with sum=add(a,b) \ No newline at end of file diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt index 0773d06ea68..35b5c561791 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt @@ -12,4 +12,4 @@ TABLE REDUCERS - SendReminder: scheduled reducer triggered by Reminder (receives a Reminder row, does nothing) -- Init (ReducerKind.Init): inserts one row into Reminder using ScheduleAt.Time with a TimeDuration of 60 seconds (60_000_000 microseconds) added to ctx.Timestamp +- Init: inserts one row into Reminder, scheduling SendReminder to fire 60 seconds in the future using ScheduleAt.Time diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt index f2d79b2773e..91fab67e6f6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt @@ -13,4 +13,3 @@ TABLE REDUCERS - send_reminder: scheduled reducer triggered by reminder (receives a Reminder row, does nothing) - init: inserts one row into reminder, scheduling send_reminder to fire 60 seconds in the future using ScheduleAt::Time - - Use ctx.timestamp + Duration::from_secs(60) to compute the fire time diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt index 88d00e86ec0..9551845afaa 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt @@ -11,4 +11,3 @@ TABLE REDUCERS - send_reminder: scheduled reducer triggered by reminder (receives a reminder row argument, does nothing) - init: inserts one row into reminder scheduling send_reminder to fire 60 seconds in the future using ScheduleAt.time - - Use ctx.timestamp + 60_000_000n microseconds to compute the fire time as a BigInt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt index 94e183745b9..faf93dbc809 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt @@ -11,5 +11,5 @@ TABLE REDUCERS - RunCleanup: scheduled reducer triggered by CleanupJob (receives a CleanupJob row, does nothing) -- Init (ReducerKind.Init): inserts one row into CleanupJob with a 60-second interval (60_000_000 microseconds) +- Init: inserts one row into CleanupJob with a 60-second interval (60_000_000 microseconds) - CancelCleanup(ctx, scheduledId: ulong): deletes the CleanupJob row with the given scheduledId, canceling the pending scheduled call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/csharp.txt index bf28e233a69..236bf54d045 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/csharp.txt @@ -7,5 +7,5 @@ TABLE - ConnectedAt: Timestamp REDUCERS -- ClientConnected (ReducerKind.ClientConnected): inserts an OnlinePlayer row using ctx.Sender as the identity and ctx.Timestamp as ConnectedAt -- ClientDisconnected (ReducerKind.ClientDisconnected): deletes the OnlinePlayer row for ctx.Sender +- ClientConnected: lifecycle reducer that inserts an OnlinePlayer row with the caller's identity and the current time as ConnectedAt +- ClientDisconnected: lifecycle reducer that deletes the OnlinePlayer row for the caller's identity diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/rust.txt index 6489899584b..6c2a3770de6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/rust.txt @@ -8,5 +8,5 @@ TABLE - connected_at: Timestamp REDUCERS -- client_connected: lifecycle reducer that inserts an OnlinePlayer row using ctx.sender() as the identity and ctx.timestamp as connected_at -- client_disconnected: lifecycle reducer that deletes the OnlinePlayer row for ctx.sender() +- client_connected: lifecycle reducer that inserts an OnlinePlayer row with the caller's identity and the current time as connected_at +- client_disconnected: lifecycle reducer that deletes the OnlinePlayer row for the caller's identity diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/typescript.txt index 392d93eca9a..e4bba770c27 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_040_lifecycle_player/tasks/typescript.txt @@ -7,5 +7,5 @@ TABLE - connectedAt: timestamp REDUCERS -- clientConnected: lifecycle reducer (spacetimedb.clientConnected) that inserts an online_player row using ctx.sender as the identity and ctx.timestamp as connectedAt -- clientDisconnected: lifecycle reducer (spacetimedb.clientDisconnected) that deletes the online_player row for ctx.sender +- clientConnected: lifecycle reducer that inserts an online_player row with the caller's identity and the current time as connectedAt +- clientDisconnected: lifecycle reducer that deletes the online_player row for the caller's identity diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/typescript.txt index be29ae851c7..fa1a200ea23 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/tasks/typescript.txt @@ -4,10 +4,10 @@ TABLES - damage_event - public event table (rows exist only for the duration of the transaction, set event: true) - Fields: - - entityId: number (u64) + - entityId: bigint (u64) - damage: number (u32) - source: string REDUCERS -- deal_damage(ctx, { entityId: number (u64), damage: number (u32), source: string }) +- deal_damage(ctx, { entityId: bigint (u64), damage: number (u32), source: string }) - Insert a row into the damage_event event table diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt index 9960fb64f79..f5cd29c907e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a table with optio TABLES - player (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - nickname: optional string (use t.option(t.string())) - highScore: optional number (use t.option(t.u32())) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt index 9436daa1913..d17ece9b5a1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt @@ -3,16 +3,16 @@ Write a SpacetimeDB backend module in TypeScript that defines an author table an TABLES - author (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - post (private) - Fields: - - id: number (u64, primary key, autoInc) - - authorId: number (u64, index btree) + - id: bigint (u64, primary key, autoInc) + - authorId: bigint (u64, index btree) - title: string REDUCERS -- delete_author(ctx, { authorId: number (u64) }) +- delete_author(ctx, { authorId: bigint (u64) }) - First, iterate all posts with matching authorId and delete each one - Then delete the author by id diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt index c3b1d7cb066..3eb2a517cf1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt @@ -3,15 +3,15 @@ Write a SpacetimeDB backend module in TypeScript that defines an order table and TABLES - order (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - category: string (index btree) - - amount: number (u64) + - amount: bigint (u64) - fulfilled: boolean - category_stats (private) - Fields: - category: string (primary key) - - totalAmount: number (u64) + - totalAmount: bigint (u64) - orderCount: number (u32) REDUCERS diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt index 3cf12d4233d..a1af054b950 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt @@ -3,19 +3,19 @@ Write a SpacetimeDB backend module in TypeScript that defines a customer table, TABLES - customer (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - order (private) - Fields: - - id: number (u64, primary key, autoInc) - - customerId: number (u64, index btree) + - id: bigint (u64, primary key, autoInc) + - customerId: bigint (u64, index btree) - product: string - amount: number (u32) - order_detail (private) - Fields: - - orderId: number (u64, primary key) + - orderId: bigint (u64, primary key) - customerName: string - product: string - amount: number (u32) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt index 9f0d668afde..d3036f7ae14 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a table with a uni TABLES - account (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - email: string (unique) - displayName: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs index 740c6cc7419..a5691b6e5e5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs @@ -2,7 +2,7 @@ use crate::eval::defaults::{default_schema_parity_scorers, make_http_route_parit use crate::eval::BenchmarkSpec; pub fn spec() -> BenchmarkSpec { - BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_http_route_parity_scorer( host_url, @@ -11,12 +11,18 @@ pub fn spec() -> BenchmarkSpec { vec![ ("GET", "/items", None), ("POST", "/items", Some("book")), - ("PUT", "/items", Some("no")), - ("GET", "/missing", None), ], false, "router_method_matrix", )); + // The prompt says reject, not how: the server's own 404 and a module's 405 both qualify. + scorers.push(crate::eval::scenario::scenario(file!(), route_tag, host_url, lang, |s| { + for (method, path) in [(reqwest::Method::PUT, "/items"), (reqwest::Method::GET, "/missing")] { + let (status, body) = s.http_method(method.clone(), path, "no")?; + anyhow::ensure!((400..500).contains(&status), "{method} {path}: expected a rejection, got {status}: {body}"); + } + Ok(()) + })); scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt index 7cc46d493b7..96319f708cc 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt @@ -3,6 +3,7 @@ Write a SpacetimeDB backend module in C# that defines a table of user profiles a TABLES - Profile - Struct: Profile + - public table - Fields: - Id: ulong (primary key, AutoInc) - Identity: Identity (unique) @@ -13,5 +14,5 @@ VIEWS - MyProfile - per-user view (uses ViewContext, not AnonymousViewContext) - public - - Returns: Profile? (the single profile matching ctx.Sender, or null) - - Logic: find the profile by the Identity column matching ctx.Sender + - Returns: Profile? (the single profile matching the caller's identity, or null) + - Logic: find the profile by the Identity column matching the caller's identity diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt index 7016433ee2d..6c18f686c33 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt @@ -14,5 +14,5 @@ VIEWS - my_profile - per-user view (uses ViewContext, not AnonymousViewContext) - public - - Returns: Option (the single profile row matching ctx.sender(), or None) - - Logic: look up the profile by the identity column matching ctx.sender() + - Returns: Option (the single profile row matching the caller's identity, or None) + - Logic: look up the profile by the identity column matching the caller's identity diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt index d2088770afd..0376822cb40 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt @@ -4,14 +4,14 @@ TABLES - profile - public table - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - identity: Identity (unique) - name: string - bio: string VIEWS - my_profile - - per-user view (uses spacetimedb.view, not spacetimedb.anonymousView) + - per-user view, not an anonymous one - public - - Returns: option of profile row (the single profile matching ctx.sender(), or undefined) - - Logic: find the profile by the identity column matching ctx.sender() + - Returns: option of profile row (the single profile matching the caller's identity, or undefined) + - Logic: find the profile by the identity column matching the caller's identity diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt index 0faeeb9f4d3..6ac2fc14b62 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt @@ -3,13 +3,13 @@ Write a SpacetimeDB backend module in TypeScript that defines a product table wi TABLES - product (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - price: number (u32, index btree) - price_range_result (private) - Fields: - - productId: number (u64, primary key) + - productId: bigint (u64, primary key) - name: string - price: number (u32) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt index 83e8368d6af..ac403927f66 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt @@ -3,15 +3,15 @@ Write a SpacetimeDB backend module in TypeScript that defines a player table wit TABLES - player (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - - score: number (u64) + - score: bigint (u64) - leaderboard (private) - Fields: - rank: number (u32, primary key) - playerName: string - - score: number (u64) + - score: bigint (u64) REDUCERS - build_leaderboard(ctx, { limit: number (u32) }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt index 59f899d0829..87d24f1fb65 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt @@ -3,13 +3,13 @@ Write a SpacetimeDB backend module in TypeScript that defines a task table and a TABLES - task (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - title: string - completed: boolean - first_incomplete (private) - Fields: - - taskId: number (u64, primary key) + - taskId: bigint (u64, primary key) - title: string REDUCERS diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt index 809d808c5d5..bc2aa5214f4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that defines an order table and TABLES - order (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - category: string - amount: number (u32) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt index 6018961e127..a7e951e3a31 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt @@ -3,14 +3,14 @@ Write a SpacetimeDB backend module in TypeScript that defines a user table and a TABLES - user (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - name: string - active: boolean - user_stats (private) - Fields: - key: string (primary key) - - count: number (u64) + - count: bigint (u64) REDUCERS - compute_user_counts(ctx) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt index 70ddcea8163..1d5656f19b3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that defines an event log table TABLES - event_log (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - category: string - severity: number (u32) - message: string @@ -11,7 +11,7 @@ TABLES - filtered_event (private) - Fields: - - eventId: number (u64, primary key) + - eventId: bigint (u64, primary key) - message: string REDUCERS diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt index 2791f08bec4..2d944333672 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt @@ -16,7 +16,7 @@ REDUCERS - CreateAccount(id: ulong, balance: long) - Insert an Account with the supplied values - Transfer(requestId: string, fromId: ulong, toId: ulong, amount: long) - - Reject invalid or insufficient transfers + - Fail with an error if amount is not positive, fromId equals toId, either account does not exist, or the source balance is below amount - Update both Account balances atomically - Record RequestId in TransferRequest - If RequestId is already recorded, make no changes diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt index 5dc491ee6be..6a4ac861784 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt @@ -16,7 +16,7 @@ REDUCERS - create_account(id: u64, balance: i64) - Insert an account with the supplied values - transfer(request_id: String, from_id: u64, to_id: u64, amount: i64) - - Reject invalid or insufficient transfers + - Fail with an error if amount is not positive, from_id equals to_id, either account does not exist, or the source balance is below amount - Update both account balances atomically - Record request_id in transfer_request - If request_id is already recorded, make no changes diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt index fc87b8ae076..365d4246c54 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt @@ -16,7 +16,7 @@ REDUCERS - create_account(id: u64, balance: i64) - Insert an account with the supplied values - transfer(requestId: string, fromId: u64, toId: u64, amount: i64) - - Reject invalid or insufficient transfers + - Fail with an error if amount is not positive, fromId equals toId, either account does not exist, or the source balance is below amount - Update both account balances atomically - Record requestId in transfer_request - If requestId is already recorded, make no changes diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt index e0d32d485f9..cb73d1746ff 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt @@ -9,6 +9,6 @@ TABLE REDUCERS - Generate() - - Store ctx.Timestamp in CreatedAt - - Store a random long from ctx.Rng in RandomValue + - Store the reducer context timestamp in CreatedAt + - Store a random long from the reducer context RNG in RandomValue - Do not use a host clock or a separately seeded Random diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt index 6d3d105d6e2..c3d3f4df77d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt @@ -9,6 +9,6 @@ TABLE REDUCERS - generate() - - Store ctx.timestamp in createdAt - - Store a random integer from ctx.random in randomValue + - Store the reducer context timestamp in createdAt + - Store a random integer from the reducer context RNG in randomValue - Do not use Date, Math.random, or a separately seeded RNG diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/typescript.txt index adf4f2dca13..ef0ba02b58e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_014_elementary_columns/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that defines one table and seed TABLE - primitive (public) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - count: number (i32) - total: bigint (i64) - price: number (f32) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/typescript.txt index 6f6ecab7a1b..f4c66f6e0af 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_015_product_type_columns/tasks/typescript.txt @@ -11,7 +11,7 @@ TYPES TABLE - profile (public) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - home: Address - work: Address - pos: Position diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/typescript.txt index fe7310071e6..27c02c3ea99 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_016_sum_type_columns/tasks/typescript.txt @@ -11,7 +11,7 @@ TYPES TABLE - drawing (public) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - a: Shape - b: Shape diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/typescript.txt index db89d6a6baf..f85d783606f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_018_constraints/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that defines one table and seed TABLE - account (public) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - email: string (unique) - name: string - Index: byName (btree on name) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt index 59e1a1563d3..1040e640ecb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_019_many_to_many/tasks/typescript.txt @@ -3,19 +3,19 @@ Write a SpacetimeDB backend module in TypeScript that defines three tables model TABLES - user (private) - Fields: - - userId: number (u64, primary key, autoInc) + - userId: bigint (u64, primary key, autoInc) - name: string - group (private) - Fields: - - groupId: number (u64, primary key, autoInc) + - groupId: bigint (u64, primary key, autoInc) - title: string - membership (public) - Fields: - - id: number (u64, primary key, autoInc) - - userId: number (u64) - - groupId: number (u64) + - id: bigint (u64, primary key, autoInc) + - userId: bigint (u64) + - groupId: bigint (u64) - Indexes: - byUser (btree on userId) - byGroup (btree on groupId) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt index 225703a9966..4d678ce5c07 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_020_ecs/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that models a minimal ECS and c TABLES - entity (private) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - position (public) - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/typescript.txt index 9ab61091b68..22608b5626f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_021_multi_column_index/tasks/typescript.txt @@ -3,7 +3,7 @@ Write a SpacetimeDB backend module in TypeScript that defines one table with a m TABLE - log (public) - Fields: - - id: number (u64, primary key, autoInc) + - id: bigint (u64, primary key, autoInc) - userId: number (i32) - day: number (i32) - message: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt index 13dd98d2208..93260365616 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt @@ -12,6 +12,6 @@ TABLE REDUCERS - store_blob(filename: string, mimeType: string, data: u8 array) - - Store ctx.sender as owner + - Store the caller's identity as owner - Derive size from the byte length - Insert the bytes unchanged diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs index 778ba91fefb..8741666440c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs @@ -1,10 +1,10 @@ -use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::defaults::{make_reducer_data_parity_scorer, view_primary_key_schema_parity_scorers}; use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; use std::time::Duration; pub fn spec() -> BenchmarkSpec { BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { - let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let mut scorers = view_primary_key_schema_parity_scorers(host_url, file!(), route_tag); scorers.push(make_reducer_data_parity_scorer( host_url, ReducerDataParityConfig { diff --git a/tools/xtask-llm-benchmark/src/eval/defaults.rs b/tools/xtask-llm-benchmark/src/eval/defaults.rs index f78fd6c1b5b..6525ba7f3bb 100644 --- a/tools/xtask-llm-benchmark/src/eval/defaults.rs +++ b/tools/xtask-llm-benchmark/src/eval/defaults.rs @@ -8,6 +8,19 @@ use crate::eval::{derive_cat_task_from_file, ReducerDataParityConfig, ReducerSql use std::time::Duration; pub fn default_schema_parity_scorers(host_url: &str, src_file: &str, route_tag: &str) -> Vec> { + schema_parity_scorers(host_url, src_file, route_tag, false) +} + +pub fn view_primary_key_schema_parity_scorers(host_url: &str, src_file: &str, route_tag: &str) -> Vec> { + schema_parity_scorers(host_url, src_file, route_tag, true) +} + +fn schema_parity_scorers( + host_url: &str, + src_file: &str, + route_tag: &str, + view_primary_keys_required: bool, +) -> Vec> { let (cat, task) = derive_cat_task_from_file(src_file); let golden_db = golden_db_name(&cat, &task, route_tag); @@ -19,6 +32,7 @@ pub fn default_schema_parity_scorers(host_url: &str, src_file: &str, route_tag: llm_db, timeout: Duration::from_secs(10), id_str: "schema_parity", + view_primary_keys_required, }) as Box] } diff --git a/tools/xtask-llm-benchmark/src/eval/scenario.rs b/tools/xtask-llm-benchmark/src/eval/scenario.rs index 45955c35b4b..6015fa77d3f 100644 --- a/tools/xtask-llm-benchmark/src/eval/scenario.rs +++ b/tools/xtask-llm-benchmark/src/eval/scenario.rs @@ -168,10 +168,14 @@ impl Scenario { } pub fn http(&self, path: &str, body: &str) -> Result<(u16, String)> { + self.http_method(reqwest::Method::POST, path, body) + } + + pub fn http_method(&self, method: reqwest::Method, path: &str, body: &str) -> Result<(u16, String)> { let response = reqwest::blocking::Client::builder() .timeout(Duration::from_secs(15)) .build()? - .post(format!("{}/v1/database/{}/route{path}", self.server, self.db)) + .request(method, format!("{}/v1/database/{}/route{path}", self.server, self.db)) .header("content-type", "text/plain") .body(body.to_owned()) .send()?; diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 99e4f90acdb..c64dfcd1ef2 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -20,6 +20,7 @@ pub struct SchemaParityScorer { pub llm_db: String, pub timeout: Duration, pub id_str: &'static str, + pub view_primary_keys_required: bool, } #[derive(Debug, Default, PartialEq, Eq)] @@ -107,12 +108,14 @@ impl Scorer for SchemaParityScorer { } } - let schema_a = extract_schema(&golden); + let mut schema_a = extract_schema(&golden); let mut schema_b = extract_schema(&llm); - // Extra view primary keys only help clients match rows; behaviour checks catch a wrong one. - schema_b - .exports - .retain(|export| !export.starts_with("view_primary_key:") || schema_a.exports.contains(export)); + compare_view_primary_keys( + &mut schema_a.exports, + &mut schema_b.exports, + self.view_primary_keys_required, + ); + accept_extra_http_routes(&schema_a.exports, &mut schema_b.exports); let candidate_tables = accept_extra_indexes(&schema_a.tables, align_table_names(&schema_a.tables, &schema_b.tables)); @@ -311,7 +314,19 @@ fn extract_schema(v: &Value) -> SchemaSnapshot { for view in schema_array(v, "views", "Views").into_iter().flatten() { schema.exports.insert(view_export(view, types, &names)); } - insert_canonical_exports(&mut schema.exports, v, "schedules", "Schedules", "schedule"); + for schedule in schema_array(v, "schedules", "Schedules").into_iter().flatten() { + // The table is compared on its own; these fields only repeat its exact spelling. + let mut schedule = schedule.clone(); + if let Some(fields) = schedule.as_object_mut() { + fields.remove("source_name"); + if let Some(table) = fields.get_mut("table_name") { + *table = Value::String(loose_name(&schema_name(Some(table)))); + } + } + schema + .exports + .insert(format!("schedule:{}", canonical_value(Some(&schedule)))); + } insert_http_routes(&mut schema.exports, v); for key in schema_array(v, "view_primary_keys", "ViewPrimaryKeys") .into_iter() @@ -405,18 +420,6 @@ fn view_export(view: &Value, types: &[Value], names: &SchemaNames) -> String { signature } -fn insert_canonical_exports( - exports: &mut BTreeSet, - schema: &Value, - direct_name: &str, - section_name: &str, - kind: &str, -) { - for export in schema_array(schema, direct_name, section_name).into_iter().flatten() { - exports.insert(format!("{kind}:{}", canonical_value(Some(export)))); - } -} - fn insert_http_routes(exports: &mut BTreeSet, schema: &Value) { for route in schema_array(schema, "http_routes", "HttpRoutes").into_iter().flatten() { let mut route = route.clone(); @@ -566,6 +569,22 @@ fn align_table_names( .collect() } +// TypeScript row types carry a table's key into a view implicitly, so only a task that asks for a key requires one. +fn compare_view_primary_keys(golden: &mut BTreeSet, candidate: &mut BTreeSet, required: bool) { + let is_key = |export: &String| export.starts_with("view_primary_key:"); + if required { + candidate.retain(|export| !is_key(export) || golden.contains(export)); + } else { + golden.retain(|export| !is_key(export)); + candidate.retain(|export| !is_key(export)); + } +} + +// A handler that rejects a method explicitly is as valid as leaving the route unregistered. +fn accept_extra_http_routes(golden: &BTreeSet, candidate: &mut BTreeSet) { + candidate.retain(|export| !export.starts_with("http_route:") || golden.contains(export)); +} + fn loose_name(name: &str) -> String { name.to_lowercase().replace('_', "") } @@ -1658,6 +1677,67 @@ mod tests { assert_ne!(golden, extract_schema(&lifecycle_schema("client_connected", "Init"))); } + #[test] + fn schedules_ignore_table_spelling_but_not_their_reducer() { + let schema = |table: &str, function: &str| { + json!({ + "sections": [ + { "Typespace": { "types": [] } }, + { "Schedules": [{ + "source_name": { "some": format!("{table}_sched") }, + "table_name": table, + "schedule_at_col": 1, + "function_name": function + }] } + ] + }) + }; + let golden = extract_schema(&schema("tick_timer", "tick")); + + assert_eq!(golden, extract_schema(&schema("TickTimer", "tick"))); + assert_ne!(golden, extract_schema(&schema("tick_timer", "other"))); + assert_ne!(golden, extract_schema(&schema("other_timer", "tick"))); + } + + #[test] + fn view_primary_keys_only_matter_when_a_task_requires_them() { + let keys = |names: &[&str]| { + names + .iter() + .map(|name| format!("view_primary_key:{name}")) + .collect::>() + }; + for (required, golden, candidate, equal) in [ + (false, vec!["id"], vec![], true), + (false, vec![], vec!["id"], true), + (true, vec!["id"], vec!["id"], true), + (true, vec![], vec!["id"], true), + (true, vec!["id"], vec![], false), + (true, vec!["id"], vec!["name"], false), + ] { + let (mut golden, mut candidate) = (keys(&golden), keys(&candidate)); + compare_view_primary_keys(&mut golden, &mut candidate, required); + assert_eq!(diff_sets(&golden, &candidate).is_null(), equal); + } + } + + #[test] + fn extra_http_routes_are_accepted_but_missing_ones_fail() { + let routes = |names: &[&str]| { + names + .iter() + .map(|name| format!("http_route:{name}")) + .collect::>() + }; + let golden = routes(&["get", "post"]); + let mut extra = routes(&["get", "post", "put"]); + accept_extra_http_routes(&golden, &mut extra); + assert!(diff_sets(&golden, &extra).is_null()); + let mut missing = routes(&["get"]); + accept_extra_http_routes(&golden, &mut missing); + assert!(!diff_sets(&golden, &missing).is_null()); + } + #[test] fn view_names_differing_only_by_case_or_underscores_match() { let schema = |view: &str| { From 0581850204357c3bd2ef2619368fc5a302db32f6 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 3 Nov 2026 20:58:22 -0500 Subject: [PATCH 24/41] State failure and query requirements in prompts instead of how to write them Prompts said to panic or throw, gave the id value that triggers auto-increment, and named collections and iterator calls. The checks on those tasks accept any rejection and never required those forms. --- .../t_026_auth_identity_check/tasks/csharp.txt | 5 ++--- .../auth/t_026_auth_identity_check/tasks/rust.txt | 5 ++--- .../t_026_auth_identity_check/tasks/typescript.txt | 5 ++--- .../t_027_private_vs_public_table/tasks/csharp.txt | 2 +- .../t_027_private_vs_public_table/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_041_registered_user_gate/tasks/csharp.txt | 6 +++--- .../auth/t_041_registered_user_gate/tasks/rust.txt | 6 +++--- .../tasks/typescript.txt | 6 +++--- .../auth/t_042_admin_bootstrap/tasks/csharp.txt | 8 ++++---- .../auth/t_042_admin_bootstrap/tasks/rust.txt | 8 ++++---- .../t_042_admin_bootstrap/tasks/typescript.txt | 8 ++++---- .../auth/t_043_role_based_access/tasks/csharp.txt | 14 +++++++------- .../auth/t_043_role_based_access/tasks/rust.txt | 14 +++++++------- .../t_043_role_based_access/tasks/typescript.txt | 14 +++++++------- .../auth/t_044_ban_list/tasks/csharp.txt | 8 ++++---- .../benchmarks/auth/t_044_ban_list/tasks/rust.txt | 8 ++++---- .../auth/t_044_ban_list/tasks/typescript.txt | 8 ++++---- .../auth/t_045_rate_limit/tasks/csharp.txt | 4 ++-- .../auth/t_045_rate_limit/tasks/rust.txt | 4 ++-- .../auth/t_045_rate_limit/tasks/typescript.txt | 4 ++-- .../auth/t_046_shared_document/tasks/csharp.txt | 10 +++++----- .../auth/t_046_shared_document/tasks/rust.txt | 10 +++++----- .../t_046_shared_document/tasks/typescript.txt | 10 +++++----- .../t_025_optional_fields/tasks/csharp.txt | 1 - .../t_025_optional_fields/tasks/rust.txt | 1 - .../t_025_optional_fields/tasks/typescript.txt | 5 ++--- .../t_031_unique_constraint/tasks/csharp.txt | 1 - .../t_031_unique_constraint/tasks/rust.txt | 1 - .../t_031_unique_constraint/tasks/typescript.txt | 1 - .../queries/t_033_sort_and_limit/tasks/csharp.txt | 2 +- .../queries/t_033_sort_and_limit/tasks/rust.txt | 2 +- .../t_033_sort_and_limit/tasks/typescript.txt | 2 +- .../queries/t_035_select_distinct/tasks/csharp.txt | 2 +- .../queries/t_035_select_distinct/tasks/rust.txt | 2 +- .../t_035_select_distinct/tasks/typescript.txt | 2 +- .../t_036_count_without_collect/tasks/csharp.txt | 4 ++-- .../t_036_count_without_collect/tasks/rust.txt | 4 ++-- .../tasks/typescript.txt | 4 ++-- 39 files changed, 98 insertions(+), 107 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt index 4daf926f242..f3aa7b8062f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/csharp.txt @@ -12,10 +12,9 @@ TABLES REDUCERS - CreateMessage(ctx, string text) - Insert a new Message with Owner set to the caller's identity - - Use Id: 0 for AutoInc - DeleteMessage(ctx, ulong id) - Find the message by Id - - If not found, throw exception with "not found" - - If the message's Owner does not equal the caller's identity, throw exception with "unauthorized" + - If not found, fail with "not found" + - If the message's Owner does not equal the caller's identity, fail with "unauthorized" - Otherwise, delete the message diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt index 41a4594d4c9..1501fdc2f5c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/rust.txt @@ -12,10 +12,9 @@ TABLES REDUCERS - create_message(ctx, text: String) - Insert a new Message with owner set to the caller's identity - - Use id: 0 for auto_inc - delete_message(ctx, id: u64) - Find the message by id - - If not found, panic with "not found" - - If the message's owner does not equal the caller's identity, panic with "unauthorized" + - If not found, fail with "not found" + - If the message's owner does not equal the caller's identity, fail with "unauthorized" - Otherwise, delete the message diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt index 625f8760c9a..c20bcfe6220 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_026_auth_identity_check/tasks/typescript.txt @@ -11,10 +11,9 @@ TABLES REDUCERS - create_message(ctx, { text: string }) - Insert a new message with owner set to the caller's identity - - Use id: 0n for autoInc - delete_message(ctx, { id: bigint (u64) }) - Find the message by id - - If not found, throw "not found" - - If the message's owner does not equal the caller's identity, throw "unauthorized" + - If not found, fail with "not found" + - If the message's owner does not equal the caller's identity, fail with "unauthorized" - Otherwise, delete the message diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/csharp.txt index 69e2026892d..16500e780ef 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/csharp.txt @@ -19,5 +19,5 @@ TABLES REDUCERS - RegisterUser(ctx, string name, string email, string passwordHash) - - Insert into UserInternal (Id: 0 for AutoInc) + - Insert into UserInternal - Also insert into UserPublic with just the Id and Name (safe fields only) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/rust.txt index 3b3ae7cf0e1..34e5c2d1398 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/rust.txt @@ -19,5 +19,5 @@ TABLES REDUCERS - register_user(ctx, name: String, email: String, password_hash: String) - - Insert into user_internal (id: 0 for auto_inc) + - Insert into user_internal - Also insert into user_public with just the id and name (safe fields only) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt index e2a2cdfd33e..28717599027 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_027_private_vs_public_table/tasks/typescript.txt @@ -17,5 +17,5 @@ TABLES REDUCERS - register_user(ctx, { name: string, email: string, passwordHash: string }) - - Insert into user_internal (id: 0n for autoInc) + - Insert into user_internal - Also insert into user_public with just the id and name (safe fields only) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt index c4cae4988db..5dcd4f1c4ce 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/csharp.txt @@ -18,9 +18,9 @@ TABLES REDUCERS - RegisterUser(ctx, string name) - - If a User row already exists for the caller's identity, throw exception with "already registered" + - If a User row already exists for the caller's identity, fail with "already registered" - Otherwise insert a new User with Identity = the caller's identity and the given Name - PostMessage(ctx, string text) - - If no User row exists for the caller's identity, throw exception with "not registered" - - Otherwise insert a Message with Sender = the caller's identity and the given Text, Id = 0 for AutoInc + - If no User row exists for the caller's identity, fail with "not registered" + - Otherwise insert a Message with Sender = the caller's identity and the given Text diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt index a30093f935b..71840f09160 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/rust.txt @@ -18,9 +18,9 @@ TABLES REDUCERS - register(ctx, name: String) - - If a User row already exists for the caller's identity, panic with "already registered" + - If a User row already exists for the caller's identity, fail with "already registered" - Otherwise insert a new User with identity = the caller's identity and the given name - post_message(ctx, text: String) - - If no User row exists for the caller's identity, panic with "not registered" - - Otherwise insert a Message with sender = the caller's identity and the given text, id = 0 for auto_inc + - If no User row exists for the caller's identity, fail with "not registered" + - Otherwise insert a Message with sender = the caller's identity and the given text diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt index ed96e6112e0..760956b7bca 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/tasks/typescript.txt @@ -16,9 +16,9 @@ TABLES REDUCERS - register(ctx, { name: string }) - - If a user row already exists for the caller's identity, throw "already registered" + - If a user row already exists for the caller's identity, fail with "already registered" - Otherwise insert a new user with identity = the caller's identity and the given name - post_message(ctx, { text: string }) - - If no user row exists for the caller's identity, throw "not registered" - - Otherwise insert a message with sender = the caller's identity and the given text, id = 0n for autoInc + - If no user row exists for the caller's identity, fail with "not registered" + - Otherwise insert a message with sender = the caller's identity and the given text diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt index e09bceb4f06..3dc21e5c7bf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/csharp.txt @@ -17,10 +17,10 @@ TABLES REDUCERS - BootstrapAdmin(ctx) - - If a Config row already exists (Id = 0), throw exception with "already bootstrapped" + - If a Config row already exists (Id = 0), fail with "already bootstrapped" - Otherwise insert Config { Id = 0, Admin = the caller's identity } - AdminAction(ctx, string action) - - Load the Config row with Id = 0; if missing, throw exception with "not bootstrapped" - - If the caller's identity does not equal config.Admin, throw exception with "not admin" - - Insert an AdminLog row with the given Action, Id = 0 for AutoInc + - Load the Config row with Id = 0; if missing, fail with "not bootstrapped" + - If the caller's identity does not equal config.Admin, fail with "not admin" + - Insert an AdminLog row with the given Action diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt index 5ee666aeb45..ffea86e5615 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/rust.txt @@ -17,10 +17,10 @@ TABLES REDUCERS - bootstrap_admin(ctx) - - If a Config row already exists (id = 0), panic with "already bootstrapped" + - If a Config row already exists (id = 0), fail with "already bootstrapped" - Otherwise insert Config { id: 0, admin: the caller's identity } - admin_action(ctx, action: String) - - Load the Config row with id = 0; if missing, panic with "not bootstrapped" - - If the caller's identity does not equal config.admin, panic with "not admin" - - Insert an AdminLog row with the given action, id = 0 for auto_inc + - Load the Config row with id = 0; if missing, fail with "not bootstrapped" + - If the caller's identity does not equal config.admin, fail with "not admin" + - Insert an AdminLog row with the given action diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt index 5ad23f5e119..5efaa025eee 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_042_admin_bootstrap/tasks/typescript.txt @@ -15,10 +15,10 @@ TABLES REDUCERS - bootstrap_admin(ctx) - - If a config row already exists (id = 0), throw "already bootstrapped" + - If a config row already exists (id = 0), fail with "already bootstrapped" - Otherwise insert config { id: 0, admin: the caller's identity } - admin_action(ctx, { action: string }) - - Load the config row with id = 0; if missing, throw "not bootstrapped" - - If the caller's identity does not equal config.admin, throw "not admin" - - Insert an admin_log row with the given action, id = 0n for autoInc + - Load the config row with id = 0; if missing, fail with "not bootstrapped" + - If the caller's identity does not equal config.admin, fail with "not admin" + - Insert an admin_log row with the given action diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt index 6f686fe2475..3ded1693f09 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/csharp.txt @@ -10,18 +10,18 @@ TABLES REDUCERS - RegisterUser(ctx) - - If a User row already exists for the caller's identity, throw exception with "already registered" + - If a User row already exists for the caller's identity, fail with "already registered" - Insert User { Identity = the caller's identity, Role = "member" } - Promote(ctx, Identity target) - - Load the User row for the caller's identity; if missing, throw exception with "not registered" - - If the caller's Role is not "admin", throw exception with "not admin" - - Load the User row for target; if missing, throw exception with "target not registered" + - Load the User row for the caller's identity; if missing, fail with "not registered" + - If the caller's Role is not "admin", fail with "not admin" + - Load the User row for target; if missing, fail with "target not registered" - Update that row's Role to "admin" - MemberAction(ctx) - - Load the User row for the caller's identity; if missing, throw exception with "not registered" + - Load the User row for the caller's identity; if missing, fail with "not registered" - AdminAction(ctx) - - Load the User row for the caller's identity; if missing, throw exception with "not registered" - - If the Role is not "admin", throw exception with "not admin" + - Load the User row for the caller's identity; if missing, fail with "not registered" + - If the Role is not "admin", fail with "not admin" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt index 6776ba27e10..7236b221af7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/rust.txt @@ -10,20 +10,20 @@ TABLES REDUCERS - register(ctx) - - If a User row already exists for the caller's identity, panic with "already registered" + - If a User row already exists for the caller's identity, fail with "already registered" - Insert User { identity: the caller's identity, role: "member" } - promote(ctx, target: Identity) - - Load the User row for the caller's identity; if missing, panic with "not registered" - - If the caller's role is not "admin", panic with "not admin" - - Load the User row for target; if missing, panic with "target not registered" + - Load the User row for the caller's identity; if missing, fail with "not registered" + - If the caller's role is not "admin", fail with "not admin" + - Load the User row for target; if missing, fail with "target not registered" - Update that row's role to "admin" - member_action(ctx) - - Load the User row for the caller's identity; if missing, panic with "not registered" + - Load the User row for the caller's identity; if missing, fail with "not registered" - (No further logic needed; presence of the row is sufficient) - admin_action(ctx) - - Load the User row for the caller's identity; if missing, panic with "not registered" - - If the role is not "admin", panic with "not admin" + - Load the User row for the caller's identity; if missing, fail with "not registered" + - If the role is not "admin", fail with "not admin" - (No further logic needed beyond the guard) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt index 11f17965b9f..31b204f148d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/tasks/typescript.txt @@ -9,18 +9,18 @@ TABLES REDUCERS - register(ctx) - - If a user row already exists for the caller's identity, throw "already registered" + - If a user row already exists for the caller's identity, fail with "already registered" - Insert user { identity: the caller's identity, role: "member" } - promote(ctx, { target: Identity }) - - Load the user row for the caller's identity; if missing, throw "not registered" - - If the caller's role is not "admin", throw "not admin" - - Load the user row for target; if missing, throw "target not registered" + - Load the user row for the caller's identity; if missing, fail with "not registered" + - If the caller's role is not "admin", fail with "not admin" + - Load the user row for target; if missing, fail with "target not registered" - Update that row's role to "admin" - member_action(ctx) - - Load the user row for the caller's identity; if missing, throw "not registered" + - Load the user row for the caller's identity; if missing, fail with "not registered" - admin_action(ctx) - - Load the user row for the caller's identity; if missing, throw "not registered" - - If the role is not "admin", throw "not admin" + - Load the user row for the caller's identity; if missing, fail with "not registered" + - If the role is not "admin", fail with "not admin" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt index 0a1a9eab365..de70d963c87 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/csharp.txt @@ -22,14 +22,14 @@ TABLES REDUCERS - AddAdmin(ctx, Identity target) - - If the caller's identity is not in the Admin table, throw exception with "not admin" + - If the caller's identity is not in the Admin table, fail with "not admin" - Insert Admin { Identity = target } (ignore if already present) - BanPlayer(ctx, Identity target) - - If the caller's identity is not in the Admin table, throw exception with "not admin" + - If the caller's identity is not in the Admin table, fail with "not admin" - Insert Banned { Identity = target } - Remove the Player row for target if one exists - JoinGame(ctx, string name) - - If the caller's identity is in the Banned table, throw exception with "banned" - - Insert Player { Identity = the caller's identity, Name = name }; if already present, throw exception with "already in game" + - If the caller's identity is in the Banned table, fail with "banned" + - Insert Player { Identity = the caller's identity, Name = name }; if already present, fail with "already in game" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt index 6ffc16f7b19..4cda5108635 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/rust.txt @@ -22,14 +22,14 @@ TABLES REDUCERS - add_admin(ctx, target: Identity) - - If the caller's identity is not in the admin table, panic with "not admin" + - If the caller's identity is not in the admin table, fail with "not admin" - Insert Admin { identity: target } (ignore if already present) - ban_player(ctx, target: Identity) - - If the caller's identity is not in the admin table, panic with "not admin" + - If the caller's identity is not in the admin table, fail with "not admin" - Insert Banned { identity: target } - Remove the Player row for target if one exists - join_game(ctx, name: String) - - If the caller's identity is in the banned table, panic with "banned" - - Insert Player { identity: the caller's identity, name }; if already present, panic with "already in game" + - If the caller's identity is in the banned table, fail with "banned" + - Insert Player { identity: the caller's identity, name }; if already present, fail with "already in game" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt index 731a70fa2bf..8cfa324d2d8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_044_ban_list/tasks/typescript.txt @@ -19,15 +19,15 @@ TABLES REDUCERS - add_admin(ctx, { target: Identity }) - - If the caller's identity is not in the admin table, throw "not admin" + - If the caller's identity is not in the admin table, fail with "not admin" - Insert { identity: target } into admin (ignore if already present) - ban_player(ctx, { target: Identity }) - - If the caller's identity is not in the admin table, throw "not admin" + - If the caller's identity is not in the admin table, fail with "not admin" - Insert { identity: target } into banned - Remove the player row for target if one exists - join_game(ctx, { name: string }) - - If the caller's identity is in the banned table, throw "banned" - - If a player row already exists for the caller's identity, throw "already in game" + - If the caller's identity is in the banned table, fail with "banned" + - If a player row already exists for the caller's identity, fail with "already in game" - Insert player { identity: the caller's identity, name } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt index 5ad4fa99e83..3b1278ec852 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/csharp.txt @@ -20,6 +20,6 @@ REDUCERS - LimitedAction(ctx, string payload) - Get the current time in microseconds - Look up the RateLimit row for the caller's identity - - If a row exists and (now - LastCallUs) < 1_000_000, throw exception with "rate limited" + - If a row exists and (now - LastCallUs) < 1_000_000, fail with "rate limited" - Insert or update the RateLimit row with LastCallUs = now - - Insert an ActionLog row with Identity = the caller's identity and the given Payload, Id = 0 for AutoInc + - Insert an ActionLog row with Identity = the caller's identity and the given Payload diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt index 27d223ef742..3ba658c1efd 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/rust.txt @@ -20,6 +20,6 @@ REDUCERS - limited_action(ctx, payload: String) - Get the current time in microseconds - Look up the RateLimit row for the caller's identity - - If a row exists and (now - last_call_us) < 1_000_000, panic with "rate limited" + - If a row exists and (now - last_call_us) < 1_000_000, fail with "rate limited" - Insert or update the RateLimit row with last_call_us = now - - Insert an ActionLog row with identity = the caller's identity and the given payload, id = 0 for auto_inc + - Insert an ActionLog row with identity = the caller's identity and the given payload diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt index 8927ce1e370..aeccd62a05d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_045_rate_limit/tasks/typescript.txt @@ -18,6 +18,6 @@ REDUCERS - limited_action(ctx, { payload: string }) - Get the current time in microseconds - Look up the rate_limit row for the caller's identity - - If a row exists and (now - last_call_us) < 1_000_000n, throw "rate limited" + - If a row exists and (now - last_call_us) < 1_000_000n, fail with "rate limited" - Insert or update the rate_limit row with last_call_us = now - - Insert an action_log row with identity = the caller's identity and the given payload, id = 0n for autoInc + - Insert an action_log row with identity = the caller's identity and the given payload diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt index 391b8302a57..5ab2c7353d7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/csharp.txt @@ -18,16 +18,16 @@ TABLES REDUCERS - CreateDocument(ctx, string title) - - Insert a Document with Owner = the caller's identity and the given Title, Id = 0 for AutoInc + - Insert a Document with Owner = the caller's identity and the given Title - ShareDocument(ctx, ulong documentId, Identity target) - - Load the Document by Id; if missing, throw exception with "not found" - - If Document.Owner != the caller's identity, throw exception with "not owner" + - Load the Document by Id; if missing, fail with "not found" + - If Document.Owner != the caller's identity, fail with "not owner" - Insert a DocumentShare { DocumentId = documentId, SharedWith = target } - EditDocument(ctx, ulong documentId, string newTitle) - - Load the Document by Id; if missing, throw exception with "not found" + - Load the Document by Id; if missing, fail with "not found" - The caller is authorized if they are the Owner OR if a DocumentShare row exists with matching DocumentId and SharedWith == the caller's identity - - If neither condition is true, throw exception with "unauthorized" + - If neither condition is true, fail with "unauthorized" - Update the Document's Title to newTitle diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt index a96af7a5b7d..b17391b2cc7 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/rust.txt @@ -18,16 +18,16 @@ TABLES REDUCERS - create_document(ctx, title: String) - - Insert a Document with owner = the caller's identity and the given title, id = 0 for auto_inc + - Insert a Document with owner = the caller's identity and the given title - share_document(ctx, document_id: u64, target: Identity) - - Load the Document by id; if missing, panic with "not found" - - If document.owner != the caller's identity, panic with "not owner" + - Load the Document by id; if missing, fail with "not found" + - If document.owner != the caller's identity, fail with "not owner" - Insert a DocumentShare { document_id, shared_with: target } - edit_document(ctx, document_id: u64, new_title: String) - - Load the Document by id; if missing, panic with "not found" + - Load the Document by id; if missing, fail with "not found" - The caller is authorized if they are the owner OR if a DocumentShare row exists with matching document_id and shared_with == the caller's identity - - If neither condition is true, panic with "unauthorized" + - If neither condition is true, fail with "unauthorized" - Update the Document's title to new_title diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt index d14050ef853..1b8d365b06a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/tasks/typescript.txt @@ -16,16 +16,16 @@ TABLES REDUCERS - create_document(ctx, { title: string }) - - Insert a document with owner = the caller's identity and the given title, id = 0n for autoInc + - Insert a document with owner = the caller's identity and the given title - share_document(ctx, { document_id: bigint (u64), target: Identity }) - - Load the document by id; if missing, throw "not found" - - If document.owner != the caller's identity, throw "not owner" + - Load the document by id; if missing, fail with "not found" + - If document.owner != the caller's identity, fail with "not owner" - Insert a document_share { document_id, shared_with: target } - edit_document(ctx, { document_id: bigint (u64), new_title: string }) - - Load the document by id; if missing, throw "not found" + - Load the document by id; if missing, fail with "not found" - The caller is authorized if they are the owner OR if a document_share row exists with matching document_id and shared_with == the caller's identity - - If neither condition is true, throw "unauthorized" + - If neither condition is true, fail with "unauthorized" - Update the document's title to new_title diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt index 371545e5e4b..b9842c4fe89 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt @@ -12,4 +12,3 @@ TABLES REDUCERS - CreatePlayer(ctx, string name, string? nickname, uint? highScore) - Insert a Player row, passing through the nullable fields as-is - - Use Id: 0 for AutoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt index 165c6d3426f..4a530aa43c6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt @@ -12,4 +12,3 @@ TABLES REDUCERS - create_player(ctx, name: String, nickname: Option, high_score: Option) - Insert a Player row, passing through the optional fields as-is - - Use id: 0 for auto_inc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt index f5cd29c907e..8d91026fbd6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt @@ -5,10 +5,9 @@ TABLES - Fields: - id: bigint (u64, primary key, autoInc) - name: string - - nickname: optional string (use t.option(t.string())) - - highScore: optional number (use t.option(t.u32())) + - nickname: optional string + - highScore: optional number (u32) REDUCERS - create_player(ctx, { name: string, nickname: optional string, highScore: optional number (u32) }) - Insert a player row, passing through the optional fields as-is - - Use id: 0n for autoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt index f71c512fbe3..ee34ec1e3a5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt @@ -11,4 +11,3 @@ TABLES REDUCERS - CreateAccount(ctx, string email, string displayName) - Insert an Account row with the given email and displayName - - Use Id: 0 for AutoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt index ab63be2c3a1..de84390b5a9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt @@ -11,4 +11,3 @@ TABLES REDUCERS - create_account(ctx, email: String, display_name: String) - Insert an Account row with the given email and display_name - - Use id: 0 for auto_inc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt index d3036f7ae14..6fa9f5cf24f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt @@ -10,4 +10,3 @@ TABLES REDUCERS - create_account(ctx, { email: string, displayName: string }) - Insert an account row with the given email and displayName - - Use id: 0n for autoInc diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt index d8b797ed2af..3a43e38cf8b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt @@ -17,6 +17,6 @@ TABLES REDUCERS - BuildLeaderboard(ctx, uint limit) - - Collect all players into a list, sort by Score descending + - Order players by Score, highest first - Take the first `limit` entries - Insert each into Leaderboard with Rank starting at 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt index 3a6ea9986f8..2325fe6812c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt @@ -17,6 +17,6 @@ TABLES REDUCERS - build_leaderboard(ctx, limit: u32) - - Collect all players into a Vec, sort by score descending + - Order players by score, highest first - Take the first `limit` entries - Insert each into leaderboard with rank starting at 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt index ac403927f66..00d8e901d25 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt @@ -15,6 +15,6 @@ TABLES REDUCERS - build_leaderboard(ctx, { limit: number (u32) }) - - Collect all players into an array, sort by score descending + - Order players by score, highest first - Take the first `limit` entries - Insert each into leaderboard with rank starting at 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt index 0fb60362dc5..eca86c7a91c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt @@ -16,5 +16,5 @@ TABLES REDUCERS - CollectDistinctCategories(ctx) - Iterate all orders - - Collect unique Category values (use a HashSet) + - Collect unique Category values - Insert each unique category into DistinctCategory diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt index 44eee546d60..c52a54ba0ec 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt @@ -16,5 +16,5 @@ TABLES REDUCERS - collect_distinct_categories(ctx) - Iterate all orders - - Collect unique category values (use a HashSet or BTreeSet) + - Collect unique category values - Insert each unique category into distinct_category diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt index bc2aa5214f4..8a346e04eb0 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt @@ -14,5 +14,5 @@ TABLES REDUCERS - collect_distinct_categories(ctx) - Iterate all orders - - Collect unique category values (use a Set) + - Collect unique category values - Insert each unique category into distinct_category diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt index 693d4d3faee..2a2d61ece1c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt @@ -16,7 +16,7 @@ TABLES REDUCERS - ComputeUserCounts(ctx) - - Count total users by iterating (do not collect into a List first) - - Count active users by iterating and checking Active == true + - Count total users without collecting them into a List first + - Count active users the same way - Insert a UserStats row with Key="total" and the total count - Insert a UserStats row with Key="active" and the active count diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt index f6a3b3dfbfb..2d2b657ae02 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt @@ -16,7 +16,7 @@ TABLES REDUCERS - compute_user_counts(ctx) - - Count total users by iterating with .count() (do not collect into a Vec) - - Count active users by iterating with .filter(|u| u.active).count() + - Count total users without collecting them into a Vec + - Count active users the same way - Insert a UserStats row with key="total" and the total count - Insert a UserStats row with key="active" and the active count diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt index a7e951e3a31..e55e54b7bee 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt @@ -14,7 +14,7 @@ TABLES REDUCERS - compute_user_counts(ctx) - - Count total users by iterating (do not collect into an array first) - - Count active users by iterating and checking active === true + - Count total users without collecting them into an array first + - Count active users the same way - Insert a user_stats row with key="total" and the total count - Insert a user_stats row with key="active" and the active count From 0cd35ea54b18d02fe5094f813b61eef6185235a8 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 3 Nov 2026 21:34:37 -0500 Subject: [PATCH 25/41] Test error delivery in its own task and say when init runs The advanced tasks test atomicity and leases, so a plain throw or panic that rejects and rolls back no longer fails them. A new task checks that the caller receives the failure text. Prompts for init now say it runs when the module is first published, since some answers wrote an ordinary reducer that never ran. --- .../t_083_atomic_reservation/tasks/csharp.txt | 2 +- .../t_083_atomic_reservation/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_084_stable_pagination/tasks/csharp.txt | 2 +- .../t_084_stable_pagination/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_085_online_backfill/tasks/csharp.txt | 2 +- .../t_085_online_backfill/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_086_incremental_totals/tasks/csharp.txt | 2 +- .../t_086_incremental_totals/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_088_lease_fencing/tasks/csharp.txt | 2 +- .../t_088_lease_fencing/tasks/rust.txt | 2 +- .../t_088_lease_fencing/tasks/typescript.txt | 2 +- .../t_089_cancel_reschedule/tasks/csharp.txt | 2 +- .../t_089_cancel_reschedule/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_090_transactional_quota/spec.rs | 10 ++++---- .../tasks/csharp.txt | 2 +- .../t_090_transactional_quota/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_091_scheduled_queue/tasks/csharp.txt | 2 +- .../t_091_scheduled_queue/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_093_permission_revocation/spec.rs | 4 ++-- .../tasks/csharp.txt | 2 +- .../tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_002_scheduled_table/tasks/csharp.txt | 2 +- .../t_002_scheduled_table/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_038_schedule_at_time/tasks/csharp.txt | 2 +- .../t_038_schedule_at_time/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../t_039_cancel_schedule/tasks/csharp.txt | 2 +- .../t_039_cancel_schedule/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../answers/csharp.cs | 23 +++++++++++++++++++ .../answers/rust.rs | 17 ++++++++++++++ .../answers/typescript.ts | 18 +++++++++++++++ .../t_102_caller_error_message/spec.rs | 19 +++++++++++++++ .../tasks/csharp.txt | 12 ++++++++++ .../t_102_caller_error_message/tasks/rust.txt | 12 ++++++++++ .../tasks/typescript.txt | 12 ++++++++++ .../t_017_scheduled_columns/tasks/csharp.txt | 2 +- .../t_017_scheduled_columns/tasks/rust.txt | 2 +- .../tasks/typescript.txt | 2 +- .../xtask-llm-benchmark/src/eval/scenario.rs | 6 +++-- .../src/generated/registry.rs | 7 ++++++ 50 files changed, 170 insertions(+), 48 deletions(-) create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/csharp.cs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/rust.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/typescript.ts create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/spec.rs create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/csharp.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/rust.txt create mode 100644 tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/typescript.txt diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt index 91cfb596583..22a6d331fe8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/csharp.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_stock(Id: ulong, Available: long): Insert a new product. Reject a negative quantity with 'invalid stock'. - reserve(RequestId: string, FirstId: ulong, FirstQty: long, SecondId: ulong, SecondQty: long): Exact replay returns success without changes. Changed replay: 'request conflict'. New request requires a nonempty ID, distinct products and positive quantities ('invalid reservation'); both products must exist ('missing product') and have enough stock ('insufficient stock'). Deduct both quantities and record the payload in one transaction. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt index f41eb6db735..f5350514b3c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/rust.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_stock(id: u64, available: i64): Insert a new product. Reject a negative quantity with 'invalid stock'. - reserve(request_id: String, first_id: u64, first_qty: i64, second_id: u64, second_qty: i64): Exact replay returns success without changes. Changed replay: 'request conflict'. New request requires a nonempty ID, distinct products and positive quantities ('invalid reservation'); both products must exist ('missing product') and have enough stock ('insufficient stock'). Deduct both quantities and record the payload in one transaction. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt index 97dcc448456..9abea3d5ac9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_083_atomic_reservation/tasks/typescript.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_stock(id: u64 (bigint), available: i64 (bigint)): Insert a new product. Reject a negative quantity with 'invalid stock'. - reserve(requestId: string, firstId: u64 (bigint), firstQty: i64 (bigint), secondId: u64 (bigint), secondQty: i64 (bigint)): Exact replay returns success without changes. Changed replay: 'request conflict'. New request requires a nonempty ID, distinct products and positive quantities ('invalid reservation'); both products must exist ('missing product') and have enough stock ('insufficient stock'). Deduct both quantities and record the payload in one transaction. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt index 3513a9e48fb..fbfc75ceb03 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/csharp.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_entry(Id: ulong, Tenant: string, OccurredAt: ulong, Visible: bool): Insert the supplied audit entry. - read_page(Tenant: string, AfterTime: ulong, AfterId: ulong, Limit: ulong): Reject limits outside 1..=10 with 'invalid limit', preserving the previous page. Otherwise replace PageEntry with matching entries strictly after the cursor, sorted by the pair, at most limit rows. Positions start at 0 and are consecutive. Empty pages clear old results. Audit rows are never changed. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt index daf77a2b877..e6f78579bf8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/rust.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_entry(id: u64, tenant: String, occurred_at: u64, visible: bool): Insert the supplied audit entry. - read_page(tenant: String, after_time: u64, after_id: u64, limit: u64): Reject limits outside 1..=10 with 'invalid limit', preserving the previous page. Otherwise replace page_entry with matching entries strictly after the cursor, sorted by the pair, at most limit rows. Positions start at 0 and are consecutive. Empty pages clear old results. Audit rows are never changed. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt index 89b69b174b1..7936c85a041 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_084_stable_pagination/tasks/typescript.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - add_entry(id: u64 (bigint), tenant: string, occurredAt: u64 (bigint), visible: bool): Insert the supplied audit entry. - read_page(tenant: string, afterTime: u64 (bigint), afterId: u64 (bigint), limit: u64 (bigint)): Reject limits outside 1..=10 with 'invalid limit', preserving the previous page. Otherwise replace page_entry with matching entries strictly after the cursor, sorted by the pair, at most limit rows. Positions start at 0 and are consecutive. Empty pages clear old results. Audit rows are never changed. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt index 4160b1f737a..fad6c482a06 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/csharp.txt @@ -14,4 +14,4 @@ Reducers (argument order is part of the contract): - capture_batch(AfterId: ulong, Limit: ulong): Reject limit outside 1..=10 with 'invalid limit'. Replace BackfillSnapshot with at most limit legacy rows with id>after_id, ordered by id. Capture tombstones too. Do not modify source or destination. - apply_batch(): For each captured ID, reconcile ItemV2 with the CURRENT authoritative legacy row, not the stale captured contents. Delete destination tombstones; upsert current live rows. Leave snapshots intact so retries are safe. Do not touch IDs outside the captured batch. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt index 177c7a75b9c..8e0cb0834a4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/rust.txt @@ -14,4 +14,4 @@ Reducers (argument order is part of the contract): - capture_batch(after_id: u64, limit: u64): Reject limit outside 1..=10 with 'invalid limit'. Replace backfill_snapshot with at most limit legacy rows with id>after_id, ordered by id. Capture tombstones too. Do not modify source or destination. - apply_batch(): For each captured ID, reconcile item_v2 with the CURRENT authoritative legacy row, not the stale captured contents. Delete destination tombstones; upsert current live rows. Leave snapshots intact so retries are safe. Do not touch IDs outside the captured batch. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt index 2b86252bc69..42bbe669102 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_085_online_backfill/tasks/typescript.txt @@ -14,4 +14,4 @@ Reducers (argument order is part of the contract): - capture_batch(afterId: u64 (bigint), limit: u64 (bigint)): Reject limit outside 1..=10 with 'invalid limit'. Replace backfill_snapshot with at most limit legacy rows with id>after_id, ordered by id. Capture tombstones too. Do not modify source or destination. - apply_batch(): For each captured ID, reconcile item_v2 with the CURRENT authoritative legacy row, not the stale captured contents. Delete destination tombstones; upsert current live rows. Leave snapshots intact so retries are safe. Do not touch IDs outside the captured batch. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt index 53bcf78e13b..0b56a37b989 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/csharp.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - set_sale(Id: ulong, Category: string, Amount: long): Insert or replace a sale. Remove its old contribution and add its new contribution. Reject empty category with 'invalid category'. - remove_sale(Id: ulong): Delete a sale and its contribution. Missing IDs are a no-op. Remove a category only when its last sale is deleted. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt index 3fd70554a3a..fdb26f6f6c5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/rust.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - set_sale(id: u64, category: String, amount: i64): Insert or replace a sale. Remove its old contribution and add its new contribution. Reject empty category with 'invalid category'. - remove_sale(id: u64): Delete a sale and its contribution. Missing IDs are a no-op. Remove a category only when its last sale is deleted. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt index a094c936a44..8e8a6c0fe13 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_086_incremental_totals/tasks/typescript.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - set_sale(id: u64 (bigint), category: string, amount: i64 (bigint)): Insert or replace a sale. Remove its old contribution and add its new contribution. Reject empty category with 'invalid category'. - remove_sale(id: u64 (bigint)): Delete a sale and its contribution. Missing IDs are a no-op. Remove a category only when its last sale is deleted. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt index b015403cb17..323bcdf52e4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_088_lease_fencing/tasks/csharp.txt @@ -10,4 +10,4 @@ Reducers (argument order is part of the contract): - claim(Id: ulong, Worker: string, LeaseMs: ulong): Require nonempty worker and lease_ms in 1..=60000 ('invalid lease'), existing job ('missing job'), not done ('already done'), and unclaimed or expired ('lease busy'). Increment generation, set worker and ExpiresAt=server_now+lease_ms*1000. - complete(Id: ulong, Worker: string, Generation: ulong, Result: string): Require existing job ('missing job'). Exact replay of an already successful completion is a no-op, even after expiry. Any other completion of a done job fails ('already done'). For pending jobs, require nonempty matching worker, positive matching generation and server_now BenchmarkSpec { scorers.push(scenario(file!(), route, host, lang, |s| { let (alice, _) = s.new_user()?; let (bob, _) = s.new_user()?; - s.expect_call_as(&alice, "submit", json!(["retry", -1]), Some("invalid request"))?; - s.expect_call_as(&alice, "submit", json!(["retry", 4]), Some("quota exceeded"))?; + s.expect_rejected_as(&alice, "submit", json!(["retry", -1]), "invalid request")?; + s.expect_rejected_as(&alice, "submit", json!(["retry", 4]), "quota exceeded")?; s.rows("quota", &["used"], json!([]))?; s.rows("accepted_request", &["request_id"], json!([]))?; s.expect_call_as(&alice, "submit", json!(["retry", 2]), None)?; s.expect_call_as(&alice, "submit", json!(["retry", 2]), None)?; - s.expect_call_as(&bob, "submit", json!(["retry", 2]), Some("request conflict"))?; - s.expect_call_as(&alice, "submit", json!(["retry", 1]), Some("request conflict"))?; + s.expect_rejected_as(&bob, "submit", json!(["retry", 2]), "request conflict")?; + s.expect_rejected_as(&alice, "submit", json!(["retry", 1]), "request conflict")?; let results = std::thread::scope(|scope| { let a = scope.spawn(|| s.call_as(&alice, "submit", json!(["a", 1]))); let b = scope.spawn(|| s.call_as(&alice, "submit", json!(["b", 1]))); @@ -36,7 +36,7 @@ pub fn spec() -> BenchmarkSpec { ); s.expect_call_as(&bob, "submit", json!(["bob", 3]), None)?; s.rows("quota", &["used"], json!([[3], [3]]))?; - s.expect_call_as(&alice, "submit", json!(["overflow", 1]), Some("quota exceeded"))?; + s.expect_rejected_as(&alice, "submit", json!(["overflow", 1]), "quota exceeded")?; s.rows("quota", &["used"], json!([[3], [3]]))?; Ok(()) })); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/csharp.txt index 11ed4f9c33b..34e5a27e362 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/csharp.txt @@ -9,4 +9,4 @@ Tables (the first field is the primary key): Reducers (argument order is part of the contract): - submit(RequestId: string, Units: long): If request ID exists, identical owner+units returns success; otherwise fail 'request conflict'. For new IDs, require nonempty request ID and units>0 ('invalid request'). Require used+units<=3 ('quota exceeded'), starting at used=0. Insert request and update quota atomically. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt index ff89c27d8d9..5192a2ea0ad 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/rust.txt @@ -9,4 +9,4 @@ Tables (the first field is the primary key): Reducers (argument order is part of the contract): - submit(request_id: String, units: i64): If request ID exists, identical owner+units returns success; otherwise fail 'request conflict'. For new IDs, require nonempty request ID and units>0 ('invalid request'). Require used+units<=3 ('quota exceeded'), starting at used=0. Insert request and update quota atomically. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt index 7af9b80cf7a..5606c8ea595 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_090_transactional_quota/tasks/typescript.txt @@ -9,4 +9,4 @@ Tables (the first field is the primary key): Reducers (argument order is part of the contract): - submit(requestId: string, units: i64 (bigint)): If request ID exists, identical owner+units returns success; otherwise fail 'request conflict'. For new IDs, require nonempty request ID and units>0 ('invalid request'). Require used+units<=3 ('quota exceeded'), starting at used=0. Insert request and update quota atomically. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt index 3c027df5190..f71955b6c46 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/csharp.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - enqueue(Id: ulong, Input: long): Exact existing id+input is a no-op even after completion or failure. Changed input fails 'request conflict'. New input must be within -1000000..=1000000 ('invalid input'). Insert QueuedWork with status='queued', attempts=0, and one timer due after 1ms. WorkTimer uses an auto-increment primary key and schedules execute_work(job: WorkTimer). For a queued job only, increment attempts once; negative input sets status='failed' without an effect. Otherwise insert work_effect(id,input*2) and set status='complete' in the same transaction. Missing or already terminal jobs are no-ops. Do not delete QueuedWork rows. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt index f4d8b5956a2..0cd2131f220 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/rust.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - enqueue(id: u64, input: i64): Exact existing id+input is a no-op even after completion or failure. Changed input fails 'request conflict'. New input must be within -1000000..=1000000 ('invalid input'). Insert queued_work with status='queued', attempts=0, and one timer due after 1ms. work_timer uses an auto-increment primary key and schedules execute_work(job: WorkTimer). For a queued job only, increment attempts once; negative input sets status='failed' without an effect. Otherwise insert work_effect(id,input*2) and set status='complete' in the same transaction. Missing or already terminal jobs are no-ops. Do not delete queued_work rows. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt index 9ce21289ec4..16e4ee96b31 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_091_scheduled_queue/tasks/typescript.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - enqueue(id: u64 (bigint), input: i64 (bigint)): Exact existing id+input is a no-op even after completion or failure. Changed input fails 'request conflict'. New input must be within -1000000..=1000000 ('invalid input'). Insert queued_work with status='queued', attempts=0, and one timer due after 1ms. work_timer uses an auto-increment primary key and schedules execute_work(job: WorkTimer). For a queued job only, increment attempts once; negative input sets status='failed' without an effect. Otherwise insert work_effect(id,input*2) and set status='complete' in the same transaction. Missing or already terminal jobs are no-ops. Do not delete queued_work rows. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/spec.rs index 49e2ffee752..ee5bbd02ec6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/spec.rs @@ -12,8 +12,8 @@ pub fn spec() -> BenchmarkSpec { let (outsider, outsider_id) = s.new_user()?; s.expect_call_as(&owner, "set_document", json!(["first", "secret-one"]), None)?; s.rows_as(&reader, "visible_document", &["id", "title"], json!([]))?; - s.expect_call_as(&reader, "set_document", json!(["stolen", "leak"]), Some("owner only"))?; - s.expect_call_as(&reader, "set_access", json!([outsider_id, true]), Some("owner only"))?; + s.expect_rejected_as(&reader, "set_document", json!(["stolen", "leak"]), "owner only")?; + s.expect_rejected_as(&reader, "set_access", json!([outsider_id, true]), "owner only")?; let (code, _) = s.sql_as( &reader, &format!( diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt index ba5197c80d3..26f6f83d27c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/csharp.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - set_access(Reader: Identity, Enabled: bool): Require an existing document whose owner is sender ('owner only'). Insert or update the reader's permission. Disabling a reader does not remove the owner's own access. Define SafeDocument with Id: ulong and Title: string only. Public sender-aware view visible_document returns an array of SafeDocument: the single document when sender is owner or ReadAccess[sender].Enabled is true, otherwise empty. Use indexed lookups so permission changes are tracked as view dependencies. Do not add public source tables or diagnostic reducers. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt index e56311bf5d0..23246a4ca5c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/rust.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - set_access(reader: spacetimedb::Identity, enabled: bool): Require an existing document whose owner is sender ('owner only'). Insert or update the reader's permission. Disabling a reader does not remove the owner's own access. Define SafeDocument with id:u64 and title:string only. Public sender-aware view visible_document returns an array of SafeDocument: the single document when sender is owner or read_access[sender].enabled is true, otherwise empty. Use indexed lookups so permission changes are tracked as view dependencies. Do not add public source tables or diagnostic reducers. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt index 6b1b5730610..632b5af5fa1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/advanced/t_093_permission_revocation/tasks/typescript.txt @@ -11,4 +11,4 @@ Reducers (argument order is part of the contract): - set_access(reader: Identity, enabled: bool): Require an existing document whose owner is sender ('owner only'). Insert or update the reader's permission. Disabling a reader does not remove the owner's own access. Define SafeDocument with id:u64 and title:string only. Public sender-aware view visible_document returns an array of SafeDocument: the single document when sender is owner or read_access[sender].enabled is true, otherwise empty. Use indexed lookups so permission changes are tracked as view dependencies. Do not add public source tables or diagnostic reducers. -Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. When an operation fails with quoted text, the caller must receive that exact text. +Do not add an exercise reducer or seed data automatically. All operations must work for arbitrary inputs, not just one example. Failed operations must leave every table unchanged. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/csharp.txt index cf4dc7f4a85..490c1f22a5c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/csharp.txt @@ -12,4 +12,4 @@ TABLE REDUCERS - Tick: scheduled reducer triggered by TickTimer -- Init: insert exactly one row into TickTimer that schedules a repeating interval of 50_000 microseconds +- Init: when the module is first published, insert exactly one row into TickTimer that schedules a repeating interval of 50_000 microseconds diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/rust.txt index 20e75450ea9..b25e59ae24a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/rust.txt @@ -12,4 +12,4 @@ TABLE REDUCERS - tick: scheduled reducer triggered by tick_timer -- init: insert exactly one row into tick_timer that schedules a repeating interval of 50_000 microseconds +- init: when the module is first published, insert exactly one row into tick_timer that schedules a repeating interval of 50_000 microseconds diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/typescript.txt index 1b1c10ae5be..bbec68741e2 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/tasks/typescript.txt @@ -9,4 +9,4 @@ TABLE REDUCERS - tick: scheduled reducer that receives the tick_timer row -- init: insert initial scheduled row with 50ms interval \ No newline at end of file +- init: when the module is first published, insert initial scheduled row with 50ms interval \ No newline at end of file diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt index 35b5c561791..f79a73465c6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/csharp.txt @@ -12,4 +12,4 @@ TABLE REDUCERS - SendReminder: scheduled reducer triggered by Reminder (receives a Reminder row, does nothing) -- Init: inserts one row into Reminder, scheduling SendReminder to fire 60 seconds in the future using ScheduleAt.Time +- Init: when the module is first published, insert one row into Reminder, scheduling SendReminder to fire 60 seconds in the future using ScheduleAt.Time diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt index 91fab67e6f6..98dd8963e47 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/rust.txt @@ -12,4 +12,4 @@ TABLE REDUCERS - send_reminder: scheduled reducer triggered by reminder (receives a Reminder row, does nothing) -- init: inserts one row into reminder, scheduling send_reminder to fire 60 seconds in the future using ScheduleAt::Time +- init: when the module is first published, insert one row into reminder, scheduling send_reminder to fire 60 seconds in the future using ScheduleAt::Time diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt index 9551845afaa..a09844b8bb6 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/tasks/typescript.txt @@ -10,4 +10,4 @@ TABLE REDUCERS - send_reminder: scheduled reducer triggered by reminder (receives a reminder row argument, does nothing) -- init: inserts one row into reminder scheduling send_reminder to fire 60 seconds in the future using ScheduleAt.time +- init: when the module is first published, insert one row into reminder scheduling send_reminder to fire 60 seconds in the future using ScheduleAt.time diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt index faf93dbc809..6160bfa67c1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/csharp.txt @@ -11,5 +11,5 @@ TABLE REDUCERS - RunCleanup: scheduled reducer triggered by CleanupJob (receives a CleanupJob row, does nothing) -- Init: inserts one row into CleanupJob with a 60-second interval (60_000_000 microseconds) +- Init: when the module is first published, insert one row into CleanupJob with a 60-second interval (60_000_000 microseconds) - CancelCleanup(ctx, scheduledId: ulong): deletes the CleanupJob row with the given scheduledId, canceling the pending scheduled call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt index 9497de9be70..cbef1a3872c 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/rust.txt @@ -11,5 +11,5 @@ TABLE REDUCERS - run_cleanup: scheduled reducer triggered by cleanup_job (receives a CleanupJob row, does nothing) -- init: inserts one row into cleanup_job with a 60-second interval +- init: when the module is first published, insert one row into cleanup_job with a 60-second interval - cancel_cleanup(ctx, scheduled_id: u64): deletes the CleanupJob row with the given scheduled_id, canceling the pending scheduled call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt index ab0404bfd89..bea5c0cee3a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/tasks/typescript.txt @@ -9,5 +9,5 @@ TABLE REDUCERS - run_cleanup: scheduled reducer triggered by cleanup_job (receives a cleanup_job row argument, does nothing) -- init: inserts one row into cleanup_job with a 60-second interval (60_000_000n microseconds) +- init: when the module is first published, insert one row into cleanup_job with a 60-second interval (60_000_000n microseconds) - cancel_cleanup(ctx, { scheduledId: u64 }): deletes the cleanup_job row with the given scheduledId, canceling the pending scheduled call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/csharp.cs new file mode 100644 index 00000000000..867c4d3fa28 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/csharp.cs @@ -0,0 +1,23 @@ +using System; +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Item", Public = true)] + public partial struct Item + { + [PrimaryKey] + public ulong Id; + public string Name; + } + + [Reducer] + public static void AddItem(ReducerContext ctx, ulong id, string name) + { + if (name == "") + { + throw new Exception("name required"); + } + ctx.Db.Item.Insert(new Item { Id = id, Name = name }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/rust.rs new file mode 100644 index 00000000000..3ca4306de04 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/rust.rs @@ -0,0 +1,17 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = item, public)] +pub struct Item { + #[primary_key] + pub id: u64, + pub name: String, +} + +#[reducer] +pub fn add_item(ctx: &ReducerContext, id: u64, name: String) -> Result<(), String> { + if name.is_empty() { + return Err("name required".to_string()); + } + ctx.db.item().insert(Item { id, name }); + Ok(()) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/typescript.ts new file mode 100644 index 00000000000..5992dc78428 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/answers/typescript.ts @@ -0,0 +1,18 @@ +import { schema, SenderError, table, t } from 'spacetimedb/server'; + +const item = table( + { name: 'item', public: true }, + { id: t.u64().primaryKey(), name: t.string() } +); +const spacetimedb = schema({ item }); +export default spacetimedb; + +export const add_item = spacetimedb.reducer( + { id: t.u64(), name: t.string() }, + (ctx, { id, name }) => { + if (name === '') { + throw new SenderError('name required'); + } + ctx.db.item.insert({ id, name }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/spec.rs new file mode 100644 index 00000000000..9486a44b695 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/spec.rs @@ -0,0 +1,19 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::scenario::scenario; +use crate::eval::BenchmarkSpec; +use serde_json::json; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route, host| { + let mut scorers = default_schema_parity_scorers(host, file!(), route); + scorers.push(scenario(file!(), route, host, lang, |s| { + let (token, _) = s.new_user()?; + s.expect_call_as(&token, "add_item", json!([1, ""]), Some("name required"))?; + s.rows("item", &["id", "name"], json!([]))?; + s.expect_call_as(&token, "add_item", json!([1, "book"]), None)?; + s.rows("item", &["id", "name"], json!([[1, "book"]]))?; + Ok(()) + })); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/csharp.txt new file mode 100644 index 00000000000..2e2c8cce4ba --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/csharp.txt @@ -0,0 +1,12 @@ +Write a SpacetimeDB backend module in C# whose reducer reports a failure back to its caller. + +TABLE +- Item (public), struct Item + - Fields: + - Id: ulong (primary key) + - Name: string + +REDUCERS +- AddItem(ctx, ulong id, string name) + - If name is empty, fail so that the caller receives the exact text "name required" + - Otherwise insert the item diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/rust.txt new file mode 100644 index 00000000000..270bb0875b4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/rust.txt @@ -0,0 +1,12 @@ +Write a SpacetimeDB backend module in Rust whose reducer reports a failure back to its caller. + +TABLE +- item (public), struct Item + - Fields: + - id: u64 (primary key) + - name: String + +REDUCERS +- add_item(id: u64, name: String) + - If name is empty, fail so that the caller receives the exact text "name required" + - Otherwise insert the item diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/typescript.txt new file mode 100644 index 00000000000..f928daced35 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_102_caller_error_message/tasks/typescript.txt @@ -0,0 +1,12 @@ +Write a SpacetimeDB backend module in TypeScript whose reducer reports a failure back to its caller. + +TABLE +- item (public) + - Fields: + - id: bigint (u64, primary key) + - name: string + +REDUCERS +- add_item(ctx, { id: bigint (u64), name: string }) + - If name is empty, fail so that the caller receives the exact text "name required" + - Otherwise insert the item diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/csharp.txt index dc1e9d4996f..b98203b164a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/csharp.txt @@ -12,4 +12,4 @@ TABLE REDUCERS - Tick: scheduled reducer that accepts the scheduled row -- Init: insert exactly one row into TickTimer that schedules a repeating interval of 50_000 microseconds +- Init: when the module is first published, insert exactly one row into TickTimer that schedules a repeating interval of 50_000 microseconds diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/rust.txt index c3a76e8e734..8591d98995d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/rust.txt @@ -12,4 +12,4 @@ TABLE REDUCERS - tick: scheduled reducer that accepts the scheduled row -- init: insert exactly one row into tick_timer that schedules a repeating interval of 50_000 microseconds +- init: when the module is first published, insert exactly one row into tick_timer that schedules a repeating interval of 50_000 microseconds diff --git a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/typescript.txt index b81c53d841c..af5018a982d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/schema/t_017_scheduled_columns/tasks/typescript.txt @@ -9,4 +9,4 @@ TABLE REDUCERS - tick: scheduled reducer that receives the tick_timer row -- init: insert initial scheduled row with 50ms interval \ No newline at end of file +- init: when the module is first published, insert initial scheduled row with 50ms interval \ No newline at end of file diff --git a/tools/xtask-llm-benchmark/src/eval/scenario.rs b/tools/xtask-llm-benchmark/src/eval/scenario.rs index 6015fa77d3f..74cc12d8d04 100644 --- a/tools/xtask-llm-benchmark/src/eval/scenario.rs +++ b/tools/xtask-llm-benchmark/src/eval/scenario.rs @@ -303,10 +303,12 @@ impl Scenario { } } - /// Require the contract's error text. Transport errors and missing reducers must not pass. + /// A panic or plain exception counts as a rejection. Transport errors and missing reducers must not pass. pub fn reject(&self, reducer: &str, args: Value, expected: &str) -> Result<()> { match self.call_result(reducer, &args) { - Err(error) if error.contains("(530") && error.contains(expected) => Ok(()), + Err(error) if error.contains("(530") && (error.contains(expected) || error.contains("fatal error")) => { + Ok(()) + } actual => bail!("{reducer}({args}): expected error {expected:?}, got {actual:?}"), } } diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index 4b719902792..2d84eef6877 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -422,6 +422,12 @@ mod reducers_t_060_reducer_result_table { include!("../benchmarks/reducers/t_060_reducer_result_table/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_102_caller_error_message { + include!("../benchmarks/reducers/t_102_caller_error_message/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod schema_t_012_spacetime_product_type { @@ -653,6 +659,7 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("reducers", "t_058_batched_delete") => reducers_t_058_batched_delete::spec, ("reducers", "t_059_deterministic_context") => reducers_t_059_deterministic_context::spec, ("reducers", "t_060_reducer_result_table") => reducers_t_060_reducer_result_table::spec, + ("reducers", "t_102_caller_error_message") => reducers_t_102_caller_error_message::spec, ("schema", "t_012_spacetime_product_type") => schema_t_012_spacetime_product_type::spec, ("schema", "t_013_spacetime_sum_type") => schema_t_013_spacetime_sum_type::spec, ("schema", "t_014_elementary_columns") => schema_t_014_elementary_columns::spec, From 7af35110c56b4f1d2f510675fca50e93510b25b4 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 3 Nov 2026 21:34:47 -0500 Subject: [PATCH 26/41] Update the benchmark README's task count --- tools/xtask-llm-benchmark/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index b1fdd4eb960..567976ec6c4 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -1,6 +1,6 @@ # LLM benchmark validation -The normal suite contains 95 tasks. The 12 `advanced` tasks are included by default, +The normal suite contains 96 tasks. The 12 `advanced` tasks are included by default, in all three languages and in both context modes. They are also included in the normal website task catalog. `--categories advanced` is only a development filter. @@ -12,7 +12,7 @@ Generated code that fails compilation or grading remains a normal benchmark scor C# uses a fresh server per model batch. Each active task publishes its own reference database and deletes it after scoring or a provider failure. This bounds live C# -references by task concurrency, rather than loading all 95 before model generation. +references by task concurrency, rather than loading all 96 before model generation. `--goldens-only` also validates and deletes each C# reference/candidate pair before starting the next task. Rust and TypeScript keep their batched reference builds. Cleanup errors block the batch, but completed scoring remains in the local record. @@ -109,7 +109,7 @@ cargo run -p xtask-llm-benchmark --bin llm_benchmark --locked -- run --lang rust The command publishes disposable local databases and runs the real graders. It makes no model requests and uploads no scores or catalog. Omitting `--tasks` -validates all 95 reference tasks. Keep the weekly action disabled until these +validates all 96 reference tasks. Keep the weekly action disabled until these checks pass and the branch is reviewed. The scenario grader checks explicit expected rows, including intermediate state, From 836badbc4dd59a10d22379eb00ed38e5ea3e401e Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 3 Nov 2026 22:54:12 -0500 Subject: [PATCH 27/41] Say that t_023 defines no reducers Answers added a reducer so the view would have data, and the schema check failed them for it. --- .../src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt | 2 ++ .../src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt | 2 ++ .../benchmarks/queries/t_023_view_per_user/tasks/typescript.txt | 2 ++ 3 files changed, 6 insertions(+) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt index 96319f708cc..7a6389875ea 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/csharp.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in C# that defines a table of user profiles and a per-user view that returns only the calling user's profile. +The module defines no reducers. + TABLES - Profile - Struct: Profile diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt index 6c18f686c33..cb99edf8f49 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/rust.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines a table of user profiles and a per-user view that returns only the calling user's profile. +The module defines no reducers. + TABLES - profile - Struct: Profile diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt index 0376822cb40..3441cd59db9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/tasks/typescript.txt @@ -1,5 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a table of user profiles and a per-user view that returns only the calling user's profile. +The module defines no reducers. + TABLES - profile - public table From 0a8878e1b0028d1cf32819f0984f3c3c4becb4bb Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Wed, 4 Nov 2026 08:45:22 -0500 Subject: [PATCH 28/41] Score content rejections as failed tasks A provider rejecting the request now fails the task with all of its checks counted, and the batch still completes and uploads. Other provider errors still block the batch. This replaces the earlier rule that refusals have no score, and its test. --- tools/xtask-llm-benchmark/README.md | 12 +- .../xtask-llm-benchmark/src/bench/records.rs | 75 +++--- tools/xtask-llm-benchmark/src/bench/runner.rs | 236 +++++++++++++++++- tools/xtask-llm-benchmark/src/bench/types.rs | 6 + 4 files changed, 287 insertions(+), 42 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index 567976ec6c4..cb6df40870d 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -9,6 +9,8 @@ batch after retries. No partial scores from that batch are uploaded. Other model still run, but the language job exits with failure if any batch fails. Response errors include the generation ID and finish reason when supplied by the provider. Generated code that fails compilation or grading remains a normal benchmark score. +A provider rejecting the content also scores: the task fails all of its checks, with +no retry. The batch still completes and uploads. C# uses a fresh server per model batch. Each active task publishes its own reference database and deletes it after scoring or a provider failure. This bounds live C# @@ -40,11 +42,13 @@ status and retry counts remain in console logs. Raw refusal text, partial provid content, and HTTP error bodies stay in artifacts, not console errors. Treat these artifacts as source and provider-response data. -Refusals and provider or infrastructure errors have no code score. Summaries keep -task counts separate from scorer-check counts and always include coverage. For -example, 99 passing tasks and one refusal means **99/99 scored tasks passed, with -99/100 tasks scored and one refusal**. It does not mean a complete 100% benchmark. +Provider and infrastructure errors have no code score. Summaries keep task counts +separate from scorer-check counts and always include coverage. For example, 99 +passing tasks and one provider error means **99/99 scored tasks passed, with 99/100 +tasks scored and one provider error**. It does not mean a complete 100% benchmark. Zero scored tasks means no score, not 0%. An incomplete batch stays off the website. +A content rejection is a failed task, so 99 passing tasks and one rejection means +**99/100 tasks passed, with `refused_tasks` at 1**. Model mistakes have no fixed list of messages, so any candidate publish failure scores normally unless it matches a short list of environment errors, such as a full disk or a refused connection, or the server fails a health check afterward. diff --git a/tools/xtask-llm-benchmark/src/bench/records.rs b/tools/xtask-llm-benchmark/src/bench/records.rs index 643d9a51152..31c8ce06edc 100644 --- a/tools/xtask-llm-benchmark/src/bench/records.rs +++ b/tools/xtask-llm-benchmark/src/bench/records.rs @@ -19,7 +19,7 @@ pub(super) struct BatchRecord { enum TaskState { Pending, Scored { passed: u32, total: u32 }, - Refused, + Refused { total: u32 }, ProviderError, InfrastructureError, } @@ -76,18 +76,21 @@ impl BatchRecord { }, json!({"status": "scored", "outcome": outcome}), ), - Err(RunOneError::Other(error)) => match error.downcast_ref::() { - Some(refusal) => ( - TaskState::Refused, - json!({"status": "refused", "refusal": refusal, "error": format!("{error:#}")}), - ), - None => ( - TaskState::ProviderError, - json!({"status": "provider_or_setup_error", "error": format!("{error:#}"), - "provider_response": error.downcast_ref::(), - "http_response": error.downcast_ref::()}), - ), - }, + Err(RunOneError::Rejected { outcome, error }) => ( + TaskState::Refused { + total: outcome.total_tests, + }, + json!({"status": "refused", "outcome": outcome, "error": format!("{error:#}"), + "refusal": error.downcast_ref::(), + "provider_response": error.downcast_ref::(), + "http_response": error.downcast_ref::()}), + ), + Err(RunOneError::Other(error)) => ( + TaskState::ProviderError, + json!({"status": "provider_or_setup_error", "error": format!("{error:#}"), + "provider_response": error.downcast_ref::(), + "http_response": error.downcast_ref::()}), + ), Err(RunOneError::Infrastructure { msg, llm_output, @@ -110,7 +113,7 @@ impl BatchRecord { if !self .tasks .values() - .all(|state| matches!(state, TaskState::Scored { .. })) + .all(|state| matches!(state, TaskState::Scored { .. } | TaskState::Refused { .. })) { bail!("incomplete batch record; refusing to upload"); } @@ -123,7 +126,11 @@ impl BatchRecord { for state in self.tasks.values() { match state { TaskState::Pending => pending += 1, - TaskState::Refused => refused += 1, + TaskState::Refused { total } => { + scored += 1; + refused += 1; + scored_checks += u64::from(*total); + } TaskState::ProviderError => provider += 1, TaskState::InfrastructureError => infrastructure += 1, TaskState::Scored { passed: p, total } => { @@ -178,38 +185,48 @@ mod tests { )); let mut record = BatchRecord::create(&root, json!({"model": "test"}), vec!["a".into(), "b".into()]).unwrap(); record.record("a", &Ok(outcome(3))).unwrap(); - let response: OACompatResp = serde_json::from_value(json!({ - "id": "gen-test", "provider": "test-provider", "choices": [{ - "finish_reason": "content_filter", "native_finish_reason": "refusal", - "message": {"content": null, "refusal": "full refusal reason"} - }] - })) - .unwrap(); record - .record("b", &Err(RunOneError::Other(response.first_text().unwrap_err()))) + .record("b", &Err(RunOneError::Other(anyhow::anyhow!("402 Payment Required")))) .unwrap(); assert!(record.complete().is_err()); let rows = read_rows(&record.path); let last = rows.last().unwrap(); assert_eq!(last["summary"]["scored_tasks"], 1); assert_eq!(last["summary"]["passed_tasks"], 1); - assert_eq!(last["summary"]["refused_tasks"], 1); + assert_eq!(last["summary"]["provider_or_setup_errors"], 1); assert_eq!(last["summary"]["passed_checks"], 3); assert_eq!(last["summary"]["scored_checks"], 3); assert_eq!(last["complete"], false); - assert_eq!(last["attempt"]["refusal"]["refusal"], "full refusal reason"); assert_eq!(rows[1]["attempt"]["outcome"]["llm_output"], "generated code"); - // A generated-code failure is a scored result, unlike a provider refusal. - record.record("b", &Ok(outcome(0))).unwrap(); + // A rejection counts every check of the task as failed and completes the batch. + let response: OACompatResp = serde_json::from_value(json!({ + "id": "gen-test", "provider": "test-provider", "choices": [{ + "finish_reason": "content_filter", "native_finish_reason": "refusal", + "message": {"content": null, "refusal": "full refusal reason"} + }] + })) + .unwrap(); + record + .record( + "b", + &Err(RunOneError::Rejected { + outcome: Box::new(outcome(0)), + error: response.first_text().unwrap_err(), + }), + ) + .unwrap(); record.complete().unwrap(); let rows = read_rows(&record.path); let last = rows.last().unwrap(); assert_eq!(last["complete"], true); assert_eq!(last["summary"]["scored_tasks"], 2); assert_eq!(last["summary"]["passed_tasks"], 1); - assert_eq!(last["summary"]["refused_tasks"], 0); + assert_eq!(last["summary"]["refused_tasks"], 1); + assert_eq!(last["summary"]["passed_checks"], 3); assert_eq!(last["summary"]["scored_checks"], 6); - assert_eq!(rows[2]["attempt"]["refusal"]["generation_id"], "gen-test"); + assert_eq!(rows[3]["attempt"]["status"], "refused"); + assert_eq!(rows[3]["attempt"]["refusal"]["refusal"], "full refusal reason"); + assert_eq!(rows[3]["attempt"]["refusal"]["generation_id"], "gen-test"); drop(record); std::fs::remove_dir_all(root).unwrap(); } diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index db40f08e53e..b4be1e5f27a 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -240,6 +240,50 @@ fn provider_retry_delay(error: &anyhow::Error) -> Option { } } +// A provider declining the request counts against the model. Any other provider error blocks the batch. +fn is_content_rejection(error: &anyhow::Error) -> bool { + use crate::llm::clients::{ + http::HttpResponseError, + oa_compat::{ProviderRefusal, ProviderResponseError}, + }; + const SIGNATURES: [&str; 10] = [ + "flagged_input", + "requires moderation", + "input was flagged", + "flagged as potentially", + "content_policy", + "content policy", + "content_filter", + "content management policy", + "safety system", + "prohibited_content", + ]; + if error.downcast_ref::().is_some() { + return true; + } + if provider_retry_delay(error).is_some() { + return false; + } + let text = if let Some(response) = error.downcast_ref::() { + response.body.to_ascii_lowercase() + } else if let Some(response) = error.downcast_ref::() { + format!( + "{} {}", + response + .error + .code + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), + response.error.message.as_deref().unwrap_or_default() + ) + .to_ascii_lowercase() + } else { + return false; + }; + SIGNATURES.iter().any(|signature| text.contains(signature)) +} + // Model mistakes have no fixed list of messages, so a candidate failure scores unless it matches one of these. fn is_environment_failure(message: &str) -> bool { const SIGNATURES: [&str; 12] = [ @@ -442,6 +486,11 @@ impl TaskRunner { ); tokio::time::sleep(std::time::Duration::from_secs(delay)).await; last_err = e; + } else if is_content_rejection(&e) { + return Err(RunOneError::Rejected { + outcome: Box::new(build_rejected_outcome(task, cfg, total_tasks, &e, started)), + error: e, + }); } else { return Err(RunOneError::Other(e)); } @@ -688,6 +737,12 @@ fn finish_reference_cleanup( provider_error: Some(error), completed_outcome: None, }), + Err(RunOneError::Rejected { outcome, error }) => Err(RunOneError::Infrastructure { + msg: cleanup, + llm_output: None, + provider_error: Some(error), + completed_outcome: Some(outcome), + }), Err(RunOneError::Infrastructure { msg, llm_output, @@ -748,6 +803,10 @@ fn partition_results( .unwrap_or_default(); bail!("benchmark infrastructure failure: {msg}{output_note}"); } + Err(RunOneError::Rejected { outcome, error }) => { + eprintln!("\u{26a0}\u{fe0f} provider rejected the request, scoring as a failure: {error:?}"); + good.push(*outcome); + } Err(RunOneError::Other(e)) => { // No output at all — provider error, retry eprintln!("\u{26a0}\u{fe0f} provider error, will retry: {e:?}"); @@ -1550,6 +1609,34 @@ fn build_fail_outcome( } } +fn build_rejected_outcome( + task: &TaskPaths, + cfg: &RunContext<'_>, + total_tests: usize, + error: &anyhow::Error, + started: chrono::DateTime, +) -> RunOutcome { + let details = ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({"phase": "generation", "error": format!("{error:#}")}), + }; + RunOutcome { + total_tests: total_tests as u32, + llm_output: None, + scorer_details: Some([("content_rejection".to_string(), details)].into_iter().collect()), + started_at: Some(started), + ..build_fail_outcome( + task, + cfg.lang_name, + cfg.route, + cfg.hash, + anyhow!("unused"), + String::new(), + ) + } +} + fn read_dirs(p: &Path) -> Result> { let mut v = Vec::new(); for e in fs::read_dir(p).with_context(|| format!("read_dir {}", p.display()))? { @@ -1822,6 +1909,114 @@ mod build_cleanup_tests { } } + fn refusal_error() -> anyhow::Error { + serde_json::from_value::(json!({ + "choices": [{"finish_reason": "content_filter", "message": {"content": null, "refusal": "private detail"}}] + })) + .unwrap() + .first_text() + .unwrap_err() + } + + #[test] + fn only_content_rejections_count_against_the_model() { + use crate::llm::clients::http::HttpResponseError; + let http = |status, body: &str| -> anyhow::Error { + HttpResponseError { + status, + body: body.to_string(), + } + .into() + }; + let body_error = |error| -> anyhow::Error { + serde_json::from_value::(json!({"error": error})) + .unwrap() + .first_text() + .unwrap_err() + }; + assert!(is_content_rejection(&refusal_error())); + assert!(is_content_rejection(&http( + 403, + r#"{"error":{"message":"m requires moderation on p. Your input was flagged"}}"# + ))); + assert!(is_content_rejection(&http( + 400, + r#"{"error":{"code":"content_policy_violation"}}"# + ))); + assert!(is_content_rejection(&body_error( + json!({"code": 403, "message": "Your input was flagged for sexual"}) + ))); + assert!(is_content_rejection(&http( + 403, + r#"{"error":{"metadata":{"flagged_input":"x"}}}"# + ))); + assert!(is_content_rejection(&body_error(json!({"code": "content_filter"})))); + assert!(is_content_rejection(&refusal_error().context("provider call"))); + for error in [ + http(402, "insufficient credits"), + http(403, "full HTTP refusal details"), + http(429, "content_filter service is rate limited"), + http(503, "safety system unavailable"), + body_error(json!({"code": 403, "message": "key limit exceeded"})), + anyhow!("no nonempty completion content"), + anyhow!("content_filter appears in a plain error"), + ] { + assert!(!is_content_rejection(&error), "{error:#}"); + } + } + + #[tokio::test] + async fn content_rejection_fails_every_check_of_the_task() { + struct RefusingProvider; + #[async_trait::async_trait] + impl crate::llm::LlmProvider for RefusingProvider { + async fn preflight_route(&self, _: &ModelRoute, _: bool) -> Result<()> { + Ok(()) + } + async fn generate( + &self, + _: &ModelRoute, + _: &crate::llm::prompt::BuiltPrompt, + ) -> Result { + Err(refusal_error()) + } + } + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("src/benchmarks"); + let task = discover_tasks(&root) + .unwrap() + .into_iter() + .find(|task| task.root.ends_with("basics/t_004_insert")) + .unwrap(); + let route = ModelRoute::new("Test", crate::llm::types::Vendor::OpenAi, "test", None); + let cfg = RunContext { + lang_name: "rust", + lang: Lang::Rust, + mode: "no_context", + route: &route, + context: "", + hash: "test", + llm: &RefusingProvider, + host: Some("http://127.0.0.1:1".into()), + }; + let runner = TaskRunner::new(root, SpacetimeRustPublisher, DotnetPublisher, TypeScriptPublisher); + let result = runner.run_one(&task, &cfg).await; + let expected = resolve_by_path(&task.root).unwrap()() + .scorers_for(Lang::Rust, "test", "local") + .len() as u32; + assert!(expected > 1, "the fixture must show the full denominator"); + let (outcomes, pending) = partition_results(vec![(task, result)], "rust", &route, "test").unwrap(); + assert!(pending.is_empty(), "a rejection must not block the batch"); + assert_eq!(outcomes[0].passed_tests, 0); + assert_eq!(outcomes[0].total_tests, expected); + assert_eq!(outcomes[0].llm_output, None); + let notes = serde_json::to_string(&outcomes[0].scorer_details).unwrap(); + assert!(notes.contains("content_rejection")); + assert!( + !notes.contains("private detail"), + "raw refusal text must not reach uploads" + ); + } + #[test] fn provider_failures_reject_the_whole_batch_but_model_failures_remain_scores() { let task = || TaskPaths { @@ -1841,19 +2036,28 @@ mod build_cleanup_tests { }), ), (task(), Err(RunOneError::Other(anyhow::anyhow!("402 Payment Required")))), - (task(), Err(RunOneError::Other( - serde_json::from_value::(json!({ - "choices": [{"finish_reason": "content_filter", "message": {"content": null, "refusal": "provider refusal"}}] - })).unwrap().first_text().unwrap_err() - ))), + ( + task(), + Err(RunOneError::Rejected { + outcome: Box::new(build_fail_outcome( + &task(), + "rust", + &route, + "test", + anyhow!("rejected"), + String::new(), + )), + error: refusal_error(), + }), + ), ], "rust", &route, "test", ) .unwrap(); - assert_eq!(outcomes.len(), 1); - assert_eq!(pending.len(), 2, "refusals and provider failures are not scored zeroes"); + assert_eq!(outcomes.len(), 2); + assert_eq!(pending.len(), 1, "provider failures are not scored zeroes"); assert_eq!(outcomes[0].passed_tests, 0); assert!(finalize_batch(outcomes.clone(), Vec::new()).is_ok()); let error = finalize_batch(outcomes, pending).unwrap_err().to_string(); @@ -1924,10 +2128,24 @@ mod build_cleanup_tests { .unwrap() .first_text() .unwrap_err(); - match finish_reference_cleanup(Err(RunOneError::Other(refusal)), Err(anyhow!("delete failed"))) { + let rejected = RunOneError::Rejected { + outcome: Box::new( + serde_json::from_value(json!({ + "hash": "test", "task": "a", "lang": "csharp", "model_name": "test", "vendor": "test", + "golden_published": false, "passed_tests": 0, "total_tests": 2 + })) + .unwrap(), + ), + error: refusal, + }; + match finish_reference_cleanup(Err(rejected), Err(anyhow!("delete failed"))) { Err(RunOneError::Infrastructure { - msg, provider_error, .. + msg, + provider_error, + completed_outcome, + .. }) => { + assert!(completed_outcome.is_some()); assert!(msg.contains("delete failed")); assert!(!msg.contains("private detail")); let error = provider_error.unwrap(); diff --git a/tools/xtask-llm-benchmark/src/bench/types.rs b/tools/xtask-llm-benchmark/src/bench/types.rs index 3359f041186..44f3fc2b0ed 100644 --- a/tools/xtask-llm-benchmark/src/bench/types.rs +++ b/tools/xtask-llm-benchmark/src/bench/types.rs @@ -143,6 +143,12 @@ pub enum RunOneError { provider_error: Option, completed_outcome: Option>, }, + // Scores as a failure. Provider details stay in local evidence, not console messages. + #[error("{error:#}")] + Rejected { + outcome: Box, + error: anyhow::Error, + }, #[error(transparent)] Other(#[from] anyhow::Error), } From 6ef60e4079d1394d98f11ccb66f057cd9841a19b Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Wed, 4 Nov 2026 09:04:23 -0500 Subject: [PATCH 29/41] Remove the added caller sentences from t_074 and t_079 No check needs them, because the URL is passed as an argument and the prompts already say to use the supplied URL. They revealed the URLs the checks use, and they did not stop provider rejections. --- .../procedures/t_074_fetch_and_store/tasks/csharp.txt | 2 -- .../benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt | 2 -- .../procedures/t_074_fetch_and_store/tasks/typescript.txt | 2 -- .../procedures/t_079_external_upload_flow/tasks/csharp.txt | 2 -- .../procedures/t_079_external_upload_flow/tasks/rust.txt | 2 -- .../procedures/t_079_external_upload_flow/tasks/typescript.txt | 2 -- 6 files changed, 12 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt index 31d5a74ebbd..4ffc184a0c8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt @@ -1,7 +1,5 @@ Write a SpacetimeDB backend module in C# that fetches external data before storing a result in a transaction. -The caller supplies a public example-page URL, such as https://example.com. Fetch that page, then store the response metadata. - TABLE - FetchedRecord (public, accessor FetchedRecord) - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt index 0baada33970..806a41fb72a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt @@ -1,7 +1,5 @@ Write a SpacetimeDB backend module in Rust that fetches external data before storing a result in a transaction. -The caller supplies a public example-page URL, such as https://example.com. Fetch that page, then store the response metadata. - TABLE - fetched_record (public), struct FetchedRecord - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt index b2bf4f40170..4aa08a1b2ae 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt @@ -1,7 +1,5 @@ Write a SpacetimeDB backend module in TypeScript that fetches external data before storing a result in a transaction. -The caller supplies a public example-page URL, such as https://example.com. Fetch that page, then store the response metadata. - TABLE - fetched_record (public) - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt index 84443a7d58b..5817b13ce75 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt @@ -1,7 +1,5 @@ Write a SpacetimeDB backend module in C# that exposes an upload route and an external upload procedure. -The caller supplies sample bytes and a public HTTP echo endpoint, such as https://postman-echo.com/post. Send only those supplied bytes, then store the response metadata. - TABLE - UploadedAsset (public, accessor UploadedAsset) - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt index aaa89d38223..f528ede25ac 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt @@ -1,7 +1,5 @@ Write a SpacetimeDB backend module in Rust that exposes an upload route and an external upload procedure. -The caller supplies sample bytes and a public HTTP echo endpoint, such as https://postman-echo.com/post. Send only those supplied bytes, then store the response metadata. - TABLE - uploaded_asset (public), struct UploadedAsset - Fields: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt index 057328fad14..3f06b4e2dcb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt @@ -1,7 +1,5 @@ Write a SpacetimeDB backend module in TypeScript that exposes an upload route and an external upload procedure. -The caller supplies sample bytes and a public HTTP echo endpoint, such as https://postman-echo.com/post. Send only those supplied bytes, then store the response metadata. - TABLE - uploaded_asset (public) - Fields: From 6fb0ee21c257c25bc59ae279110e336b1aa74d22 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Wed, 4 Nov 2026 15:43:29 -0500 Subject: [PATCH 30/41] Stop failing C# tables that set Name without Accessor The C# SDK treats both options as optional and these modules publish and behave correctly, so a text match should not fail them. Each language is now graded by what its SDK accepts. This removes the check and its two tests. --- tools/xtask-llm-benchmark/README.md | 3 +- tools/xtask-llm-benchmark/src/eval/scorers.rs | 61 +------------------ 2 files changed, 3 insertions(+), 61 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index cb6df40870d..c886fc7b09a 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -24,8 +24,7 @@ Schema parity treats table and view names that differ only by case or underscore as the same, because prompts do not fix their casing. It matches lifecycle reducers by kind, not by name, because clients cannot call them. It accepts extra indexes and HTTP routes but fails a missing one. It ignores view primary keys except on the task that asks for -one. It fails any C# table that sets `Name` without `Accessor`, because that is the -1.x form. Behaviour checks +one. Behaviour checks query a table or view by the module's own spelling when it differs only by case or underscores. diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index c64dfcd1ef2..677cb66a5f6 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -89,7 +89,7 @@ impl Scorer for SchemaParityScorer { self.id_str } - fn score(&self, llm_output: &str) -> ScoreDetails { + fn score(&self, _llm_output: &str) -> ScoreDetails { let golden = match describe_db(&self.server, &self.golden_db, self.timeout) { Ok(v) => v, Err(e) => return err_details("describe_golden", e), @@ -123,12 +123,7 @@ impl Scorer for SchemaParityScorer { let reducers_diff = diff_sets(&schema_a.reducers, &schema_b.reducers); let exports_diff = diff_sets(&schema_a.exports, &schema_b.exports); let rls_diff = diff_sets(&schema_a.row_level_security, &schema_b.row_level_security); - let name_without_accessor = has_table_name_without_accessor(llm_output); - let pass = tables_diff.is_null() - && reducers_diff.is_null() - && exports_diff.is_null() - && rls_diff.is_null() - && !name_without_accessor; + let pass = tables_diff.is_null() && reducers_diff.is_null() && exports_diff.is_null() && rls_diff.is_null(); ScoreDetails { pass, @@ -137,7 +132,6 @@ impl Scorer for SchemaParityScorer { "server": self.server, "golden_db": self.golden_db, "llm_db": self.llm_db, - "table_name_without_accessor": name_without_accessor, "tables_equal": tables_diff.is_null(), "reducers_equal": reducers_diff.is_null(), "exports_equal": exports_diff.is_null(), @@ -615,23 +609,6 @@ fn accept_extra_indexes( candidate } -// shortcut: text match for the 1.x C# form, which the schema cannot show; delete once the C# SDK rejects it as Rust does. -fn has_table_name_without_accessor(source: &str) -> bool { - static COMMENT: LazyLock = LazyLock::new(|| Regex::new(r"(?s)/\*.*?\*/|//[^\n]*").unwrap()); - static TABLE: LazyLock = LazyLock::new(|| { - Regex::new(r"[\[,]\s*(?:global::)?(?:SpacetimeDB\.)?Table\s*\(([^()]*(?:\([^()]*\)[^()]*)*)\)").unwrap() - }); - - TABLE.captures_iter(&COMMENT.replace_all(source, "")).any(|table| { - let sets = |property: &str| { - table[1] - .split(',') - .any(|argument| argument.split('=').next().is_some_and(|key| key.trim() == property)) - }; - sets("Name") && !sets("Accessor") - }) -} - fn diff_maps(a: &BTreeMap>, b: &BTreeMap>) -> Value { let mut only_a = BTreeMap::new(); let mut only_b = BTreeMap::new(); @@ -1807,40 +1784,6 @@ mod tests { assert!(diff_maps(&golden, &candidate)["changed"]["child_item"].is_object()); } - #[test] - fn table_name_without_accessor_is_detected() { - for source in [ - r#"[Table(Name = "User", Public = true)]"#, - r#"[SpacetimeDB.Table(Name = "user")]"#, - r#"[global::SpacetimeDB.Table(Public = true, Name = "User")]"#, - r#"[Table(Name = "TickTimer", Scheduled = nameof(Tick))]"#, - "[Table(\n Name = \"Reminder\",\n Scheduled = nameof(SendReminder),\n ScheduledAt = nameof(Reminder.ScheduledAt))]", - "[Table(Accessor = \"A\")]\n[Table(Name = \"B\")]", - r#"[Table(Accessor = "A"), Table(Name = "B")]"#, - ] { - assert!(has_table_name_without_accessor(source), "{source}"); - } - } - - #[test] - fn valid_table_declarations_are_not_flagged() { - for source in [ - "[Table]", - "[Table(Public = true)]", - r#"[Table(Accessor = "User", Public = true)]"#, - r#"[SpacetimeDB.Table(Name = "Parent", Public = true, Accessor = "Parent")]"#, - r#"[Table(Accessor = "TickTimer", Scheduled = nameof(Tick))]"#, - r#"[SpacetimeDB.Index.BTree(Name = "ByOwner", Columns = new[] { nameof(Owner) })]"#, - r#"[View(Name = "ActiveUsers")]"#, - r#"// [Table(Name = "User")]"#, - "/* [Table(Name = \"User\")] */", - r#"#[table(accessor = user, name = "User")]"#, - r#"const user = table({ name: 'User' }, {});"#, - ] { - assert!(!has_table_name_without_accessor(source), "{source}"); - } - } - #[test] fn missing_default_value_produces_a_schema_diff() { let mut golden = current_schema(true); From 77f52325760585ceb70f0d8f7c43191152092991 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Wed, 4 Nov 2026 21:06:11 -0500 Subject: [PATCH 31/41] Give Grok the longer LLM call timeout --- tools/xtask-llm-benchmark/src/bench/runner.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index b4be1e5f27a..7f03a6734f1 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -458,9 +458,9 @@ impl TaskRunner { let mut gen_start = Instant::now(); let llm_result = { const MAX_ATTEMPTS: u32 = 3; - // Slow models (Gemini 3.1 Pro, DeepSeek Reasoner) can take 8+ minutes on large contexts. + // Slow models (Gemini 3.1 Pro, DeepSeek Reasoner, Grok 4.7) can take 8+ minutes on large contexts. let timeout_secs = match cfg.route.display_name.to_ascii_lowercase() { - n if n.contains("gemini") || n.contains("deepseek") => 600, + n if n.contains("gemini") || n.contains("deepseek") || n.contains("grok") => 600, _ => 300, }; let mut last_err: anyhow::Error = anyhow!("no attempts made"); From 63d2908ed2e83abba9b87076c4b3d69fe5924128 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Wed, 4 Nov 2026 22:40:00 -0500 Subject: [PATCH 32/41] Give Grok a 20 minute LLM call timeout --- tools/xtask-llm-benchmark/src/bench/runner.rs | 3 ++- tools/xtask-llm-benchmark/src/llm/clients/http.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 7f03a6734f1..9745bf0ff66 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -460,7 +460,8 @@ impl TaskRunner { const MAX_ATTEMPTS: u32 = 3; // Slow models (Gemini 3.1 Pro, DeepSeek Reasoner, Grok 4.7) can take 8+ minutes on large contexts. let timeout_secs = match cfg.route.display_name.to_ascii_lowercase() { - n if n.contains("gemini") || n.contains("deepseek") || n.contains("grok") => 600, + n if n.contains("grok") => 1200, + n if n.contains("gemini") || n.contains("deepseek") => 600, _ => 300, }; let mut last_err: anyhow::Error = anyhow!("no attempts made"); diff --git a/tools/xtask-llm-benchmark/src/llm/clients/http.rs b/tools/xtask-llm-benchmark/src/llm/clients/http.rs index f5a9eb59a6c..dcad9ae0dae 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/http.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/http.rs @@ -75,7 +75,7 @@ impl HttpClient { pub fn new() -> Result { let inner = Client::builder() .user_agent("xtask-llm-benchmark/0.1") - .timeout(Duration::from_secs(660)) // longer than any per-model Tokio timeout + .timeout(Duration::from_secs(1260)) // longer than any per-model Tokio timeout .pool_idle_timeout(Duration::from_secs(30)) .build() .context("build reqwest client")?; From e085f0c3790ad38c84d5c12072c1196cb67c8805 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Thu, 5 Nov 2026 07:49:20 -0500 Subject: [PATCH 33/41] Count a blank reply at the output limit as a failed task --- tools/xtask-llm-benchmark/README.md | 3 +- tools/xtask-llm-benchmark/src/bench/runner.rs | 3 +- .../src/llm/clients/oa_compat.rs | 36 ++++++++++++++++++- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index c886fc7b09a..023b0d45533 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -47,7 +47,8 @@ passing tasks and one provider error means **99/99 scored tasks passed, with 99/ tasks scored and one provider error**. It does not mean a complete 100% benchmark. Zero scored tasks means no score, not 0%. An incomplete batch stays off the website. A content rejection is a failed task, so 99 passing tasks and one rejection means -**99/100 tasks passed, with `refused_tasks` at 1**. +**99/100 tasks passed, with `refused_tasks` at 1**. A blank reply at the output limit +counts the same way. Model mistakes have no fixed list of messages, so any candidate publish failure scores normally unless it matches a short list of environment errors, such as a full disk or a refused connection, or the server fails a health check afterward. diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 9745bf0ff66..8a3fc759bf9 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -240,7 +240,8 @@ fn provider_retry_delay(error: &anyhow::Error) -> Option { } } -// A provider declining the request counts against the model. Any other provider error blocks the batch. +// A provider declining the request, or a blank reply at the output limit, counts against the model. +// Any other provider error blocks the batch. fn is_content_rejection(error: &anyhow::Error) -> bool { use crate::llm::clients::{ http::HttpResponseError, diff --git a/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs b/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs index 0e64459ccdc..fd5d72958f7 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs @@ -145,7 +145,14 @@ impl OACompatResp { || choice .message .as_ref() - .is_some_and(|m| m.refusal.as_deref().is_some_and(|text| !text.trim().is_empty()))) + .is_some_and(|m| m.refusal.as_deref().is_some_and(|text| !text.trim().is_empty())) + // Hitting the output limit before writing anything counts against the model too. + || (finish == Some("length") + && choice + .message + .as_ref() + .and_then(|m| m.content.as_deref()) + .is_none_or(|text| text.trim().is_empty()))) { return Err(ProviderRefusal { generation_id: self.id.clone(), @@ -189,6 +196,33 @@ mod tests { assert_eq!(response.first_text().unwrap(), "valid code"); } + #[test] + fn only_a_blank_reply_at_the_output_limit_counts_as_a_refusal() { + let response = |finish: &str, content: Option<&str>| -> OACompatResp { + serde_json::from_value(serde_json::json!({ + "choices": [{"finish_reason": finish, "message": {"content": content}}] + })) + .unwrap() + }; + for content in [ + None, + Some(""), + Some( + " +", + ), + ] { + let error = response("length", content).first_text().unwrap_err(); + assert!(error.downcast_ref::().is_some(), "{error:#}"); + } + assert_eq!( + response("length", Some("partial code")).first_text().unwrap(), + "partial code" + ); + let error = response("stop", None).first_text().unwrap_err(); + assert!(error.downcast_ref::().is_none(), "{error:#}"); + } + #[test] fn refusals_preserve_details_without_logging_text_or_scoring_partial_code() { for (finish, native, refusal) in [ From 815d6c526bbc083cc79f7e1595b40e987f475f4e Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Thu, 5 Nov 2026 07:53:33 -0500 Subject: [PATCH 34/41] Retry a blank reply before counting it as a failed task --- tools/xtask-llm-benchmark/README.md | 2 +- tools/xtask-llm-benchmark/src/bench/runner.rs | 14 +++++++++++--- .../src/llm/clients/oa_compat.rs | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index 023b0d45533..8dcfab45e16 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -48,7 +48,7 @@ tasks scored and one provider error**. It does not mean a complete 100% benchmar Zero scored tasks means no score, not 0%. An incomplete batch stays off the website. A content rejection is a failed task, so 99 passing tasks and one rejection means **99/100 tasks passed, with `refused_tasks` at 1**. A blank reply at the output limit -counts the same way. +is retried first, then counts the same way. Model mistakes have no fixed list of messages, so any candidate publish failure scores normally unless it matches a short list of environment errors, such as a full disk or a refused connection, or the server fails a health check afterward. diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 8a3fc759bf9..f6b19428cf6 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -208,8 +208,9 @@ fn provider_retry_delay(error: &anyhow::Error) -> Option { 502..=504 => Some(30), _ => None, }; - if error.downcast_ref::().is_some() { - return None; + if let Some(refusal) = error.downcast_ref::() { + // A blank reply at the output limit can succeed on another try. A real refusal is final. + return (refusal.finish_reason.as_deref() == Some("length")).then_some(30); } if let Some(response) = error.downcast_ref::() { return status_delay(u64::from(response.status)); @@ -240,7 +241,7 @@ fn provider_retry_delay(error: &anyhow::Error) -> Option { } } -// A provider declining the request, or a blank reply at the output limit, counts against the model. +// A provider declining the request, or a blank reply at the output limit after retries, counts against the model. // Any other provider error blocks the batch. fn is_content_rejection(error: &anyhow::Error) -> bool { use crate::llm::clients::{ @@ -1900,6 +1901,13 @@ mod build_cleanup_tests { })) .unwrap(); assert_eq!(provider_retry_delay(&refusal.first_text().unwrap_err()), None); + let blank: OACompatResp = serde_json::from_value(serde_json::json!({ + "id": "gen-blank", "choices": [{"finish_reason":"length", "message":{"content":null}}] + })) + .unwrap(); + let blank = blank.first_text().unwrap_err(); + assert_eq!(provider_retry_delay(&blank), Some(30)); + assert!(is_content_rejection(&blank)); for (status, expected) in [(429, Some(60)), (503, Some(30)), (402, None)] { let error = crate::llm::clients::http::HttpResponseError { status, diff --git a/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs b/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs index fd5d72958f7..5fcde08ae9f 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/oa_compat.rs @@ -146,7 +146,7 @@ impl OACompatResp { .message .as_ref() .is_some_and(|m| m.refusal.as_deref().is_some_and(|text| !text.trim().is_empty())) - // Hitting the output limit before writing anything counts against the model too. + // Hitting the output limit before writing anything counts against the model too, once retries run out. || (finish == Some("length") && choice .message From 7c9b31314061e48041449f5fcc6afd276f8445df Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Thu, 5 Nov 2026 09:55:40 -0500 Subject: [PATCH 35/41] Add rules for writing LLM benchmark tasks --- tools/xtask-llm-benchmark/AGENTS.md | 114 ++++++++++++++++++++++++++++ tools/xtask-llm-benchmark/CLAUDE.md | 1 + 2 files changed, 115 insertions(+) create mode 100644 tools/xtask-llm-benchmark/AGENTS.md create mode 100644 tools/xtask-llm-benchmark/CLAUDE.md diff --git a/tools/xtask-llm-benchmark/AGENTS.md b/tools/xtask-llm-benchmark/AGENTS.md new file mode 100644 index 00000000000..5d68c49e055 --- /dev/null +++ b/tools/xtask-llm-benchmark/AGENTS.md @@ -0,0 +1,114 @@ +# Benchmark tasks + +The goal is simple: valid code using the current API passes, and outdated or wrong code fails. A failure must measure the model, not our setup. + +Every prompt change alters the test for every model. Change as little as the fix needs, and never rewrite prompts for style. + +## Files + +A task lives in `src/benchmarks//t_NNN_name/`: + +- `spec.rs`: the graders +- `tasks/rust.txt`, `tasks/typescript.txt`, `tasks/csharp.txt`: the prompts +- `answers/rust.rs`, `answers/typescript.ts`, `answers/csharp.cs`: the reference answers + +The build regenerates `src/generated/registry.rs`, which is tracked, so commit it with a new task. Update the task count and the task table in `README.md`. + +## Choosing a task + +- Test one concept. Compare `advanced/t_087_webhook_ordering` for a hard task and `basics/t_004_insert` for a simple one. +- Test a SpacetimeDB contract, not another product's API. +- Use local fixtures for outside calls, never a paid service. +- Do not name other products in task files, the README or commit messages. + +## Prompts + +Before writing a prompt, read the prompts of the nearest tasks in the same category and copy their format. Most tasks use `TABLE` and `REDUCERS` sections. The advanced tasks use short paragraphs followed by a table list. + +**State requirements, not how to meet them.** + +- Say "the caller's identity", "the current time" and "fail with", not `ctx.sender`, a time method or `panic!`. Models are being tested on that syntax. +- Say when something runs, such as "when the module is first published", not which lifecycle API to use. +- Do not list the collections or iterator calls to use. +- Do not reveal values that the checks use, such as a test URL. + +**State everything the checks enforce.** A model cannot guess these: + +- Table and view names, field names, field types, and which tables are public +- Primary keys, auto-increment and indexes +- Reducer names, argument types and any required return type +- Exact text, whenever a check compares text +- That a module has no tables or no reducers, when an extra one would fail the schema check + +**Keep the three languages the same.** Only names and types differ. + +- Use each language's declared names in the prose too: `event_id` in Rust, `eventId` in TypeScript, `EventId` in C#. +- Use each language's types. Mark 64-bit TypeScript integers as bigint, as in `u64 (bigint)`. +- Check every name you write against the SDK. Earlier prompts named a C# method that does not exist. + +**Add nothing that no model needs.** + +- Never add a sentence just to get past a check or a content filter. If a valid answer fails, fix the grader. +- The runner already tells models to output one source file and to copy identifiers exactly. New prompts need not repeat this. +- Start with `Write a SpacetimeDB backend module in that ...`. Migration tasks and a few others vary, so follow the nearest task. + +## Reference answers + +- Write each answer the way the current guidelines in the root `skills/` directory teach it. +- Use the current API in all three languages. +- Include only what the prompt asks for. Extra reducers or tables make the schema check compare against the wrong thing. + +## Graders + +Start from `default_schema_parity_scorers`, then add a behaviour check. `scenario` in `src/eval/scenario.rs` is the usual tool. A schema check alone passes a module whose reducers do nothing. + +**A check must fail a module that does nothing.** + +- Assert the rows or replies you expect, including intermediate state. +- Confirm that setup rows exist before a delete, and that rows outside the delete survive. +- Assert explicit expected rows. Agreement between two copies of the reference proves little. +- Cover concurrency with real parallel calls where the task promises it, as `t_087` does. + +**A check must pass every design the prompt allows.** + +- Trace each assertion to a sentence in the prompt. If none exists, delete the assertion. State the requirement instead only when the task cannot test its concept without it. +- Accept any rejection unless the prompt quotes the error text. A plain panic or throw that rolls back is a valid rejection. +- Test error delivery to the caller in its own task, not as part of atomicity or lease tasks. +- Accept extra indexes and extra HTTP routes. Ignore case and underscores in names. The shared helpers already do this, so use them instead of hand-written SQL. +- Do not treat a transport error as an expected rejection. + +## Validate + +Check only the tasks you touched. A prompt-only change does not need a full reference run. + +1. Run the crate's tests, `cargo fmt` and clippy. +2. Run the reference answers for each language, after the local builds in `README.md`. This makes no model calls and uploads nothing. + + ```text + cargo run -p xtask-llm-benchmark --bin llm_benchmark --locked -- run --lang rust --goldens-only --dry-run --skip-task-catalog-upload --tasks t_NNN + ``` + +3. Prove the behaviour check can fail. Temporarily replace the answer with a version that does nothing, then with one that breaks a single requirement. Run the same command and confirm both fail. Restore the answer. The schema check compares an answer with itself here, so only the behaviour check can catch these. +4. Run real models without uploading. Use the `Periodic LLM benchmarks` workflow with `model_set=explicit`, one or two cheap models, the `tasks` input and `dry_run=true`. Enabling, dispatching and disabling that workflow needs the owner's approval. +5. Read the failures and sort them with the table below. Do not assume a failure is the model's fault. + +## Whose fault is a failure? + +| What happened | What to do | +| --- | --- | +| Valid current-API code fails | Our bug. Fix the grader. | +| Outdated or wrong code passes | Our bug. Fix the grader. | +| Many models fail the same way on a requirement | Suspect the prompt. Check that it states the requirement and that the requirement is real. | +| The prompt names something wrong | Our bug. Fix the prompt. | +| Models use outdated syntax | A model failure. Keep it. | +| Guidelines mode misses a pattern that models need | Fix the root `skills/` files, only where a run shows a benefit. `llms.md` is generated from them. | +| The SDK itself is at fault | Keep the failure and report it to the core team. Do not work around it. | +| The provider rejects the content | Keep the failure. Do not reword the prompt to avoid it. | +| The provider or the setup fails | The batch is refused. Fix the cause and rerun the batch. | + +## Changing a published task + +- Scores from before the change no longer compare with scores after it. The README should say so when the suite changes. +- Website uploads only add rows. Old rows stay on Trends until someone deletes them from the admin page. +- A refused batch uploads nothing, so a rerun has to cover the whole batch. Group changes so that one rerun covers them all. +- Weakening or deleting a check must be stated in the pull request, as the root `AGENTS.md` requires. diff --git a/tools/xtask-llm-benchmark/CLAUDE.md b/tools/xtask-llm-benchmark/CLAUDE.md new file mode 100644 index 00000000000..43c994c2d36 --- /dev/null +++ b/tools/xtask-llm-benchmark/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md From 04616dd05ad05b0411b1fb32eff7623a9518cc95 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Thu, 5 Nov 2026 11:19:20 -0500 Subject: [PATCH 36/41] Remove a product name from the benchmark README --- tools/xtask-llm-benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/README.md b/tools/xtask-llm-benchmark/README.md index 8dcfab45e16..235f36da408 100644 --- a/tools/xtask-llm-benchmark/README.md +++ b/tools/xtask-llm-benchmark/README.md @@ -82,7 +82,7 @@ write failure blocks upload. This does not resume or upload partial runs. | 093 | Private source tables, owner checks, live permission revocation and reconnect | | 094 | Real multi-connection presence, partial disconnect and reconnect | -These are SpacetimeDB contracts, not copies of Convex APIs. Pagination checks +Pagination checks correctness, not index scan cost. Backfill checks an online data migration algorithm, not deployment schema conversion. Quota is lifetime-based, not a timed rate limiter. Presence checks module behavior; it does not grade generated From 175ecfa1b01b6e8ac3f392424331110adec3bbef Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Fri, 6 Nov 2026 09:02:26 -0500 Subject: [PATCH 37/41] Match the benchmark's C# project to the new .NET 10 module shape Master's C# build change makes .NET 10 projects managed by default and no longer pulls the LLVM compiler in through the runtime package. The old template therefore built without NativeAOT and produced no StdbModule.wasm, so every C# task failed to publish. The template now follows templates/basic-cs. --- .../templates/csharp/server/StdbModule.csproj | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj b/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj index 6a14cc3a269..01214292422 100644 --- a/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj +++ b/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj @@ -2,14 +2,32 @@ net8.0;net10.0 - wasi-wasm + wasi-wasm enable enable $(NoWarn);STDB_UNSTABLE + + true + true + false + + https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json;https://api.nuget.org/v3/index.json;$(RestoreAdditionalProjectSources) + + + + + + + + + + + + From 2206be919fe1598cce830ca208d02f2f5f77e14f Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 10 Nov 2026 09:35:23 -0500 Subject: [PATCH 38/41] Format benchmark task specs and sync the codex-plugin skills copy --- .../spacetimedb/skills/csharp-server/SKILL.md | 15 ++++--- .../spacetimedb/skills/rust-server/SKILL.md | 4 +- .../skills/typescript-server/SKILL.md | 4 +- .../auth/t_041_registered_user_gate/spec.rs | 6 ++- .../auth/t_043_role_based_access/spec.rs | 6 ++- .../auth/t_046_shared_document/spec.rs | 12 ++++- .../basics/t_002_scheduled_table/spec.rs | 10 ++++- .../benchmarks/basics/t_006_delete/spec.rs | 38 ++++++++++------ .../basics/t_038_schedule_at_time/spec.rs | 9 +++- .../basics/t_039_cancel_schedule/spec.rs | 5 ++- .../data_modeling/t_024_event_table/spec.rs | 5 ++- .../t_029_filter_and_aggregate/spec.rs | 18 ++++++-- .../procedures/t_077_http_router/spec.rs | 28 +++++++----- .../queries/t_023_view_per_user/spec.rs | 14 +++++- .../queries/t_032_range_query/spec.rs | 8 +++- .../queries/t_035_select_distinct/spec.rs | 6 ++- .../queries/t_037_multi_column_filter/spec.rs | 13 +++++- .../reducers/t_058_batched_delete/spec.rs | 45 +++++++++++-------- 18 files changed, 177 insertions(+), 69 deletions(-) diff --git a/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md index 2deadf70c2c..9c22f377d0b 100644 --- a/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md @@ -187,15 +187,17 @@ The pattern is `ctx.Db.{Accessor}.{ColumnName}.{Method}(value)` for all indexed ```csharp [SpacetimeDB.Reducer(ReducerKind.Init)] -public static void OnInit(ReducerContext ctx) { ... } +public static void Init(ReducerContext ctx) { ... } [SpacetimeDB.Reducer(ReducerKind.ClientConnected)] -public static void OnConnect(ReducerContext ctx) { ... } +public static void ClientConnected(ReducerContext ctx) { ... } [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)] -public static void OnDisconnect(ReducerContext ctx) { ... } +public static void ClientDisconnected(ReducerContext ctx) { ... } ``` +Reducer and procedure method names must not start with `On`, `on`, or `__`; the code generator rejects them (STDB0010), so name lifecycle reducers `ClientConnected`, not `OnConnect`. + `ctx.ConnectionId` is `ConnectionId?`, including in connection lifecycle reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. ## Views @@ -333,7 +335,7 @@ public partial struct TickTimer [SpacetimeDB.Reducer] public static void Tick(ReducerContext ctx, TickTimer timer) { - // timer row is auto-deleted after this reducer runs + // a one-time row is deleted after this runs; an interval row stays } // One-time: fires once at a specific time @@ -344,7 +346,7 @@ var at = new ScheduleAt.Interval(TimeSpan.FromSeconds(5)); ctx.Db.TickTimer.Insert(new TickTimer { ScheduledId = 0, ScheduledAt = at }); ``` -Scheduled reducer callbacks use the ordinary `[SpacetimeDB.Reducer]` attribute. There is no `ReducerKind.Scheduled`; the table's `Scheduled` option associates the callback. +Scheduled reducer callbacks use the ordinary `[SpacetimeDB.Reducer]` attribute. There is no `ReducerKind.Scheduled`; the table's `Scheduled` option associates the callback. To let only the scheduler run a callback, reject calls where `ctx.Sender != ctx.DatabaseIdentity`. To construct a `ConnectionId` from a 128-bit numeric representation, encode it as exactly 16 little-endian bytes and call `ConnectionId.From(bytes)`. The result is nullable and must be checked before use. @@ -364,6 +366,7 @@ public static ResultValue Inspect(ProcedureContext ctx, string input) => Outbound HTTP is available through `ctx.Http`; handle its success/error result before using the response. Open short database transactions with `ctx.WithTx`. Perform network I/O before opening the transaction, and keep only database work inside its callback. ```csharp +string uri = "https://example.com/data"; // the C# HTTP API takes the URI as a string var result = ctx.Http.Get(uri); var text = result.Match( response => response.Body.ToStringUtf8Lossy(), @@ -435,3 +438,5 @@ public partial record Shape : SpacetimeDB.TaggedEnum<(Circle Circle, Rectangle R var a = new Shape.Circle(new Circle { Radius = 10 }); var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 }); ``` + +Declare `TaggedEnum` records inside your `Module` class. At file scope, a variant named after its own type, such as `Circle Circle`, fails with CS8910. diff --git a/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md index a924c2b6615..094b6f2c72f 100644 --- a/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md @@ -273,7 +273,7 @@ pub struct TickTimer { #[spacetimedb::reducer] pub fn tick(ctx: &ReducerContext, timer: TickTimer) { - // timer row is auto-deleted after this reducer runs + // a one-time row is deleted after this runs; an interval row stays } // One-time: fires once at a specific time @@ -284,6 +284,8 @@ let at = ScheduleAt::Interval(std::time::Duration::from_secs(5).into()); ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, scheduled_at: at }); ``` +To let only the scheduler run a callback, reject calls where `ctx.sender() != ctx.database_identity()`. + Construct a connection ID from a numeric representation with `ConnectionId::from_u128(value)`. ## Inter-Database Calls (Async IDC) diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md index 5fa71481235..d3348b5a04d 100644 --- a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md @@ -248,7 +248,7 @@ const tickTimer = table({ export const tick = spacetimedb.reducer( { onSchedule: tickTimer }, { timer: tickTimer.rowType }, - (ctx, { timer }) => { /* timer row auto-deleted after this runs */ } + (ctx, { timer }) => { /* a one-time row is deleted after this runs; an interval row stays */ } ); // `onSchedule` also works for scheduled procedures whose return type is `t.unit()`. @@ -261,6 +261,8 @@ export const tick = spacetimedb.reducer( // const micros = at.tag === 'time' ? at.value : at.value.microsSinceUnixEpoch; // bigint ``` +To let only the scheduler run a callback, reject calls where `!ctx.sender.isEqual(ctx.databaseIdentity)`. + ## Submodules Import the library's full module-namespace object, not just its default schema: diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs index 2cfc82b2863..73a2b3fa196 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_041_registered_user_gate/spec.rs @@ -8,7 +8,11 @@ pub fn spec() -> BenchmarkSpec { let mut scorers = default_schema_parity_scorers(host, file!(), route); scorers.push(scenario(file!(), route, host, lang, |s| { // C# uses RegisterUser because Register collides with a generated member. - let register = if s.lang == crate::eval::Lang::CSharp { "register_user" } else { "register" }; + let register = if s.lang == crate::eval::Lang::CSharp { + "register_user" + } else { + "register" + }; let (user, user_id) = s.new_user()?; s.expect_rejected_as(&user, "post_message", json!(["early"]), "not registered")?; s.rows("message", &["id"], json!([]))?; diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs index 8e87785c3cb..7f51e910a0d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_043_role_based_access/spec.rs @@ -8,7 +8,11 @@ pub fn spec() -> BenchmarkSpec { let mut scorers = default_schema_parity_scorers(host, file!(), route); scorers.push(scenario(file!(), route, host, lang, |s| { // C# uses RegisterUser because Register collides with a generated member. - let register = if s.lang == crate::eval::Lang::CSharp { "register_user" } else { "register" }; + let register = if s.lang == crate::eval::Lang::CSharp { + "register_user" + } else { + "register" + }; let (admin, admin_id) = s.new_user()?; let (member, member_id) = s.new_user()?; let (outsider, outsider_id) = s.new_user()?; diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs index 297da04aea8..f3002968176 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_046_shared_document/spec.rs @@ -19,9 +19,17 @@ pub fn spec() -> BenchmarkSpec { s.expect_call_as(&owner, "share_document", json!([1, friend_id]), None)?; s.expect_call_as(&friend, "edit_document", json!([1, "shared edit"]), None)?; s.expect_rejected_as(&stranger, "edit_document", json!([1, "stolen"]), "unauthorized")?; - s.rows("document", &["id", "owner", "title"], json!([[1, owner_id, "shared edit"]]))?; + s.rows( + "document", + &["id", "owner", "title"], + json!([[1, owner_id, "shared edit"]]), + )?; s.expect_call_as(&owner, "edit_document", json!([1, "owner edit"]), None)?; - s.rows("document", &["id", "owner", "title"], json!([[1, owner_id, "owner edit"]]))?; + s.rows( + "document", + &["id", "owner", "title"], + json!([[1, owner_id, "owner edit"]]), + )?; Ok(()) })); scorers diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/spec.rs index 8b444b2d0da..9808f0173ef 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_002_scheduled_table/spec.rs @@ -8,8 +8,14 @@ pub fn spec() -> BenchmarkSpec { scorers.push(scenario(file!(), route, host, lang, |s| { let query = format!("SELECT * FROM \"{}\"", crate::eval::table_name("tick_timer", s.lang)); let rows = s.owner_sql(&query)?; - anyhow::ensure!(rows.as_array().map(Vec::len) == Some(1), "expected one timer, got {rows}"); - anyhow::ensure!(rows.to_string().contains("50000"), "expected a 50ms interval, got {rows}"); + anyhow::ensure!( + rows.as_array().map(Vec::len) == Some(1), + "expected one timer, got {rows}" + ); + anyhow::ensure!( + rows.to_string().contains("50000"), + "expected a 50ms interval, got {rows}" + ); std::thread::sleep(std::time::Duration::from_millis(300)); // An interval row must survive its ticks. anyhow::ensure!(s.owner_sql(&query)? == rows, "interval row changed"); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs index 2c391201a02..7544bd6ce08 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_006_delete/spec.rs @@ -41,20 +41,30 @@ pub fn spec() -> BenchmarkSpec { )); // The count check alone passes when insert does nothing, so prove the row existed and others survive. - v.push(crate::eval::scenario::scenario(file!(), route_tag, host_url, lang, |s| { - s.call("insert_user", serde_json::json!(["Ada", 36, true]))?; - s.call("insert_user", serde_json::json!(["Bob", 25, false]))?; - let columns = ["name", "age", "active"]; - s.rows("user", &columns, serde_json::json!([["Ada", 36, true], ["Bob", 25, false]]))?; - let ids = s.owner_sql(&s.query("user", &["id", "name"]))?; - let ada = ids - .as_array() - .and_then(|rows| rows.iter().find(|row| row[1] == "Ada")) - .map(|row| row[0].clone()) - .ok_or_else(|| anyhow::anyhow!("no id for Ada in {ids}"))?; - s.call("delete_user", serde_json::json!([ada]))?; - s.rows("user", &columns, serde_json::json!([["Bob", 25, false]])) - })); + v.push(crate::eval::scenario::scenario( + file!(), + route_tag, + host_url, + lang, + |s| { + s.call("insert_user", serde_json::json!(["Ada", 36, true]))?; + s.call("insert_user", serde_json::json!(["Bob", 25, false]))?; + let columns = ["name", "age", "active"]; + s.rows( + "user", + &columns, + serde_json::json!([["Ada", 36, true], ["Bob", 25, false]]), + )?; + let ids = s.owner_sql(&s.query("user", &["id", "name"]))?; + let ada = ids + .as_array() + .and_then(|rows| rows.iter().find(|row| row[1] == "Ada")) + .map(|row| row[0].clone()) + .ok_or_else(|| anyhow::anyhow!("no id for Ada in {ids}"))?; + s.call("delete_user", serde_json::json!([ada]))?; + s.rows("user", &columns, serde_json::json!([["Bob", 25, false]])) + }, + )); v }) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs index 7cf3dc5256c..df26b51dfc4 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_038_schedule_at_time/spec.rs @@ -9,9 +9,14 @@ pub fn spec() -> BenchmarkSpec { let table = crate::eval::table_name("reminder", s.lang); let column = crate::eval::ident("scheduled_at", crate::eval::casing_for_lang(s.lang)); let rows = s.owner_sql(&format!("SELECT \"{column}\" FROM \"{table}\""))?; - anyhow::ensure!(rows.as_array().map(Vec::len) == Some(1), "expected one reminder, got {rows}"); + anyhow::ensure!( + rows.as_array().map(Vec::len) == Some(1), + "expected one reminder, got {rows}" + ); // Tag 1 is ScheduleAt::Time, about 60 seconds after publishing, not an interval. - let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_micros() as i64; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_micros() as i64; let due = rows[0][0][1][0].as_i64().unwrap_or_default(); anyhow::ensure!( rows[0][0][0] == 1 && due > now && due <= now + 61_000_000, diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs index 4587c529c87..4468fa0af8a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_039_cancel_schedule/spec.rs @@ -14,7 +14,10 @@ pub fn spec() -> BenchmarkSpec { let id = rows[0][0].clone(); s.call("cancel_cleanup", json!([id]))?; let rows = s.owner_sql(&query)?; - anyhow::ensure!(rows.as_array().map(Vec::is_empty) == Some(true), "job not cancelled: {rows}"); + anyhow::ensure!( + rows.as_array().map(Vec::is_empty) == Some(true), + "job not cancelled: {rows}" + ); Ok(()) })); scorers diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs index c3460e27e0d..7d4c76e035e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_024_event_table/spec.rs @@ -9,7 +9,10 @@ pub fn spec() -> BenchmarkSpec { scorers.push(scenario(file!(), route, host, lang, |s| { let query = format!("SELECT * FROM \"{}\"", s.relation("damage_event")); let update = s.watch_update(&query, || s.call("deal_damage", json!([7, 12, "fire"])))?; - anyhow::ensure!(update.contains("fire") && update.contains("12"), "missing event: {update}"); + anyhow::ensure!( + update.contains("fire") && update.contains("12"), + "missing event: {update}" + ); s.rows("damage_event", &["entity_id", "damage", "source"], json!([]))?; Ok(()) })); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs index 1fb5df8a9cf..e6be11be9d9 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/spec.rs @@ -13,10 +13,22 @@ pub fn spec() -> BenchmarkSpec { json!([[1, "tools", 10, true], [2, "tools", 25, false], [3, "toys", 99, true]]), )?; s.call("compute_stats", json!(["tools"]))?; - s.owner_rows("category_stats", &["category", "total_amount", "order_count"], json!([["tools", 35, 2]]))?; - s.seed("order", &["id", "category", "amount", "fulfilled"], json!([[4, "tools", 5, true]]))?; + s.owner_rows( + "category_stats", + &["category", "total_amount", "order_count"], + json!([["tools", 35, 2]]), + )?; + s.seed( + "order", + &["id", "category", "amount", "fulfilled"], + json!([[4, "tools", 5, true]]), + )?; s.call("compute_stats", json!(["tools"]))?; - s.owner_rows("category_stats", &["category", "total_amount", "order_count"], json!([["tools", 40, 3]]))?; + s.owner_rows( + "category_stats", + &["category", "total_amount", "order_count"], + json!([["tools", 40, 3]]), + )?; Ok(()) })); scorers diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs index a5691b6e5e5..1652ad9a866 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs @@ -8,21 +8,27 @@ pub fn spec() -> BenchmarkSpec { host_url, file!(), route_tag, - vec![ - ("GET", "/items", None), - ("POST", "/items", Some("book")), - ], + vec![("GET", "/items", None), ("POST", "/items", Some("book"))], false, "router_method_matrix", )); // The prompt says reject, not how: the server's own 404 and a module's 405 both qualify. - scorers.push(crate::eval::scenario::scenario(file!(), route_tag, host_url, lang, |s| { - for (method, path) in [(reqwest::Method::PUT, "/items"), (reqwest::Method::GET, "/missing")] { - let (status, body) = s.http_method(method.clone(), path, "no")?; - anyhow::ensure!((400..500).contains(&status), "{method} {path}: expected a rejection, got {status}: {body}"); - } - Ok(()) - })); + scorers.push(crate::eval::scenario::scenario( + file!(), + route_tag, + host_url, + lang, + |s| { + for (method, path) in [(reqwest::Method::PUT, "/items"), (reqwest::Method::GET, "/missing")] { + let (status, body) = s.http_method(method.clone(), path, "no")?; + anyhow::ensure!( + (400..500).contains(&status), + "{method} {path}: expected a rejection, got {status}: {body}" + ); + } + Ok(()) + }, + )); scorers }) } diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs index 1beb87ed548..6f037dfcb05 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_023_view_per_user/spec.rs @@ -15,8 +15,18 @@ pub fn spec() -> BenchmarkSpec { &["id", "identity", "name", "bio"], json!([[1, first_id, "Ann", "first"], [2, second_id, "Bo", "second"]]), )?; - s.rows_as(&first, "my_profile", &["id", "name", "bio"], json!([[1, "Ann", "first"]]))?; - s.rows_as(&second, "my_profile", &["id", "name", "bio"], json!([[2, "Bo", "second"]]))?; + s.rows_as( + &first, + "my_profile", + &["id", "name", "bio"], + json!([[1, "Ann", "first"]]), + )?; + s.rows_as( + &second, + "my_profile", + &["id", "name", "bio"], + json!([[2, "Bo", "second"]]), + )?; s.rows_as(&stranger, "my_profile", &["id", "name", "bio"], json!([]))?; Ok(()) })); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs index 3269ff9fc0a..97fa1ddcd9b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/spec.rs @@ -10,7 +10,13 @@ pub fn spec() -> BenchmarkSpec { s.seed( "product", &["id", "name", "price"], - json!([[1, "low", 9], [2, "min", 10], [3, "mid", 15], [4, "max", 20], [5, "high", 21]]), + json!([ + [1, "low", 9], + [2, "min", 10], + [3, "mid", 15], + [4, "max", 20], + [5, "high", 21] + ]), )?; s.call("find_in_price_range", json!([10, 20]))?; s.owner_rows( diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs index 02a3d803fd9..fd27245b437 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/spec.rs @@ -13,7 +13,11 @@ pub fn spec() -> BenchmarkSpec { json!([[1, "tools", 1], [2, "toys", 2], [3, "tools", 3], [4, "books", 4]]), )?; s.call("collect_distinct_categories", json!([]))?; - s.owner_rows("distinct_category", &["category"], json!([["books"], ["tools"], ["toys"]]))?; + s.owner_rows( + "distinct_category", + &["category"], + json!([["books"], ["tools"], ["toys"]]), + )?; Ok(()) })); scorers diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs index 23eb918d611..f306e085342 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/spec.rs @@ -10,10 +10,19 @@ pub fn spec() -> BenchmarkSpec { s.seed( "event_log", &["id", "category", "severity", "message"], - json!([[1, "db", 2, "match"], [2, "db", 3, "wrong severity"], [3, "net", 2, "wrong category"], [4, "db", 2, "also match"]]), + json!([ + [1, "db", 2, "match"], + [2, "db", 3, "wrong severity"], + [3, "net", 2, "wrong category"], + [4, "db", 2, "also match"] + ]), )?; s.call("filter_events", json!(["db", 2]))?; - s.owner_rows("filtered_event", &["event_id", "message"], json!([[1, "match"], [4, "also match"]]))?; + s.owner_rows( + "filtered_event", + &["event_id", "message"], + json!([[1, "match"], [4, "also match"]]), + )?; Ok(()) })); scorers diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs index 005f65fe766..e921749971a 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs @@ -36,26 +36,35 @@ pub fn spec() -> BenchmarkSpec { Duration::from_secs(10), )); // The count check alone passes when seeding does nothing, so prove rows existed and other groups survive. - scorers.push(crate::eval::scenario::scenario(file!(), route_tag, host_url, lang, |s| { - let count = |group: u64| -> anyhow::Result { - let query = format!( - "SELECT COUNT(*) AS n FROM \"{}\" WHERE \"{}\" = {group}", - s.relation("work_item"), - ident("group_id", casing_for_lang(s.lang)) + scorers.push(crate::eval::scenario::scenario( + file!(), + route_tag, + host_url, + lang, + |s| { + let count = |group: u64| -> anyhow::Result { + let query = format!( + "SELECT COUNT(*) AS n FROM \"{}\" WHERE \"{}\" = {group}", + s.relation("work_item"), + ident("group_id", casing_for_lang(s.lang)) + ); + Ok(s.owner_sql(&query)?[0][0].clone()) + }; + s.call("seed_group", json!([8, 3]))?; + s.call("seed_group", json!([9, 5]))?; + anyhow::ensure!( + count(8)? == json!(3) && count(9)? == json!(5), + "seed_group did not insert the rows" ); - Ok(s.owner_sql(&query)?[0][0].clone()) - }; - s.call("seed_group", json!([8, 3]))?; - s.call("seed_group", json!([9, 5]))?; - anyhow::ensure!(count(8)? == json!(3) && count(9)? == json!(5), "seed_group did not insert the rows"); - s.call("request_delete", json!([9]))?; - s.eventually(|| { - anyhow::ensure!(count(9)? == json!(0), "group 9 still has rows"); + s.call("request_delete", json!([9]))?; + s.eventually(|| { + anyhow::ensure!(count(9)? == json!(0), "group 9 still has rows"); + Ok(()) + })?; + anyhow::ensure!(count(8)? == json!(3), "deleting group 9 touched group 8"); Ok(()) - })?; - anyhow::ensure!(count(8)? == json!(3), "deleting group 9 touched group 8"); - Ok(()) - })); + }, + )); scorers }) } From f8cef1108c0cca5acd2c385b15914a4fa46fa408 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 10 Nov 2026 09:53:04 -0500 Subject: [PATCH 39/41] Keep guard unchanged, run the migration seed off the async thread, allow 24 hours per job, and remove the unused WithOutput error --- .github/workflows/llm-benchmark-periodic.yml | 4 +- crates/guard/src/lib.rs | 182 +++++---------- .../xtask-llm-benchmark/src/bench/records.rs | 1 - tools/xtask-llm-benchmark/src/bench/runner.rs | 214 +++++------------- tools/xtask-llm-benchmark/src/bench/types.rs | 2 - .../src/bin/llm_benchmark.rs | 4 +- 6 files changed, 126 insertions(+), 281 deletions(-) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 4516fc83dcf..e81399a2567 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -76,8 +76,8 @@ jobs: name: Run benchmarks (${{ matrix.lang }}) needs: prepare_matrix runs-on: spacetimedb-linux - # Models run one at a time, and a C# batch takes about 34 minutes. - timeout-minutes: 720 + # One mode for twelve models took up to 5.6 hours in C#, and the default run does both modes in one job. + timeout-minutes: 1440 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target TARGET_TRIPLE: x86_64-unknown-linux-gnu diff --git a/crates/guard/src/lib.rs b/crates/guard/src/lib.rs index b0cbe101ce6..eead324c936 100644 --- a/crates/guard/src/lib.rs +++ b/crates/guard/src/lib.rs @@ -121,20 +121,12 @@ impl SpacetimeDbGuard { Self::spawn_in_temp_data_dir_with_pg_port(None) } - /// Start a temporary server, returning startup errors after cleaning up the child. - pub fn try_spawn_in_temp_data_dir() -> std::io::Result { - let temp_dir = tempfile::tempdir()?; - let data_dir = temp_dir.path().to_path_buf(); - Self::spawn_spacetime_start_with_data_dir(false, None, data_dir, Some(temp_dir)) - } - /// Start `spacetimedb` in a temporary data directory with optional PostgreSQL wire protocol. pub fn spawn_in_temp_data_dir_with_pg_port(pg_port: Option) -> Self { let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); let data_dir_path = temp_dir.path().to_path_buf(); Self::spawn_spacetime_start_with_data_dir(false, pg_port, data_dir_path, Some(temp_dir)) - .expect("failed to start SpacetimeDB") } /// Start `spacetimedb` in a temporary data directory via: @@ -144,7 +136,6 @@ impl SpacetimeDbGuard { let data_dir_path = temp_dir.path().to_path_buf(); Self::spawn_spacetime_start_with_data_dir(true, None, data_dir_path, Some(temp_dir)) - .expect("failed to start SpacetimeDB") } /// Start `spacetimedb` with an explicit data directory (for restart scenarios). @@ -152,7 +143,7 @@ impl SpacetimeDbGuard { /// Unlike `spawn_in_temp_data_dir`, this method does not create a temporary directory. /// The caller is responsible for managing the data directory lifetime. pub fn spawn_with_data_dir(data_dir: PathBuf, pg_port: Option) -> Self { - Self::spawn_spacetime_start_with_data_dir(false, pg_port, data_dir, None).expect("failed to start SpacetimeDB") + Self::spawn_spacetime_start_with_data_dir(false, pg_port, data_dir, None) } fn spawn_spacetime_start_with_data_dir( @@ -160,11 +151,11 @@ impl SpacetimeDbGuard { pg_port: Option, data_dir: PathBuf, _data_dir_handle: Option, - ) -> std::io::Result { + ) -> Self { let spawn_id = next_spawn_id(); let (child, logs, host_url, reader_threads) = - Self::spawn_server(&data_dir, pg_port, spawn_id, use_installed_cli)?; - Ok(SpacetimeDbGuard { + Self::spawn_server(&data_dir, pg_port, spawn_id, use_installed_cli); + SpacetimeDbGuard { child, host_url, logs, @@ -173,7 +164,7 @@ impl SpacetimeDbGuard { _data_dir_handle, reader_threads, use_installed_cli, - }) + } } /// Stop the server process without dropping the guard. @@ -201,8 +192,7 @@ impl SpacetimeDbGuard { eprintln!("[RESTART-{:03}] Spawning new server", spawn_id); let (child, logs, host_url, reader_threads) = - Self::spawn_server(&self.data_dir, self.pg_port, spawn_id, self.use_installed_cli) - .expect("failed to restart SpacetimeDB"); + Self::spawn_server(&self.data_dir, self.pg_port, spawn_id, self.use_installed_cli); eprintln!( "[RESTART-{:03}] New server ready, pid={}, url={}", spawn_id, @@ -224,11 +214,7 @@ impl SpacetimeDbGuard { /// Kills the current server process and waits for it to exit. fn kill_process(&mut self) { - Self::kill_child(&mut self.child, &mut self.reader_threads); - } - - fn kill_child(child: &mut Child, reader_threads: &mut Vec>) { - let pid = child.id(); + let pid = self.child.id(); eprintln!("[KILL] Killing process tree for pid={}", pid); // Kill the process tree to ensure all child processes are terminated. @@ -246,16 +232,16 @@ impl SpacetimeDbGuard { #[cfg(not(windows))] { - let result = child.kill(); + let result = self.child.kill(); eprintln!("[KILL] kill result for pid={}: {:?}", pid, result); } - let wait_result = child.wait(); + let wait_result = self.child.wait(); eprintln!("[KILL] wait() result for pid={}: {:?}", pid, wait_result); // Join reader threads to prevent leaks. // The threads will exit naturally once the process is killed and pipes close. - let threads = std::mem::take(reader_threads); + let threads = std::mem::take(&mut self.reader_threads); for handle in threads { let _ = handle.join(); } @@ -264,19 +250,18 @@ impl SpacetimeDbGuard { /// Spawns a new server process with the given data directory. /// Returns (child, logs, host_url, reader_threads). - #[allow(clippy::type_complexity)] fn spawn_server( data_dir: &Path, pg_port: Option, spawn_id: u64, use_installed_cli: bool, - ) -> std::io::Result<(Child, Arc>, String, Vec>)> { + ) -> (Child, Arc>, String, Vec>) { let cmd = if use_installed_cli { eprintln!("[SPAWN-{:03}] START Using installed CLI", spawn_id); Command::new("spacetime") } else { // Use the built CLI (common case) - let cli_path = cli_binary_path(); + let cli_path = ensure_binaries_built(); Command::new(&cli_path) }; eprintln!( @@ -303,73 +288,64 @@ impl SpacetimeDbGuard { } eprintln!("[SPAWN-{:03}] Spawning child process", spawn_id); - let (mut child, logs, mut reader_threads) = Self::spawn_child(cmd, &args, spawn_id)?; + let (child, logs, reader_threads) = Self::spawn_child(cmd, &args, spawn_id); eprintln!("[SPAWN-{:03}] Child spawned pid={}", spawn_id, child.id()); - let host_url = Self::finish_server_start( - &mut child, - &logs, - &mut reader_threads, - spawn_id, - Duration::from_secs(10), - )?; - Ok((child, logs, host_url, reader_threads)) - } - fn finish_server_start( - child: &mut Child, - logs: &Arc>, - reader_threads: &mut Vec>, - spawn_id: u64, - timeout: Duration, - ) -> std::io::Result { - let result = (|| { - let listen_addr = wait_for_listen_addr(logs, timeout, spawn_id).ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::TimedOut, - "SpacetimeDB did not report its listen address", - ) - })?; - let host_url = format!("http://{listen_addr}"); - let client = Client::builder() - .timeout(timeout) - .build() - .map_err(std::io::Error::other)?; - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if let Ok(resp) = client - .get(format!("{host_url}/v1/ping")) - .timeout(deadline.saturating_duration_since(Instant::now())) - .send() - && resp.status().is_success() - { - eprintln!("[SPAWN-{:03}] HTTP ready at {}", spawn_id, host_url); - return Ok(host_url); - } - sleep(Duration::from_millis(50)); - } - Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - format!("SpacetimeDB HTTP /v1/ping not ready at {host_url}"), - )) - })(); - if result.is_err() { - Self::kill_child(child, reader_threads); - if let Ok(logs) = logs.lock() { - eprintln!("[SPAWN-{spawn_id:03}] startup failed; server logs:\n{logs}"); + // Wait for the server to be ready + eprintln!("[SPAWN-{:03}] Waiting for listen address", spawn_id); + let listen_addr = wait_for_listen_addr(&logs, Duration::from_secs(10), spawn_id).unwrap_or_else(|| { + // Dump diagnostic info on failure + let buf = logs.lock().unwrap(); + eprintln!("[SPAWN-{:03}] TIMEOUT after 10s", spawn_id); + eprintln!( + "[SPAWN-{:03}] Captured {} bytes, {} lines", + spawn_id, + buf.len(), + buf.lines().count() + ); + eprintln!( + "[SPAWN-{:03}] Contains 'Starting SpacetimeDB': {}", + spawn_id, + buf.contains("Starting SpacetimeDB") + ); + // Check if process is still running + drop(buf); // Release lock before try_wait + panic!("Timed out waiting for SpacetimeDB to report listen address") + }); + eprintln!("[SPAWN-{:03}] Got listen_addr={}", spawn_id, listen_addr); + + let host_url = format!("http://{}", listen_addr); + + // Wait until HTTP is ready + eprintln!("[SPAWN-{:03}] Waiting for HTTP ready", spawn_id); + let client = Client::new(); + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + let url = format!("{}/v1/ping", host_url); + if let Ok(resp) = client.get(&url).send() + && resp.status().is_success() + { + eprintln!("[SPAWN-{:03}] HTTP ready at {}", spawn_id, host_url); + return (child, logs, host_url, reader_threads); } + sleep(Duration::from_millis(50)); } - result + panic!("Timed out waiting for SpacetimeDB HTTP /v1/ping at {}", host_url); } - #[allow(clippy::type_complexity)] fn spawn_child( mut cmd: Command, args: &[&str], spawn_id: u64, - ) -> std::io::Result<(Child, Arc>, Vec>)> { + ) -> (Child, Arc>, Vec>) { eprintln!("[SPAWN-{:03}] spawn_child: about to spawn", spawn_id); - let mut child = cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn()?; + let mut child = cmd + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("failed to spawn spacetimedb-cli"); let pid = child.id(); eprintln!("[SPAWN-{:03}] spawn_child: spawned pid={}", spawn_id, pid); @@ -430,7 +406,7 @@ impl SpacetimeDbGuard { } eprintln!("[SPAWN-{:03}] spawn_child: readers attached", spawn_id); - Ok((child, logs, reader_threads)) + (child, logs, reader_threads) } } @@ -520,43 +496,3 @@ impl Drop for SpacetimeDbGuard { } } } - -#[cfg(test)] -mod startup_tests { - use super::*; - - #[test] - fn waiting_child() { - if env::var_os("SPACETIMEDB_GUARD_TEST_CHILD").is_some() { - sleep(Duration::from_secs(60)); - } - } - - #[test] - fn startup_timeout_reaps_child_for_both_readiness_phases() { - for reported_address in [false, true] { - let mut command = Command::new(env::current_exe().unwrap()); - command.env("SPACETIMEDB_GUARD_TEST_CHILD", "1"); - let (mut child, logs, mut readers) = - SpacetimeDbGuard::spawn_child(command, &["--exact", "startup_tests::waiting_child"], 0).unwrap(); - assert!(!readers.is_empty()); - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let log = if reported_address { - format!("Starting SpacetimeDB listening on {}\n", listener.local_addr().unwrap()) - } else { - String::new() - }; - *logs.lock().unwrap() = log; - let error = - SpacetimeDbGuard::finish_server_start(&mut child, &logs, &mut readers, 0, Duration::from_millis(10)) - .unwrap_err(); - assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); - assert_eq!(error.to_string().contains("HTTP"), reported_address); - assert!( - child.try_wait().unwrap().is_some(), - "failed startup left its child alive" - ); - assert!(readers.is_empty()); - } - } -} diff --git a/tools/xtask-llm-benchmark/src/bench/records.rs b/tools/xtask-llm-benchmark/src/bench/records.rs index 31c8ce06edc..52587a19241 100644 --- a/tools/xtask-llm-benchmark/src/bench/records.rs +++ b/tools/xtask-llm-benchmark/src/bench/records.rs @@ -103,7 +103,6 @@ impl BatchRecord { "provider_response": provider_error.as_ref().and_then(|error| error.downcast_ref::()), "http_response": provider_error.as_ref().and_then(|error| error.downcast_ref::())}), ), - Err(RunOneError::WithOutput { .. }) => bail!("normalize generated-code failures before recording"), }; *self.tasks.get_mut(task).context("task not in planned batch")? = state; self.append(json!({"event": "attempt", "task": task, "attempt": attempt}), false) diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index f6b19428cf6..f51a692c661 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -64,23 +64,11 @@ where fn record_task_result( record: &mut BatchRecord, - cfg: &BenchRunContext<'_>, task: TaskPaths, result: Result, started: chrono::DateTime, ) -> Result<(TaskPaths, Result)> { - let result = match result { - Err(RunOneError::WithOutput { msg, llm_output }) => Ok(build_fail_outcome( - &task, - cfg.lang.as_str(), - cfg.route, - cfg.hash, - anyhow!(msg), - llm_output, - )), - other => other, - } - .map(|mut outcome| { + let result = result.map(|mut outcome| { outcome.started_at.get_or_insert(started); outcome }); @@ -144,6 +132,12 @@ async fn publish_async( .await? } +async fn seed_async(db: String, host: String) -> Result<(), String> { + task::spawn_blocking(move || call_reducer_json_out(&db, "seed", &[], Some(&host)).map(drop)) + .await + .map_err(|error| format!("seed worker failed: {error}"))? +} + async fn delete_database_async(database: PublishedDatabase) -> Result<()> { task::spawn_blocking(move || database.delete()).await??; Ok(()) @@ -551,14 +545,14 @@ impl TaskRunner { .await; match setup_result { Ok(database) => { - call_reducer_json_out(&llm_db, "seed", &[], Some(cfg.host.as_deref().unwrap_or("local"))).map_err( - |error| RunOneError::Infrastructure { + seed_async(llm_db.clone(), cfg.host.clone().unwrap_or_else(|| "local".into())) + .await + .map_err(|error| RunOneError::Infrastructure { msg: format!("migration seed failed: {error}"), llm_output: Some(llm_output.clone()), provider_error: None, completed_outcome: None, - }, - )?; + })?; Ok(Some(database)) } Err(error) => Err(error), @@ -757,12 +751,6 @@ fn finish_reference_cleanup( provider_error, completed_outcome, }), - Err(RunOneError::WithOutput { msg, llm_output }) => Err(RunOneError::Infrastructure { - msg: format!("{msg}; {cleanup}"), - llm_output: Some(llm_output), - provider_error: None, - completed_outcome: None, - }), Ok(outcome) => Err(RunOneError::Infrastructure { msg: cleanup, llm_output: outcome.llm_output.clone(), @@ -777,9 +765,6 @@ fn finish_reference_cleanup( /// Tasks where the LLM responded but the code failed to compile/score are kept as outcomes. fn partition_results( results: Vec<(TaskPaths, Result)>, - lang_name: &str, - route: &ModelRoute, - hash: &str, ) -> Result<(Vec, Vec)> { let mut good = Vec::new(); let mut retry = Vec::new(); @@ -787,18 +772,6 @@ fn partition_results( for (task, r) in results { match r { Ok(v) => good.push(v), - Err(RunOneError::WithOutput { msg, llm_output }) => { - // LLM responded but something else failed — keep as a result - eprintln!("\u{26a0}\u{fe0f} task failed but has output: {msg}"); - good.push(build_fail_outcome( - &task, - lang_name, - route, - hash, - anyhow::anyhow!(msg), - llm_output, - )); - } Err(RunOneError::Infrastructure { msg, llm_output, .. }) => { let output_note = llm_output .as_deref() @@ -966,11 +939,11 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu } }), buf, - |(task, result, started)| record_task_result(&mut record, cfg, task, result, started), + |(task, result, started)| record_task_result(&mut record, task, result, started), ) .await?; - let (mut outcomes, mut retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; + let (mut outcomes, mut retry_tasks) = partition_results(results)?; // Retry provider-error tasks until all pass or none make progress const MAX_RETRY_ROUNDS: usize = 3; @@ -1015,11 +988,11 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu } }), buf, - |(task, result, started)| record_task_result(&mut record, cfg, task, result, started), + |(task, result, started)| record_task_result(&mut record, task, result, started), ) .await?; - let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash)?; + let (new_outcomes, still_failing) = partition_results(retry_results)?; if new_outcomes.is_empty() && !still_failing.is_empty() { // No progress — provider is likely down. Keep the failures and stop retrying. @@ -1151,11 +1124,11 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> } }), buf, - |(task, result, started)| record_task_result(&mut record, cfg, task, result, started), + |(task, result, started)| record_task_result(&mut record, task, result, started), ) .await?; - let (mut outcomes, retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; + let (mut outcomes, retry_tasks) = partition_results(results)?; // Retry provider-error tasks until all pass or none make progress let pending = { @@ -1202,11 +1175,11 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> } }), buf, - |(task, result, started)| record_task_result(&mut record, cfg, task, result, started), + |(task, result, started)| record_task_result(&mut record, task, result, started), ) .await?; - let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash)?; + let (new_outcomes, still_failing) = partition_results(retry_results)?; if new_outcomes.is_empty() && !still_failing.is_empty() { eprintln!( @@ -1347,7 +1320,8 @@ pub async fn build_goldens_only_for_lang( clear_database: true, }) .await?; - call_reducer_json_out(&golden_db, "seed", &[], Some(host_clone.as_deref().unwrap_or("local"))) + seed_async(golden_db.clone(), host_clone.clone().unwrap_or_else(|| "local".into())) + .await .map_err(anyhow::Error::msg)?; let database = runner .publish_golden_only(PublishParams { @@ -1471,7 +1445,9 @@ pub async fn validate_goldens_for_lang( "golden-validation-setup", ) .await?; - call_reducer_json_out(&candidate_db, "seed", &[], Some(host_url)).map_err(anyhow::Error::msg)?; + seed_async(candidate_db.clone(), host_url.to_string()) + .await + .map_err(anyhow::Error::msg)?; let database = runner .publish( PublishParams { @@ -1559,59 +1535,6 @@ fn discover_tasks(benchmarks_root: &Path) -> Result> { Ok(out) } -fn build_fail_outcome( - task: &TaskPaths, - lang_name: &str, - route: &ModelRoute, - hash: &str, - err: anyhow::Error, - llm_output: String, -) -> RunOutcome { - let category = category_slug(&task.root); - let task_id = task_slug(&task.root); - let now = Utc::now(); - let mut sd: HashMap = HashMap::new(); - sd.insert( - "publish_error".to_string(), - ScoreDetails { - pass: false, - partial: 0.0, - notes: json!({ - "phase": "build_or_publish", - "error": format!("{:#}", err), - }), - }, - ); - - RunOutcome { - hash: hash.to_string(), - task: task_id.clone(), - lang: lang_name.to_string(), - golden_published: false, - category: Some(category), - - model_name: route.model_id(), - total_tests: 1, - passed_tests: 0, - - llm_output: Some(llm_output), - - route_api_model: Some(route.api_model.to_string()), - golden_db: None, - llm_db: None, - work_dir_golden: None, - work_dir_llm: None, - scorer_details: Some(sd), - - vendor: route.vendor.slug().to_string(), - input_tokens: None, - output_tokens: None, - generation_duration_ms: None, - started_at: Some(now), - finished_at: Some(now), - } -} - fn build_rejected_outcome( task: &TaskPaths, cfg: &RunContext<'_>, @@ -1625,18 +1548,27 @@ fn build_rejected_outcome( notes: json!({"phase": "generation", "error": format!("{error:#}")}), }; RunOutcome { + hash: cfg.hash.to_string(), + task: task_slug(&task.root), + lang: cfg.lang_name.to_string(), + golden_published: false, + category: Some(category_slug(&task.root)), + model_name: cfg.route.model_id(), total_tests: total_tests as u32, + passed_tests: 0, llm_output: None, + route_api_model: Some(cfg.route.api_model.to_string()), + golden_db: None, + llm_db: None, + work_dir_golden: None, + work_dir_llm: None, scorer_details: Some([("content_rejection".to_string(), details)].into_iter().collect()), + vendor: cfg.route.vendor.slug().to_string(), + input_tokens: None, + output_tokens: None, + generation_duration_ms: None, started_at: Some(started), - ..build_fail_outcome( - task, - cfg.lang_name, - cfg.route, - cfg.hash, - anyhow!("unused"), - String::new(), - ) + finished_at: Some(Utc::now()), } } @@ -1746,7 +1678,7 @@ mod build_cleanup_tests { }) } } - let server = spacetimedb_guard::SpacetimeDbGuard::try_spawn_in_temp_data_dir().unwrap(); + let server = spacetimedb_guard::SpacetimeDbGuard::spawn_in_temp_data_dir(); let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -2014,7 +1946,7 @@ mod build_cleanup_tests { .scorers_for(Lang::Rust, "test", "local") .len() as u32; assert!(expected > 1, "the fixture must show the full denominator"); - let (outcomes, pending) = partition_results(vec![(task, result)], "rust", &route, "test").unwrap(); + let (outcomes, pending) = partition_results(vec![(task, result)]).unwrap(); assert!(pending.is_empty(), "a rejection must not block the batch"); assert_eq!(outcomes[0].passed_tests, 0); assert_eq!(outcomes[0].total_tests, expected); @@ -2035,36 +1967,25 @@ mod build_cleanup_tests { answers_rust: PathBuf::new(), answers_typescript: PathBuf::new(), }; - let route = ModelRoute::new("Test", crate::llm::types::Vendor::OpenAi, "test", None); - let (outcomes, pending) = partition_results( - vec![ - ( - task(), - Err(RunOneError::WithOutput { - msg: "compile failed".into(), - llm_output: "invalid code".into(), - }), - ), - (task(), Err(RunOneError::Other(anyhow::anyhow!("402 Payment Required")))), - ( - task(), - Err(RunOneError::Rejected { - outcome: Box::new(build_fail_outcome( - &task(), - "rust", - &route, - "test", - anyhow!("rejected"), - String::new(), - )), - error: refusal_error(), - }), - ), - ], - "rust", - &route, - "test", - ) + // A model's compile failure arrives as a scored outcome. + let failed = || { + serde_json::from_value::(json!({ + "hash": "test", "task": "t_001", "lang": "rust", "model_name": "test", "vendor": "test", + "golden_published": false, "passed_tests": 0, "total_tests": 2 + })) + .unwrap() + }; + let (outcomes, pending) = partition_results(vec![ + (task(), Ok(failed())), + (task(), Err(RunOneError::Other(anyhow::anyhow!("402 Payment Required")))), + ( + task(), + Err(RunOneError::Rejected { + outcome: Box::new(failed()), + error: refusal_error(), + }), + ), + ]) .unwrap(); assert_eq!(outcomes.len(), 2); assert_eq!(pending.len(), 1, "provider failures are not scored zeroes"); @@ -2121,17 +2042,6 @@ mod build_cleanup_tests { } other => panic!("completed scoring lost after cleanup failure: {other:?}"), } - let failed_code = Err(RunOneError::WithOutput { - msg: "compile failed".into(), - llm_output: "candidate code".into(), - }); - match finish_reference_cleanup(failed_code, Err(anyhow!("delete failed"))) { - Err(RunOneError::Infrastructure { msg, llm_output, .. }) => { - assert!(msg.contains("compile failed") && msg.contains("delete failed")); - assert_eq!(llm_output.as_deref(), Some("candidate code")); - } - other => panic!("expected unscored cleanup failure: {other:?}"), - } let refusal = serde_json::from_value::(json!({ "choices": [{"finish_reason":"content_filter", "message":{"content":null,"refusal":"private detail"}}] })) diff --git a/tools/xtask-llm-benchmark/src/bench/types.rs b/tools/xtask-llm-benchmark/src/bench/types.rs index 44f3fc2b0ed..28cd5477ce5 100644 --- a/tools/xtask-llm-benchmark/src/bench/types.rs +++ b/tools/xtask-llm-benchmark/src/bench/types.rs @@ -133,8 +133,6 @@ pub struct RouteRun { #[derive(Debug, Error)] pub enum RunOneError { - #[error("{msg}")] - WithOutput { msg: String, llm_output: String }, #[error("{msg}")] Infrastructure { msg: String, diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index 376a1dd5ecf..c994f051982 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -972,7 +972,9 @@ fn initialize_runtime(skip_runtime: bool) -> Result { } // Use locally built SpacetimeDB so server supports spacetime:sys@2.0 (required by local TypeScript SDK). - let spacetime = SpacetimeDbGuard::try_spawn_in_temp_data_dir().context("start local benchmark server")?; + // The guard panics when the server fails to start; catch it so one C# model's server cannot end the whole job. + let spacetime = std::panic::catch_unwind(SpacetimeDbGuard::spawn_in_temp_data_dir) + .map_err(|_| anyhow::anyhow!("start local benchmark server"))?; let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; From 835e5d0a1c1a3567ed3c38e941eeed1499cc2de6 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 10 Nov 2026 09:54:33 -0500 Subject: [PATCH 40/41] Drop the timeout comment --- .github/workflows/llm-benchmark-periodic.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index e81399a2567..ae5d8d0b91b 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -76,7 +76,6 @@ jobs: name: Run benchmarks (${{ matrix.lang }}) needs: prepare_matrix runs-on: spacetimedb-linux - # One mode for twelve models took up to 5.6 hours in C#, and the default run does both modes in one job. timeout-minutes: 1440 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target From ac70695886390a96a7fc0e14cd43da303ae19aa0 Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Tue, 10 Nov 2026 09:55:01 -0500 Subject: [PATCH 41/41] Use one timeout for every reference check --- .github/workflows/llm-benchmark-validate-goldens.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/llm-benchmark-validate-goldens.yml b/.github/workflows/llm-benchmark-validate-goldens.yml index 10fde3ef062..513300168c8 100644 --- a/.github/workflows/llm-benchmark-validate-goldens.yml +++ b/.github/workflows/llm-benchmark-validate-goldens.yml @@ -27,7 +27,7 @@ concurrency: jobs: validate-goldens: runs-on: spacetimedb-linux - timeout-minutes: ${{ matrix.lang == 'csharp' && 180 || 60 }} + timeout-minutes: 180 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target TARGET_TRIPLE: x86_64-unknown-linux-gnu