diff --git a/crates/grill-perf/src/media.rs b/crates/grill-perf/src/media.rs index b192e23..4120437 100644 --- a/crates/grill-perf/src/media.rs +++ b/crates/grill-perf/src/media.rs @@ -85,6 +85,9 @@ enum Metric { Generation, #[serde(rename = "load_to_generation_us")] LoadToGeneration, + #[serde(rename = "first_audio_us")] + FirstAudio, + DurationRatio, } #[derive(Clone, Copy, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] @@ -279,9 +282,24 @@ struct Gate { max_reference_spread_bps: u32, } #[derive(Deserialize)] +#[serde(tag = "kind")] +enum Runner { + #[serde(rename = "media-runner-v1")] + Cli(Cli), + #[serde(rename = "media-speech-server-v1")] + Speech(capture::speech::Server), +} +impl Runner { + fn setup(&self) -> Option<&Setup> { + match self { + Self::Cli(runner) => runner.setup.as_ref(), + Self::Speech(_) => None, + } + } +} +#[derive(Deserialize)] #[serde(deny_unknown_fields)] -struct Runner { - kind: String, +struct Cli { version: u32, id: String, execution: Execution, @@ -379,7 +397,7 @@ fn reject_positional_records(value: &serde_json::Value) -> Result<()> { for (key, v) in map { match key.as_str() { "env" if !v.is_object() => return Err("environment must be a JSON object".into()), - "request" | "limits" | "expect" | "setup" | "cache" if !v.is_null() => { + "request" | "limits" | "expect" | "setup" | "cache" | "pcm" if !v.is_null() => { reject_positional_records(v)? } "outputs" | "cells" | "slots" | "gates" | "pins" => { @@ -411,12 +429,22 @@ struct FieldRule { name: &'static str, min: u64, max: u64, - request: [Presence; 3], - expect: [Presence; 3], + request: [Presence; 4], + expect: [Presence; 4], +} + +/// Rule-table column after the three modalities: audio requests sent to a speech server. +const SPEECH: usize = 3; +fn column(modality: Modality) -> usize { + match modality { + Modality::Image => 0, + Modality::Audio => 1, + Modality::Video => 2, + } } fn validate_fields( - modality: Modality, + index: usize, context: &str, expect: bool, values: &[(&str, Option)], @@ -427,78 +455,73 @@ fn validate_fields( name: "width", min: 1, max: MAX_CANVAS as u64, - request: [R, F, R], - expect: [R, F, R], + request: [R, F, R, F], + expect: [R, F, R, F], }, FieldRule { name: "height", min: 1, max: MAX_CANVAS as u64, - request: [R, F, R], - expect: [R, F, R], + request: [R, F, R, F], + expect: [R, F, R, F], }, FieldRule { name: "steps", min: 1, max: MAX_STEPS as u64, - request: [R, O, O], - expect: [F, F, F], + request: [R, O, O, F], + expect: [F, F, F, F], }, FieldRule { name: "images_per_run", min: 1, max: MAX_IMAGES as u64, - request: [R, F, F], - expect: [F, F, F], + request: [R, F, F, F], + expect: [F, F, F, F], }, FieldRule { name: "frames", min: 1, max: MAX_FRAMES, - request: [F, F, R], - expect: [F, F, R], + request: [F, F, R, F], + expect: [F, F, R, F], }, FieldRule { name: "fps", min: 1, max: MAX_FPS, - request: [F, F, R], - expect: [F, F, R], + request: [F, F, R, F], + expect: [F, F, R, F], }, FieldRule { name: "duration_us", min: 1, max: MAX_DURATION_US, - request: [F, R, O], - expect: [F, R, O], + request: [F, R, O, F], + expect: [F, R, O, F], }, FieldRule { name: "sample_rate", min: 1, max: u32::MAX as u64, - request: [F, R, F], - expect: [F, R, F], + request: [F, R, F, R], + expect: [F, R, F, F], }, FieldRule { name: "channels", min: 1, max: u16::MAX as u64, - request: [F, F, F], - expect: [F, R, F], + request: [F, F, F, F], + expect: [F, R, F, F], }, FieldRule { name: "audio", min: 0, max: 1, - request: [F, F, F], - expect: [F, F, O], + request: [F, F, F, F], + expect: [F, F, O, F], }, ]; - let index = match modality { - Modality::Image => 0, - Modality::Audio => 1, - Modality::Video => 2, - }; for rule in &RULES { let value = values .iter() @@ -515,7 +538,7 @@ fn validate_fields( } (Presence::Forbidden, Some(_)) => { return Err(format!( - "{context}.{} is forbidden for this modality", + "{context}.{} is forbidden for this modality and runner", rule.name )); } @@ -543,7 +566,7 @@ fn validate_expect(output: &Output, modality: Modality) -> Result<()> { } let e = &output.expect; validate_fields( - modality, + column(modality), &format!("outputs.{name}.expect"), true, &[ @@ -559,7 +582,7 @@ fn validate_expect(output: &Output, modality: Modality) -> Result<()> { ) } -fn validate_request(r: &Request, modality: Modality, context: &str) -> Result<()> { +fn validate_request(r: &Request, index: usize, context: &str) -> Result<()> { for (field, value) in [ ("prompt", Some(r.prompt.as_str())), ("negative_prompt", r.negative_prompt.as_deref()), @@ -574,7 +597,7 @@ fn validate_request(r: &Request, modality: Modality, context: &str) -> Result<() return Err(format!("{context}.seeds needs 1..={MAX_SEEDS} entries")); } validate_fields( - modality, + index, context, false, &[ @@ -589,7 +612,7 @@ fn validate_request(r: &Request, modality: Modality, context: &str) -> Result<() ], ) } -fn validate(w: &Workload, r: &Runner) -> Result<()> { +fn validate(w: &Workload, runner: &Runner) -> Result<()> { if w.kind != "media-workload-v1" { return Err("workload.kind must be media-workload-v1".into()); } @@ -599,7 +622,12 @@ fn validate(w: &Workload, r: &Runner) -> Result<()> { if !identifier(&w.name) { return Err(format!("invalid workload.name: {}", w.name)); } - validate_request(&w.request, w.modality, "request")?; + let index = match runner { + Runner::Cli(_) => column(w.modality), + Runner::Speech(_) if w.modality == Modality::Audio => SPEECH, + Runner::Speech(_) => return Err("speech server runners require audio modality".into()), + }; + validate_request(&w.request, index, "request")?; if !(1..=MAX_CELLS).contains(&w.cells.len()) { return Err(format!("cells needs 1..={MAX_CELLS} entries")); } @@ -621,10 +649,13 @@ fn validate(w: &Workload, r: &Runner) -> Result<()> { )); } if let Some(req) = &c.request { - validate_request(req, w.modality, &format!("cell {}.request", c.id))?; + validate_request(req, index, &format!("cell {}.request", c.id))?; } } - validate_outputs(w, r)?; + match runner { + Runner::Cli(r) => validate_outputs(w, r)?, + Runner::Speech(server) => server.validate(w)?, + } let mut slots = BTreeSet::new(); let mut arms = BTreeSet::new(); for s in &w.slots { @@ -646,9 +677,19 @@ fn validate(w: &Workload, r: &Runner) -> Result<()> { g.cell )); } - if g.metric != Metric::EndToEnd && r.markers.is_none() { + let requirement = match g.metric { + Metric::EndToEnd => None, + Metric::Generation | Metric::LoadToGeneration => { + (!matches!(runner, Runner::Cli(r) if r.markers.is_some())) + .then_some("runner.markers") + } + Metric::FirstAudio => (!matches!(runner, Runner::Speech(s) if s.stream)) + .then_some("a streaming speech server runner"), + Metric::DurationRatio => (w.modality != Modality::Audio).then_some("audio modality"), + }; + if let Some(requirement) = requirement { return Err(format!( - "gate {}: metric {} requires runner.markers", + "gate {}: metric {} requires {requirement}", g.cell, serde_json::to_value(g.metric) .expect("metric string") @@ -678,9 +719,12 @@ fn validate(w: &Workload, r: &Runner) -> Result<()> { return Err(format!("limits.{field} must be positive")); } } - validate_runner(r, l) + match runner { + Runner::Cli(r) => validate_runner(r, l), + Runner::Speech(_) => Ok(()), + } } -fn validate_outputs(w: &Workload, r: &Runner) -> Result<()> { +fn validate_outputs(w: &Workload, r: &Cli) -> Result<()> { if w.outputs.is_empty() { return Err("outputs needs at least one declaration".into()); } @@ -740,7 +784,7 @@ fn validate_output_args( } Ok(()) } -fn validate_runner(r: &Runner, limits: &Limits) -> Result<()> { +fn validate_runner(r: &Cli, limits: &Limits) -> Result<()> { let marker_count = r .args .iter() @@ -758,9 +802,6 @@ fn validate_runner(r: &Runner, limits: &Limits) -> Result<()> { { return Err("runner.setup.args: {marker_file} is forbidden".into()); } - if r.kind != "media-runner-v1" { - return Err("runner.kind must be media-runner-v1".into()); - } if r.version != 1 { return Err("runner.version must be 1".into()); } @@ -887,11 +928,41 @@ fn substitute( Ok(result) } #[derive(Serialize)] +#[serde(rename_all = "snake_case")] +enum Rendered { + Args(Vec), + /// The exact JSON request body sent to a speech server. + Body(String), +} +#[derive(Serialize)] struct Sample { cell: String, phase: Phase, seed: u64, - args: Vec, + #[serde(flatten)] + request: Rendered, +} +#[derive(Serialize)] +#[serde(untagged)] +#[expect( + clippy::large_enum_variant, + reason = "one value per preflight, moved into the receipt" +)] +enum Admitted { + Cli { + execution: Execution, + markers: Option, + program: Program, + setup_program: Option, + setup_args: Option>, + env: BTreeMap, + cwd: Option, + cache_paths: Vec, + cache_declared_state: CacheState, + }, + Speech { + endpoint: String, + }, } #[derive(Serialize)] struct Preflight { @@ -899,18 +970,11 @@ struct Preflight { kind: &'static str, claim: &'static str, support: &'static str, - execution: Execution, - markers: Option, + #[serde(flatten)] + runner: Admitted, workload_sha256: String, workload_typed_sha256: String, runner_sha256: String, - program: Program, - setup_program: Option, - setup_args: Option>, - env: BTreeMap, - cwd: Option, - cache_paths: Vec, - cache_declared_state: CacheState, warmup_runs: u64, measured_runs: u64, total_runs: u64, @@ -995,9 +1059,7 @@ fn render( .collect() } -fn preflight(plan: &Plan) -> Result { - let w = &plan.workload; - let r = &plan.runner; +fn admit_cli(r: &Cli, w: &Workload) -> Result { let program = resolve_program(&r.program)?; let setup_program = r .setup @@ -1031,6 +1093,32 @@ fn preflight(plan: &Plan) -> Result { .map_err(|error| format!("runner.setup.args: {error}")) }) .transpose()?; + Ok(Admitted::Cli { + execution: r.execution, + markers: r.markers, + program, + setup_program, + setup_args, + env: r.env.clone(), + cwd, + cache_paths: r + .cache + .paths + .iter() + .map(|p| expand(p)) + .collect::>()?, + cache_declared_state: r.cache.declared_state, + }) +} + +fn preflight(plan: &Plan) -> Result { + let w = &plan.workload; + let runner = match &plan.runner { + Runner::Cli(r) => admit_cli(r, w)?, + Runner::Speech(server) => Admitted::Speech { + endpoint: server.url()?.to_string(), + }, + }; let mut samples = Vec::new(); let (mut warmup_runs, mut measured_runs, mut wall_clock_allowance_ms) = (0u64, 0u64, 0u64); for sample in schedule(w) { @@ -1042,18 +1130,21 @@ fn preflight(plan: &Plan) -> Result { cell: sample.cell.id.clone(), phase: sample.phase, seed: sample.seed, - args: render( - &r.args, - sample.request, - sample.seed, - Path::new(&sample.directory()), - w, - )?, + request: match &plan.runner { + Runner::Cli(r) => Rendered::Args(render( + &r.args, + sample.request, + sample.seed, + Path::new(&sample.directory()), + w, + )?), + Runner::Speech(server) => Rendered::Body(server.body(sample.request)?), + }, }); wall_clock_allowance_ms += u64::from(w.limits.run_deadline_ms) + u64::from(sample.cell.pause_between_runs_ms); } - if let Some(s) = &r.setup { + if let Some(s) = plan.runner.setup() { wall_clock_allowance_ms += u64::from(s.deadline_ms.unwrap_or(w.limits.setup_deadline_ms)); } Ok(Preflight { @@ -1061,25 +1152,12 @@ fn preflight(plan: &Plan) -> Result { kind: "media-preflight-v1", claim: "offline-declared-media-plan-admission-launches-nothing", support: SUPPORT, - execution: r.execution, - markers: r.markers, + runner, workload_sha256: evidence::digest(&plan.workload_bytes), workload_typed_sha256: evidence::digest( &serde_json::to_vec(&w).map_err(|e| e.to_string())?, ), runner_sha256: evidence::digest(&plan.runner_bytes), - program, - setup_program, - setup_args, - env: r.env.clone(), - cwd, - cache_paths: r - .cache - .paths - .iter() - .map(|p| expand(p)) - .collect::>()?, - cache_declared_state: r.cache.declared_state, warmup_runs, measured_runs, total_runs: warmup_runs + measured_runs, @@ -1089,16 +1167,16 @@ fn preflight(plan: &Plan) -> Result { } #[derive(Subcommand)] pub enum Command { - /// Validate an image workload and runner, hash programs and print sample argv without starting a process. + /// Validate a media workload and runner, hash programs and print sample argv or request bodies without starting a process or contacting a server. Preflight { - /// Shared image workload document. + /// Shared media workload document. #[arg(long)] workload: PathBuf, - /// Per-arm process runner declaration. + /// Per-arm CLI or speech-server runner declaration. #[arg(long)] runner: PathBuf, }, - /// Capture serial fresh-process image samples. + /// Capture serial fresh-process or speech-server samples. Capture { #[arg(long)] workload: PathBuf, diff --git a/crates/grill-perf/src/media/capture.rs b/crates/grill-perf/src/media/capture.rs index a873bfa..aba37e4 100644 --- a/crates/grill-perf/src/media/capture.rs +++ b/crates/grill-perf/src/media/capture.rs @@ -1,4 +1,5 @@ pub(super) mod compare; +pub(super) mod speech; use super::cache::{self, Observation}; use super::*; use crate::{ @@ -11,6 +12,13 @@ use std::{ }; const CLAIM: &str = "collector-observed-fresh-process-wall-time-and-container-checked-outputs-not-attested-execution-or-quality"; +const SPEECH_CLAIM: &str = "collector-observed-http-reply-time-and-framing-checked-audio-not-attested-execution-or-quality"; +fn claim(runner: &Runner) -> &'static str { + match runner { + Runner::Cli(_) => CLAIM, + Runner::Speech(_) => SPEECH_CLAIM, + } +} const TOP_FILES: [&str; 7] = [ "workload.json", "runner.json", @@ -46,7 +54,8 @@ struct Receipt { workload_typed_sha256: String, runner_sha256: String, deployment_sha256: String, - program: Program, + #[serde(default, skip_serializing_if = "Option::is_none")] + program: Option, setup_program: Option, execution: Execution, markers: Option, @@ -114,6 +123,11 @@ enum Reason { CaptureFailed, CacheMutatedDuringMeasurement, CacheObservationIncomplete, + HttpStatus, + FinishNotStop, + PartialFrame, + SampleRateMismatch, + TransportFailed, } #[derive(Deserialize, Serialize, PartialEq)] #[serde(rename_all = "snake_case")] @@ -210,7 +224,10 @@ struct Run { cell: String, phase: Phase, seed: u64, - process: ProcessRecord, + #[serde(default, skip_serializing_if = "Option::is_none")] + process: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + reply: Option, markers: Option, eligible: bool, outputs: Vec, @@ -219,6 +236,22 @@ struct Run { cache_after: Vec, cache_mutated_during_measurement: Option, } +impl Run { + // Inspect admits exactly one of the process and reply records for the runner kind. + fn end_to_end_us(&self) -> u64 { + match (&self.process, &self.reply) { + (Some(process), _) => process.end_to_end_us, + (None, Some(reply)) => reply.end_to_end_us, + (None, None) => 0, + } + } + fn cancelled(&self) -> bool { + self.process + .as_ref() + .is_some_and(|process| process.status == Status::Cancelled) + || self.reply.as_ref().is_some_and(speech::Reply::cancelled) + } +} #[derive(Deserialize, Serialize)] #[serde(deny_unknown_fields)] struct Manifest { @@ -270,6 +303,9 @@ struct CellSummary { eligible: usize, ineligible_by_reason: BTreeMap, end_to_end_us: Option, + first_audio_us: Option, + /// Eligible measured speech replies without `X-Finish-Reason`; null for CLI runners. + finish_reason_unknown: Option, markers_by_status: BTreeMap, generation_us: Option, load_to_generation_us: Option, @@ -304,19 +340,24 @@ fn json_digest(value: &impl Serialize) -> Result { &serde_json::to_vec(value).map_err(|error| error.to_string())?, )) } -fn reservation(sample: &Scheduled<'_>, plan: &Plan, root: &Path) -> Result { +fn reservation( + sample: &Scheduled<'_>, + runner: &Cli, + workload: &Workload, + root: &Path, +) -> Result { Ok(Reservation { cell: sample.cell.id.clone(), phase: sample.phase, seed: sample.seed, args: render( - &plan.runner.args, + &runner.args, sample.request, sample.seed, &root.join(sample.directory()), - &plan.workload, + workload, )?, - env: plan.runner.env.clone(), + env: runner.env.clone(), }) } #[derive(Serialize)] @@ -334,7 +375,11 @@ struct Rates { } fn produced(run: &Run) -> (u128, u128) { - let mut audio_us = 0; + let mut audio_us = run + .reply + .as_ref() + .and_then(|reply| reply.audio.as_ref()) + .map_or(0, |audio| u128::from(audio.duration_us)); let mut frames = 0; for output in &run.outputs { let validation = &output.validation; @@ -350,7 +395,7 @@ fn rates<'a>(runs: impl Iterator) -> Rates { let (mut elapsed_us, mut audio_us, mut frames) = (0u128, 0u128, 0u128); for run in runs.filter(|run| run.eligible) { let (audio, video) = produced(run); - elapsed_us += u128::from(run.process.end_to_end_us); + elapsed_us += u128::from(run.end_to_end_us()); audio_us += audio; frames += video; } @@ -666,6 +711,7 @@ fn reason_for(status: Status) -> Option { struct Session<'a> { plan: &'a Plan, + runner: &'a Cli, receipt: &'a Receipt, receipt_sha256: &'a str, guard: &'a Guard, @@ -683,7 +729,6 @@ impl Session<'_> { .map(std::ffi::OsString::from) .collect::>(); let env = self - .plan .runner .env .iter() @@ -746,7 +791,7 @@ impl Session<'_> { "reservation.json", &SetupReservation { args: args.clone(), - env: self.plan.runner.env.clone(), + env: self.runner.env.clone(), }, )?; let mut process = self.launch( @@ -773,11 +818,11 @@ impl Session<'_> { } fn sample(&self, root: &Path, sample: &Scheduled<'_>, previous: &[Observation]) -> Result { let dir = root.join(sample.directory()); - prepare_directory(&dir, &self.plan.runner.args)?; - let reservation = reservation(sample, self.plan, root)?; + prepare_directory(&dir, &self.runner.args)?; + let reservation = reservation(sample, self.runner, &self.plan.workload, root)?; evidence::publish(&dir, "reservation.json", &reservation)?; let mut process = self.launch( - &self.receipt.program, + self.receipt.program.as_ref().ok_or("missing program")?, &reservation.args, self.plan.workload.limits.run_deadline_ms, &dir, @@ -786,7 +831,7 @@ impl Session<'_> { &dir, &self.plan.workload, sample.request, - &self.plan.runner.args, + &self.runner.args, false, )?; demote_outputs(&mut process, &outputs.reasons); @@ -809,11 +854,11 @@ impl Session<'_> { eligible: process.status == Status::Complete && cache_mutated_during_measurement == Some(false), markers: self - .plan .runner .markers .map(|_| markers::collect(&dir, process.end_to_end_us)), - process, + process: Some(process), + reply: None, outputs: outputs.retained, reasons: outputs.reasons, details: outputs.details, @@ -860,24 +905,25 @@ fn make_receipt( root: &Path, started_unix_ms: u64, ) -> Result { - Ok(Receipt { + let mut receipt = Receipt { version: 1, kind: "media-capture-v1".into(), - claim: CLAIM.into(), + claim: claim(&plan.runner).into(), support: SUPPORT.into(), slot, workload_sha256: admitted.workload_sha256, workload_typed_sha256: admitted.workload_typed_sha256, runner_sha256: admitted.runner_sha256, deployment_sha256: evidence::digest(deployment), - program: admitted.program, - setup_program: admitted.setup_program, - execution: plan.runner.execution, - markers: plan.runner.markers, - env: admitted.env, - cwd: admitted.cwd, - cache_declared_state: plan.runner.cache.declared_state, - cache_paths: admitted.cache_paths, + // A speech server runs outside the capture: no program, environment or caches. + program: None, + setup_program: None, + execution: Execution::DelegatesToResidentService, + markers: None, + env: BTreeMap::new(), + cwd: None, + cache_declared_state: CacheState::Unknown, + cache_paths: Vec::new(), collector: Collector { version: env!("CARGO_PKG_VERSION").into(), source: env!("GRILL_PERF_SOURCE").into(), @@ -891,7 +937,34 @@ fn make_receipt( }, started_unix_ms, projection_root: root.into(), - }) + }; + if let Admitted::Cli { + execution, + markers, + program, + setup_program, + env, + cwd, + cache_paths, + cache_declared_state, + .. + } = admitted.runner + { + receipt.program = Some(program); + receipt.setup_program = setup_program; + receipt.execution = execution; + receipt.markers = markers; + receipt.env = env; + receipt.cwd = cwd; + receipt.cache_declared_state = cache_declared_state; + receipt.cache_paths = cache_paths; + } + Ok(receipt) +} + +enum Sampler<'a> { + Cli(Session<'a>), + Speech(speech::Session<'a>), } pub(super) fn capture( @@ -935,18 +1008,29 @@ pub(super) fn capture( "cache-before.json", &cache::observe(&receipt.cache_paths), )?; - let session = Session { - plan: &plan, - receipt: &receipt, - receipt_sha256: &receipt_sha256, - guard: &guard, + let sampler = match &plan.runner { + Runner::Cli(runner) => Sampler::Cli(Session { + plan: &plan, + runner, + receipt: &receipt, + receipt_sha256: &receipt_sha256, + guard: &guard, + }), + Runner::Speech(server) => Sampler::Speech(speech::Session::new( + server, + &plan.workload, + &receipt_sha256, + )?), + }; + let setup = match &sampler { + Sampler::Cli(session) => session + .runner + .setup + .as_ref() + .map(|setup| session.setup(&root, setup)) + .transpose()?, + Sampler::Speech(_) => None, }; - let setup = plan - .runner - .setup - .as_ref() - .map(|setup| session.setup(&root, setup)) - .transpose()?; let after_setup = cache::observe(&receipt.cache_paths); evidence::publish(&root, "cache-after-setup.json", &after_setup)?; let setup_failed = setup @@ -958,7 +1042,10 @@ pub(super) fn capture( let previous = runs .last() .map_or(after_setup.as_slice(), |run| &run.cache_after); - runs.push(session.sample(&root, &sample, previous)?); + runs.push(match &sampler { + Sampler::Cli(session) => session.sample(&root, &sample, previous)?, + Sampler::Speech(session) => session.sample(&root, &sample)?, + }); pause(sample.cell.pause_between_runs_ms); } } @@ -1164,28 +1251,38 @@ fn output_counts_match( .zip(&workload.outputs) .all(|(count, output)| *count == expected_outputs(output, request, args, setup)) } -fn verify_run(dir: &Path, run: &Run, sample: &Scheduled<'_>, plan: &Plan) -> Result<()> { - if run.markers.is_some() != plan.runner.markers.is_some() { +fn verify_run( + dir: &Path, + run: &Run, + sample: &Scheduled<'_>, + runner: &Cli, + workload: &Workload, +) -> Result<()> { + let process = run + .process + .as_ref() + .filter(|_| run.reply.is_none()) + .ok_or("run record does not match its runner kind")?; + if run.markers.is_some() != runner.markers.is_some() { return Err("marker record declaration mismatch".into()); } if run .markers .as_ref() - .is_some_and(|record| *record != markers::collect(dir, run.process.end_to_end_us)) + .is_some_and(|record| *record != markers::collect(dir, process.end_to_end_us)) { return Err("retained evidence hash mismatch: markers".into()); } - let workload = &plan.workload; verify_process( dir, - &run.process, + process, &workload.limits, workload.limits.run_deadline_ms, )?; let counts = verify_outputs(dir, &run.outputs, workload)?; - let eligible = run.process.status == Status::Complete + let eligible = process.status == Status::Complete && run.reasons.is_empty() - && output_counts_match(&counts, workload, sample.request, &plan.runner.args, false) + && output_counts_match(&counts, workload, sample.request, &runner.args, false) && run.outputs.iter().all(|output| output.validation.valid()); if run.eligible != eligible { return Err("sample eligibility mismatch".into()); @@ -1200,27 +1297,40 @@ fn verify_run(dir: &Path, run: &Run, sample: &Scheduled<'_>, plan: &Plan) -> Res return Err("cache eligibility reason mismatch".into()); } } - if run.cell != sample.cell.id || run.phase != sample.phase || run.seed != sample.seed { - return Err("sample schedule mismatch".into()); - } Ok(()) } fn verify_receipt(receipt: &Receipt, plan: &Plan, deployment: &[u8]) -> Result { - if receipt.kind != "media-capture-v1" + let declared = match &plan.runner { + Runner::Cli(runner) => { + receipt.program.is_some() + && receipt.execution == runner.execution + && receipt.markers == runner.markers + && receipt.env == runner.env + && receipt.cache_declared_state == runner.cache.declared_state + && receipt.setup_program.is_some() == runner.setup.is_some() + } + Runner::Speech(_) => { + receipt.program.is_none() + && receipt.setup_program.is_none() + && receipt.execution == Execution::DelegatesToResidentService + && receipt.markers.is_none() + && receipt.env.is_empty() + && receipt.cwd.is_none() + && receipt.cache_declared_state == CacheState::Unknown + && receipt.cache_paths.is_empty() + } + }; + if !declared + || receipt.kind != "media-capture-v1" || receipt.version != 1 || receipt.support != SUPPORT - || receipt.claim != CLAIM + || receipt.claim != claim(&plan.runner) || receipt.workload_sha256 != evidence::digest(&plan.workload_bytes) || receipt.workload_typed_sha256 != json_digest(&plan.workload)? || receipt.runner_sha256 != evidence::digest(&plan.runner_bytes) || receipt.deployment_sha256 != evidence::digest(deployment) || !plan.workload.slots.contains(&receipt.slot) - || receipt.execution != plan.runner.execution - || receipt.markers != plan.runner.markers - || receipt.env != plan.runner.env - || receipt.cache_declared_state != plan.runner.cache.declared_state - || receipt.setup_program.is_some() != plan.runner.setup.is_some() { return Err("media receipt declaration mismatch".into()); } @@ -1234,7 +1344,12 @@ fn inspect_setup( receipt: &Receipt, manifest: &Manifest, ) -> Result> { - let Some(setup) = &plan.runner.setup else { + let Runner::Cli( + runner @ Cli { + setup: Some(setup), .. + }, + ) = &plan.runner + else { return Ok(None); }; let reservation: SetupReservation = verified_json(root, "setup/reservation.json", manifest)?; @@ -1248,7 +1363,7 @@ fn inspect_setup( if reservation != (SetupReservation { args, - env: plan.runner.env.clone(), + env: runner.env.clone(), }) { return Err("setup reservation projection mismatch".into()); @@ -1317,7 +1432,7 @@ fn load(root: &Path) -> Result { if manifest.runs > schedule(&plan.workload).count() { return Err("capture run count exceeds plan".into()); } - let expected_names = manifest_names(manifest.runs, plan.runner.setup.is_some()) + let expected_names = manifest_names(manifest.runs, plan.runner.setup().is_some()) .into_iter() .collect::>(); if manifest.files.keys().cloned().collect::>() != expected_names { @@ -1341,16 +1456,32 @@ fn load(root: &Path) -> Result { let mut runs: Vec = Vec::with_capacity(expected_runs); for sample in schedule(&plan.workload).take(expected_runs) { let prefix = sample.directory(); - let reserved: Reservation = - verified_json(root, &format!("{prefix}/reservation.json"), &manifest)?; - if reserved != reservation(&sample, &plan, &receipt.projection_root)? { + let reserved = format!("{prefix}/reservation.json"); + let projected = match &plan.runner { + Runner::Cli(runner) => { + verified_json::(root, &reserved, &manifest)? + == reservation(&sample, runner, &plan.workload, &receipt.projection_root)? + } + Runner::Speech(server) => { + verified_json::(root, &reserved, &manifest)? + == server.reservation(&sample)? + } + }; + if !projected { return Err("sample reservation projection mismatch".into()); } let run: Run = verified_json(root, &format!("{prefix}/run.json"), &manifest)?; if &run.receipt_sha256 != receipt_sha256 { return Err("run record is bound to a different capture receipt".into()); } - verify_run(&root.join(prefix), &run, &sample, &plan)?; + if run.cell != sample.cell.id || run.phase != sample.phase || run.seed != sample.seed { + return Err("sample schedule mismatch".into()); + } + let dir = root.join(prefix); + match &plan.runner { + Runner::Cli(runner) => verify_run(&dir, &run, &sample, runner, &plan.workload)?, + Runner::Speech(server) => server.verify(&dir, &run, &sample, &plan.workload.limits)?, + } let previous = runs .last() .map_or(after_setup.as_slice(), |run| &run.cache_after); @@ -1361,9 +1492,7 @@ fn load(root: &Path) -> Result { } runs.push(run); } - let cancelled = runs - .iter() - .any(|run| run.process.status == Status::Cancelled) + let cancelled = runs.iter().any(Run::cancelled) || setup .as_ref() .is_some_and(|setup| setup.process.status == Status::Cancelled); @@ -1375,7 +1504,7 @@ fn load(root: &Path) -> Result { } let process_us = runs .iter() - .map(|run| u128::from(run.process.end_to_end_us)) + .map(|run| u128::from(run.end_to_end_us())) .sum::() + setup .as_ref() @@ -1426,7 +1555,7 @@ fn inspection( Inspection { version: 1, kind: "media-inspection-v1".into(), - claim: CLAIM.into(), + claim: receipt.claim.clone(), support: SUPPORT.into(), status, execution: receipt.execution, @@ -1480,6 +1609,7 @@ fn hashes_by_seed<'a>(runs: impl Iterator) -> BTreeMap(cell: &Cell, runs: impl Iterator) -> CellSummar let times = measured .iter() .filter(|run| run.eligible) - .map(|run| run.process.end_to_end_us) + .map(|run| run.end_to_end_us()) .collect::>(); let end_to_end_us = summary(×); + let first_audio_us = summary( + &measured + .iter() + .filter(|run| run.eligible) + .filter_map(|run| run.reply.as_ref()?.first_audio_us) + .collect::>(), + ); + let finish_reason_unknown = measured.iter().any(|run| run.reply.is_some()).then(|| { + measured + .iter() + .filter(|run| { + run.eligible + && run + .reply + .as_ref() + .is_some_and(|reply| reply.finish_reason.is_none()) + }) + .count() + }); let marker_summary = |metric| { summary( &measured @@ -1542,6 +1691,8 @@ fn summarize<'a>(cell: &Cell, runs: impl Iterator) -> CellSummar eligible: times.len(), ineligible_by_reason: reasons, end_to_end_us, + first_audio_us, + finish_reason_unknown, markers_by_status, generation_us, load_to_generation_us, diff --git a/crates/grill-perf/src/media/capture/compare.rs b/crates/grill-perf/src/media/capture/compare.rs index eb07123..87f8c10 100644 --- a/crates/grill-perf/src/media/capture/compare.rs +++ b/crates/grill-perf/src/media/capture/compare.rs @@ -35,6 +35,7 @@ struct ArmSummary { seconds_per_image: Option, images_per_second_end_to_end: Option, images_per_second_generation: Option, + duration_ratio_bps: Option, #[serde(flatten)] rates: Rates, } @@ -59,17 +60,12 @@ struct GateResult { } fn same_runner(a: &Loaded, b: &Loaded) -> bool { + fn identity(program: Option<&Program>) -> Option<(&PathBuf, &String)> { + program.map(|program| (&program.declared_path, &program.resolved_sha256)) + } a.plan.runner_bytes == b.plan.runner_bytes - && a.receipt.program.declared_path == b.receipt.program.declared_path - && a.receipt.program.resolved_sha256 == b.receipt.program.resolved_sha256 - && a.receipt - .setup_program - .as_ref() - .map(|program| (&program.declared_path, &program.resolved_sha256)) - == b.receipt - .setup_program - .as_ref() - .map(|program| (&program.declared_path, &program.resolved_sha256)) + && identity(a.receipt.program.as_ref()) == identity(b.receipt.program.as_ref()) + && identity(a.receipt.setup_program.as_ref()) == identity(b.receipt.setup_program.as_ref()) && a.deployment == b.deployment } @@ -114,7 +110,7 @@ fn admit(captures: &[Loaded], arms: &[Arm]) -> Result<()> { Ok(()) } -fn arm_summary(cell: &Cell, workload: &Workload, runs: &[&Run]) -> ArmSummary { +fn arm_summary(cell: &Cell, workload: &Workload, runs: &[&Run], ratios: &[Rational]) -> ArmSummary { let samples = summarize(cell, runs.iter().copied()); let images = cell .request @@ -138,6 +134,16 @@ fn arm_summary(cell: &Cell, workload: &Workload, runs: &[&Run]) -> ArmSummary { .filter(|us| *us > 0.0) .zip(images) .map(|(us, images)| images * 1_000_000.0 / us), + // Display only, floored to whole basis points; gates use the exact ratios. + duration_ratio_bps: summary( + &ratios + .iter() + .map(|ratio| { + let bps = u128::from(ratio.numerator) * 10_000 / u128::from(ratio.denominator); + u64::try_from(bps).unwrap_or(u64::MAX) + }) + .collect::>(), + ), rates: rates( runs.iter() .copied() @@ -147,6 +153,42 @@ fn arm_summary(cell: &Cell, workload: &Workload, runs: &[&Run]) -> ArmSummary { } } +fn eligible_measured<'a>(runs: &[&'a Run]) -> impl Iterator { + runs.iter() + .copied() + .filter(|run| run.phase == Phase::Measured && run.eligible) +} + +/// Each eligible measured sample's audio duration over the reference median for its seed. +fn duration_ratios(reference: &[&Run], runs: &[&Run]) -> Vec { + let mut durations = BTreeMap::>::new(); + for run in eligible_measured(reference) { + durations.entry(run.seed).or_default().push(produced(run).0); + } + let medians = durations + .into_iter() + .map(|(seed, mut values)| { + values.sort_unstable(); + let middle = values.len() / 2; + let median = if values.len() % 2 == 1 { + (values[middle], 1) + } else { + (values[middle - 1] + values[middle], 2) + }; + (seed, median) + }) + .collect::>(); + eligible_measured(runs) + .filter_map(|run| { + let (sum, count) = medians.get(&run.seed)?; + Some(Rational { + numerator: u64::try_from(produced(run).0 * count).ok()?, + denominator: u64::try_from(*sum).ok().filter(|sum| *sum > 0)?, + }) + }) + .collect() +} + fn hash_equality(reference: &[&Run], candidate: &[&Run]) -> Vec { let a = hashes_by_seed(reference.iter().copied()); let b = hashes_by_seed(candidate.iter().copied()); @@ -195,25 +237,25 @@ fn gate_result(gate: &Gate, captures: &[Loaded]) -> GateResult { .filter(|run| run.cell == gate.cell), ); } - let values = arms.each_ref().map(|runs| { - runs.iter() - .filter(|run| run.phase == Phase::Measured && run.eligible) - .filter_map(|run| match gate.metric { - Metric::EndToEnd => Some(Rational { - numerator: run.process.end_to_end_us, - denominator: 1, - }), - metric => run - .markers - .as_ref()? - .delta_ns(metric) - .map(|numerator| Rational { - numerator, - denominator: 1000, - }), - }) - .collect::>() - }); + let ratios = arms.each_ref().map(|runs| duration_ratios(&arms[0], runs)); + let values = match gate.metric { + Metric::DurationRatio => ratios.clone(), + metric => arms.each_ref().map(|runs| { + eligible_measured(runs) + .filter_map(|run| match metric { + Metric::EndToEnd => Some((run.end_to_end_us(), 1).into()), + Metric::FirstAudio => { + run.reply.as_ref()?.first_audio_us.map(|us| (us, 1).into()) + } + marker => run + .markers + .as_ref()? + .delta_ns(marker) + .map(|numerator| (numerator, 1000).into()), + }) + .collect::>() + }), + }; let mut result = GateResult { gate: gate.clone(), outcome: Outcome::Inconclusive, @@ -221,8 +263,8 @@ fn gate_result(gate: &Gate, captures: &[Loaded]) -> GateResult { reference: None, candidate: None, adverse_bounds: None, - reference_summary: arm_summary(cell, workload, &arms[0]), - candidate_summary: arm_summary(cell, workload, &arms[1]), + reference_summary: arm_summary(cell, workload, &arms[0], &ratios[0]), + candidate_summary: arm_summary(cell, workload, &arms[1], &ratios[1]), cross_arm_hash_equality: hash_equality(&arms[0], &arms[1]), }; match (envelope::range(&values[0]), envelope::range(&values[1])) { @@ -256,24 +298,46 @@ fn gate_result(gate: &Gate, captures: &[Loaded]) -> GateResult { != values.len() }) { - result.reason = Some("markers_unavailable".into()); + result.reason = Some( + match gate.metric { + Metric::Generation | Metric::LoadToGeneration => "markers_unavailable", + _ => "audio_unavailable", + } + .into(), + ); return result; } let reference = result.reference.expect("at least three reference samples"); let candidate = result.candidate.expect("at least three candidate samples"); - match envelope::assess( - reference, - candidate, - gate.max_adverse_bps, - gate.max_reference_spread_bps, - Direction::LowerBetter, - ) { - Ok(assessment) => { - result.outcome = match assessment.decision { + // Shortened and stretched audio are both adverse for the duration ratio. + let directions: &[Direction] = if gate.metric == Metric::DurationRatio { + &[Direction::LowerBetter, Direction::HigherBetter] + } else { + &[Direction::LowerBetter] + }; + match directions + .iter() + .map(|direction| { + envelope::assess( + reference, + candidate, + gate.max_adverse_bps, + gate.max_reference_spread_bps, + *direction, + ) + }) + .collect::, _>>() + { + Ok(assessments) => { + let decisive = assessments + .iter() + .find(|assessment| assessment.decision == EnvelopeDecision::Regression) + .unwrap_or(&assessments[0]); + result.outcome = match decisive.decision { EnvelopeDecision::Pass => Outcome::Pass, EnvelopeDecision::Regression => Outcome::Regression, }; - result.adverse_bounds = Some(assessment.adverse_bounds); + result.adverse_bounds = Some(decisive.adverse_bounds); } Err(reason) => envelope_failure(&mut result, reason), } @@ -281,13 +345,14 @@ fn gate_result(gate: &Gate, captures: &[Loaded]) -> GateResult { } fn matching_pins(a: &Runner, b: &Runner) -> bool { - if a.pins.is_empty() || b.pins.is_empty() { - return true; - } fn pins(runner: &Runner) -> BTreeSet<&str> { - runner.pins.iter().map(|pin| pin.sha256.as_str()).collect() + match runner { + Runner::Cli(runner) => runner.pins.iter().map(|pin| pin.sha256.as_str()).collect(), + Runner::Speech(_) => BTreeSet::new(), + } } - pins(a) == pins(b) + let (a, b) = (pins(a), pins(b)); + a.is_empty() || b.is_empty() || a == b } pub(in crate::media) fn compare( diff --git a/crates/grill-perf/src/media/capture/speech.rs b/crates/grill-perf/src/media/capture/speech.rs new file mode 100644 index 0000000..9edf70c --- /dev/null +++ b/crates/grill-perf/src/media/capture/speech.rs @@ -0,0 +1,516 @@ +//! OpenAI-style speech servers: serial requests against a server the operator already runs. +use super::*; +use crate::wire; +use reqwest::header::{ACCEPT_ENCODING, CONTENT_TYPE, USER_AGENT}; + +const BODY: &str = "body.bin"; +const FINISH_REASON: &str = "x-finish-reason"; +/// The OpenAI-style speech `input` limit, counted in characters. +const INPUT_CHARS: usize = 4096; + +#[derive(Clone, Copy, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +enum Format { + Wav, + Pcm, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(in crate::media) struct Server { + version: u32, + id: String, + endpoint: String, + model: String, + #[serde(default, deserialize_with = "present")] + voice: Option, + response_format: Format, + pub(in crate::media) stream: bool, + #[serde(default, deserialize_with = "present")] + pcm: Option, +} + +/// Raw reply framing; the sample rate is the workload request's. +#[derive(Clone, Copy, Deserialize)] +#[serde(deny_unknown_fields)] +struct Pcm { + channels: u16, + sample_bytes: u16, +} + +#[derive(Serialize)] +struct Body<'a> { + model: &'a str, + input: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + voice: Option<&'a str>, + response_format: Format, + #[serde(skip_serializing_if = "std::ops::Not::not")] + stream: bool, +} + +#[derive(Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(super) struct Reservation { + cell: String, + phase: Phase, + seed: u64, + body: String, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Reply { + pub(super) end_to_end_us: u64, + pub(super) first_audio_us: Option, + http_status: Option, + headers: Vec<[String; 2]>, + /// `X-Finish-Reason` values; null when the server sent none, so the reason is unknown. + pub(super) finish_reason: Option, + pub(super) body: Stream, + /// Deadline, transport or cancellation outcome that ended the exchange early. + failure: Option, + pub(super) audio: Option, + wav_failure: Option, +} +impl Reply { + pub(super) fn cancelled(&self) -> bool { + matches!( + self.failure, + Some(Reason::Cancelled | Reason::CancelledBeforeLaunch) + ) + } +} + +struct Judged { + finish_reason: Option, + audio: Option, + wav_failure: Option, + reasons: Vec, +} + +fn sample_rate(request: &Request) -> u32 { + request + .sample_rate + .expect("speech requests require sample_rate") +} + +impl Server { + pub(in crate::media) fn validate(&self, w: &Workload) -> Result<()> { + if self.version != 1 { + return Err("runner.version must be 1".into()); + } + if !identifier(&self.id) { + return Err(format!("runner.id is invalid: {}", self.id)); + } + self.url()?; + for (field, value) in [ + ("model", Some(self.model.as_str())), + ("voice", self.voice.as_deref()), + ] { + if value.is_some_and(|v| v.is_empty() || v.chars().any(char::is_control)) { + return Err(format!("runner.{field} must be nonempty and control-free")); + } + } + if self.stream && self.response_format != Format::Pcm { + return Err("runner.stream requires response_format pcm".into()); + } + match (self.response_format, self.pcm) { + (Format::Pcm, Some(pcm)) if pcm.channels > 0 && pcm.sample_bytes > 0 => {} + (Format::Wav, None) => {} + _ => { + return Err( + "runner.pcm declares positive channels and sample_bytes exactly for response_format pcm" + .into(), + ); + } + } + if !w.outputs.is_empty() { + return Err("outputs must be empty: the reply body is a speech server's output".into()); + } + for (context, request) in + std::iter::once(("request".to_string(), &w.request)).chain(w.cells.iter().filter_map( + |cell| Some((format!("cell {}.request", cell.id), cell.request.as_ref()?)), + )) + { + if request.negative_prompt.is_some() || request.guidance_milli.is_some() { + return Err(format!( + "{context}: speech server requests accept only prompt, seeds and sample_rate" + )); + } + if request.prompt.chars().count() > INPUT_CHARS { + return Err(format!( + "{context}.prompt must be at most {INPUT_CHARS} characters for a speech server" + )); + } + } + Ok(()) + } + + pub(in crate::media) fn url(&self) -> Result { + wire::endpoint(&self.endpoint, true).map_err(|_| { + "runner.endpoint must be a control-free URL of at most 4096 bytes with a host and no credentials, query or fragment, using HTTPS or plain HTTP to a literal loopback address".into() + }) + } + + pub(in crate::media) fn body(&self, request: &Request) -> Result { + serde_json::to_string(&Body { + model: &self.model, + input: &request.prompt, + voice: self.voice.as_deref(), + response_format: self.response_format, + stream: self.stream, + }) + .map_err(|error| error.to_string()) + } + + pub(super) fn reservation(&self, sample: &Scheduled<'_>) -> Result { + Ok(Reservation { + cell: sample.cell.id.clone(), + phase: sample.phase, + seed: sample.seed, + body: self.body(sample.request)?, + }) + } + + /// Derives audio and eligibility from retained facts only, so inspect repeats it offline. + fn judge(&self, reply: &Reply, body: &[u8], sample_rate: u32, deadline_ms: u32) -> Judged { + let values = reply + .headers + .iter() + .filter(|[name, _]| name == FINISH_REASON) + .map(|[_, value]| value.as_str()) + .collect::>(); + let mut judged = Judged { + finish_reason: (!values.is_empty()).then(|| values.join(", ")), + audio: None, + wav_failure: None, + reasons: reply.failure.into_iter().collect(), + }; + let reasons = &mut judged.reasons; + if reply.end_to_end_us >= u64::from(deadline_ms) * 1000 { + reasons.push(Reason::DeadlineExceeded); + } + if reply.body.truncated { + reasons.push(Reason::OutputOversized); + } + if reply.http_status.is_some_and(|status| status != 200) { + reasons.push(Reason::HttpStatus); + } + if values.iter().any(|value| *value != "stop") { + reasons.push(Reason::FinishNotStop); + } + if reply.failure.is_none() && !reply.body.truncated && reply.http_status == Some(200) { + let audio = match self.pcm { + None => crate::wavfile::check(body).map_err(|failure| { + judged.wav_failure = Some(failure); + Reason::OutputInvalid + }), + Some(pcm) => pcm_audio(body, pcm, sample_rate), + }; + match audio { + Ok(audio) => { + if audio.sample_rate != sample_rate { + reasons.push(Reason::SampleRateMismatch); + } + if audio.duration_us == 0 { + reasons.push(Reason::OutputMissing); + } + judged.audio = Some(audio); + } + Err(reason) => reasons.push(reason), + } + } + reasons.sort(); + reasons.dedup(); + judged + } + + pub(super) fn verify( + &self, + dir: &Path, + run: &Run, + sample: &Scheduled<'_>, + limits: &Limits, + ) -> Result<()> { + let reply = run + .reply + .as_ref() + .filter(|_| { + run.process.is_none() + && run.markers.is_none() + && run.outputs.is_empty() + && run.cache_after.is_empty() + }) + .ok_or("run record does not match its runner kind")?; + let cap = + usize::try_from(limits.output_bytes).map_err(|_| "output cap exceeds platform")?; + let body = evidence::read(&dir.join(BODY), cap)?; + if body.len() != reply.body.bytes || evidence::digest(&body) != reply.body.sha256 { + return Err("retained reply body hash mismatch".into()); + } + let streamed_audio = self.stream && reply.http_status == Some(200) && !body.is_empty(); + if !matches!( + reply.failure, + None | Some( + Reason::DeadlineExceeded + | Reason::TransportFailed + | Reason::Cancelled + | Reason::CancelledBeforeLaunch + ) + ) || (reply.failure.is_none() && reply.http_status.is_none()) + || reply.first_audio_us.is_some() != streamed_audio + || reply + .first_audio_us + .is_some_and(|us| us > reply.end_to_end_us) + || (reply.body.truncated && body.len() != cap) + { + return Err("reply record mismatch".into()); + } + let judged = self.judge( + reply, + &body, + sample_rate(sample.request), + limits.run_deadline_ms, + ); + if reply.finish_reason != judged.finish_reason + || reply.audio != judged.audio + || reply.wav_failure != judged.wav_failure + || run.reasons != judged.reasons + || run.eligible != run.reasons.is_empty() + { + return Err("reply eligibility mismatch".into()); + } + Ok(()) + } +} + +fn pcm_audio( + body: &[u8], + pcm: Pcm, + sample_rate: u32, +) -> std::result::Result { + let frame = usize::from(pcm.channels) * usize::from(pcm.sample_bytes); + if !body.len().is_multiple_of(frame) { + return Err(Reason::PartialFrame); + } + // Bodies are bounded by memory, so whole frames times 10^6 fit in u64. + let frames = (body.len() / frame) as u64; + Ok(crate::wavfile::Checked { + channels: pcm.channels, + sample_rate, + duration_us: frames * 1_000_000 / u64::from(sample_rate), + }) +} + +#[derive(Default)] +struct Exchanged { + end_to_end_us: u64, + first_audio_us: Option, + http_status: Option, + headers: Vec<[String; 2]>, + body: Vec, + truncated: bool, + failure: Option, + detail: Option, +} + +async fn exchange( + client: &reqwest::Client, + request: reqwest::Request, + deadline: Duration, + cap: usize, + stream: bool, +) -> Exchanged { + let mut exchanged = Exchanged::default(); + let cancelled = async { + while !process::program_signals::cancelled() { + tokio::time::sleep(process::PROGRAM_POLL).await; + } + }; + tokio::pin!(cancelled); + let start = Instant::now(); + let expired = tokio::time::sleep_until(tokio::time::Instant::from_std(start + deadline)); + tokio::pin!(expired); + let mut response = tokio::select! { + biased; + response = client.execute(request) => match response { + Ok(response) => response, + Err(error) => { + exchanged.end_to_end_us = wire::us(start); + exchanged.failure = Some(Reason::TransportFailed); + exchanged.detail = Some(wire::error_kind(&error).into()); + return exchanged; + } + }, + () = &mut expired => { + exchanged.end_to_end_us = wire::us(start); + exchanged.failure = Some(Reason::DeadlineExceeded); + return exchanged; + } + () = &mut cancelled => { + exchanged.end_to_end_us = wire::us(start); + exchanged.failure = Some(Reason::Cancelled); + return exchanged; + } + }; + exchanged.http_status = Some(response.status().as_u16()); + let audio = stream && exchanged.http_status == Some(200); + // Keep received chunks as they arrive; copying waits until timing ends. + let mut chunks = Vec::new(); + let mut bytes = 0; + loop { + let chunk = tokio::select! { + biased; + chunk = response.chunk() => chunk, + () = &mut expired => { + exchanged.failure = Some(Reason::DeadlineExceeded); + break; + } + () = &mut cancelled => { + exchanged.failure = Some(Reason::Cancelled); + break; + } + }; + match chunk { + Ok(Some(chunk)) => { + if audio && exchanged.first_audio_us.is_none() && !chunk.is_empty() { + exchanged.first_audio_us = Some(wire::us(start)); + } + if chunk.len() > cap - bytes { + chunks.push(chunk.slice(..cap - bytes)); + exchanged.truncated = true; + break; + } + bytes += chunk.len(); + chunks.push(chunk); + } + Ok(None) => break, + Err(error) => { + exchanged.failure = Some(Reason::TransportFailed); + exchanged.detail = Some(wire::error_kind(&error).into()); + break; + } + } + } + exchanged.end_to_end_us = wire::us(start); + exchanged.headers = response + .headers() + .iter() + .map(|(name, value)| { + [ + name.as_str().into(), + String::from_utf8_lossy(value.as_bytes()).into_owned(), + ] + }) + .collect(); + exchanged.body = chunks.concat(); + exchanged +} + +pub(super) struct Session<'a> { + server: &'a Server, + workload: &'a Workload, + receipt_sha256: &'a str, + url: reqwest::Url, + client: reqwest::Client, + runtime: tokio::runtime::Runtime, +} +impl<'a> Session<'a> { + pub(super) fn new( + server: &'a Server, + workload: &'a Workload, + receipt_sha256: &'a str, + ) -> Result { + let url = server.url()?; + Ok(Self { + client: wire::client(url.scheme() == "http", 1)?, + runtime: tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("runtime initialization failed: {error}"))?, + url, + server, + workload, + receipt_sha256, + }) + } + + pub(super) fn sample(&self, root: &Path, sample: &Scheduled<'_>) -> Result { + let dir = root.join(sample.directory()); + evidence::fresh(&dir)?; + let reservation = self.server.reservation(sample)?; + evidence::publish(&dir, "reservation.json", &reservation)?; + let limits = &self.workload.limits; + let cap = usize::try_from(limits.output_bytes) + .map_err(|_| "output_bytes exceeds platform size")?; + let request = self + .client + .post(self.url.clone()) + .header(CONTENT_TYPE, "application/json") + .header(ACCEPT_ENCODING, "identity") + .header( + USER_AGENT, + concat!("grill-perf/", env!("CARGO_PKG_VERSION")), + ) + .body(reservation.body) + .build() + .map_err(|_| "request construction failed")?; + let exchanged = if process::program_signals::cancelled() { + Exchanged { + failure: Some(Reason::CancelledBeforeLaunch), + ..Exchanged::default() + } + } else { + self.runtime.block_on(exchange( + &self.client, + request, + Duration::from_millis(u64::from(limits.run_deadline_ms)), + cap, + self.server.stream, + )) + }; + evidence::write(&dir.join(BODY), &exchanged.body)?; + let mut reply = Reply { + end_to_end_us: exchanged.end_to_end_us, + first_audio_us: exchanged.first_audio_us, + http_status: exchanged.http_status, + headers: exchanged.headers, + finish_reason: None, + body: Stream { + sha256: evidence::digest(&exchanged.body), + bytes: exchanged.body.len(), + truncated: exchanged.truncated, + }, + failure: exchanged.failure, + audio: None, + wav_failure: None, + }; + let judged = self.server.judge( + &reply, + &exchanged.body, + sample_rate(sample.request), + limits.run_deadline_ms, + ); + reply.finish_reason = judged.finish_reason; + reply.audio = judged.audio; + reply.wav_failure = judged.wav_failure; + let run = Run { + receipt_sha256: self.receipt_sha256.into(), + cell: sample.cell.id.clone(), + phase: sample.phase, + seed: sample.seed, + process: None, + reply: Some(reply), + markers: None, + eligible: judged.reasons.is_empty(), + outputs: Vec::new(), + reasons: judged.reasons, + details: exchanged.detail.into_iter().collect(), + cache_after: Vec::new(), + cache_mutated_during_measurement: Some(false), + }; + evidence::publish(&dir, "run.json", &run)?; + Ok(run) + } +} diff --git a/crates/grill-perf/src/media/markers.rs b/crates/grill-perf/src/media/markers.rs index c96e41b..1d3a623 100644 --- a/crates/grill-perf/src/media/markers.rs +++ b/crates/grill-perf/src/media/markers.rs @@ -80,7 +80,7 @@ impl Markers { let (start, end) = match metric { Metric::Generation => (Name::GenerationStarted, Name::GenerationFinished), Metric::LoadToGeneration => (Name::RunnerStarted, Name::GenerationStarted), - Metric::EndToEnd => return None, + Metric::EndToEnd | Metric::FirstAudio | Metric::DurationRatio => return None, }; let stamp = |name| { self.events diff --git a/crates/grill-perf/src/wavfile.rs b/crates/grill-perf/src/wavfile.rs index afb86f5..ad86300 100644 --- a/crates/grill-perf/src/wavfile.rs +++ b/crates/grill-perf/src/wavfile.rs @@ -1,7 +1,8 @@ //! Container framing and sample metadata only; sample values are not decoded. //! Durations use integer division, rounding down to whole microseconds. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] pub(crate) struct Checked { pub channels: u16, pub sample_rate: u32, diff --git a/crates/grill-perf/tests/support/media.rs b/crates/grill-perf/tests/support/media.rs index af3d65e..0576333 100644 --- a/crates/grill-perf/tests/support/media.rs +++ b/crates/grill-perf/tests/support/media.rs @@ -4,6 +4,8 @@ use std::os::unix::fs::{PermissionsExt, symlink}; #[path = "mp4_fixture.rs"] mod mp4_fixture; use mp4_fixture::{TrackSpec, progressive}; +#[path = "media_speech.rs"] +mod speech; struct Fixture { root: Temp, diff --git a/crates/grill-perf/tests/support/media_speech.rs b/crates/grill-perf/tests/support/media_speech.rs new file mode 100644 index 0000000..f1ec09b --- /dev/null +++ b/crates/grill-perf/tests/support/media_speech.rs @@ -0,0 +1,370 @@ +//! Speech-server runner fixtures: a loopback server answering by input text and voice. +use super::*; +use crate::Server; +use std::io::Write; +use std::net::TcpStream; +use std::thread; +use std::time::Duration; + +const RATE: u32 = 8000; + +fn wav(frames: usize) -> Vec { + let data = frames * 2; + let mut bytes = b"RIFF".to_vec(); + bytes.extend((36 + data as u32).to_le_bytes()); + bytes.extend(b"WAVEfmt "); + bytes.extend(16u32.to_le_bytes()); + bytes.extend(1u16.to_le_bytes()); + bytes.extend(1u16.to_le_bytes()); + bytes.extend(RATE.to_le_bytes()); + bytes.extend((RATE * 2).to_le_bytes()); + bytes.extend(2u16.to_le_bytes()); + bytes.extend(16u16.to_le_bytes()); + bytes.extend(b"data"); + bytes.extend((data as u32).to_le_bytes()); + bytes.extend(vec![0; data]); + bytes +} + +// Writes are best-effort: the collector closes timed-out and oversized replies. +fn reply(mut stream: TcpStream, status: u16, finish: &str, delay_ms: u64, chunks: &[&[u8]]) { + let _ = write!( + stream, + "HTTP/1.1 {status} X\r\nConnection: close\r\nX-Finish-Reason: {finish}\r\nTransfer-Encoding: chunked\r\n\r\n" + ); + thread::sleep(Duration::from_millis(delay_ms)); + for chunk in chunks { + let _ = write!(stream, "{:x}\r\n", chunk.len()); + let _ = stream.write_all(chunk); + let _ = stream.write_all(b"\r\n"); + let _ = stream.flush(); + } + let _ = stream.write_all(b"0\r\n\r\n"); +} + +/// Input text selects the reply shape; voice selects first-audio delay and length. +fn speech_server() -> Server { + Server::new(|mut stream, _, request| { + let (delay_ms, frames) = match request["voice"].as_str() { + Some("slow") => (300, 800), + Some("short") => (0, 400), + Some("long") => (0, 1600), + _ => (150, 800), + }; + let pcm = vec![0; frames * 2]; + let (first, rest) = pcm.split_at(frames); + match request["input"].as_str().unwrap() { + "wav" => { + let body = wav(frames); + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: {}\r\n\r\n", + body.len() + ); + let _ = stream.write_all(&body); + } + "status" => reply(stream, 422, "stop", 0, &[br#"{"error":"refused"}"#]), + "length" => reply(stream, 200, "length", 0, &[first, rest]), + "partial" => reply(stream, 200, "stop", 0, &[&[0; 3]]), + "slow" => reply(stream, 200, "stop", 1500, &[first]), + _ => reply(stream, 200, "stop", delay_ms, &[first, rest]), + } + }) +} + +fn speech_fixture(server: &Server, input: &str) -> Fixture { + let mut f = Fixture::new(); + f.workload = json!({ + "kind":"media-workload-v1","version":1,"name":"speech","modality":"audio", + "request":{"prompt":input,"seeds":[7],"sample_rate":RATE}, + "outputs":[], + "cells":[{"id":"c1","warmup_runs":1,"measured_runs":3}], + "slots":[{"id":"a","arm":"a"},{"id":"b","arm":"b"},{"id":"a2","arm":"a2"}], + "limits":{"run_deadline_ms":500,"setup_deadline_ms":1,"output_bytes":65536,"stdout_bytes":1,"stderr_bytes":1}, + "gates":[{"cell":"c1","metric":"end_to_end_us","max_adverse_bps":500,"max_reference_spread_bps":1000000}] + }); + f.runner = json!({ + "kind":"media-speech-server-v1","version":1,"id":"speech", + "endpoint":server.endpoint.replace("chat/completions", "audio/speech"), + "model":"fixture/speech-1","response_format":"pcm","stream":true, + "pcm":{"channels":1,"sample_bytes":2} + }); + fs::write( + f.root.path("deployment.json"), + br#"{"model_revision":"fixture","runtime":"loopback","hardware":"synthetic","settings":"local"}"#, + ) + .unwrap(); + f +} + +#[test] +fn speech_preflight_renders_request_bodies_and_closes_the_contract() { + let server = speech_server(); + let mut f = speech_fixture(&server, "Hello there."); + f.runner["voice"] = json!("alice"); + let report = f.report(); + assert_eq!(report["endpoint"], f.runner["endpoint"]); + assert!(report.get("program").is_none()); + assert_eq!(report["total_runs"], 4); + assert_eq!(report["wall_clock_allowance_ms"], 2000); + assert_eq!( + report["samples"][0]["body"], + r#"{"model":"fixture/speech-1","input":"Hello there.","voice":"alice","response_format":"pcm","stream":true}"# + ); + assert_eq!(server.count.load(std::sync::atomic::Ordering::SeqCst), 0); + let runner = f.runner.clone(); + let workload = f.workload.clone(); + for (change, expected) in [ + ( + json!({"endpoint":"http://192.0.2.1/v1/audio/speech"}), + "plain HTTP to a literal loopback address", + ), + ( + json!({"response_format":"wav"}), + "runner.stream requires response_format pcm", + ), + (json!({"pcm":{"channels":0,"sample_bytes":2}}), "runner.pcm"), + (json!({"model":""}), "runner.model"), + (json!({"probe":true}), "unknown field"), + (json!({"kind":"media-speech-v0"}), "unknown variant"), + ] { + f.runner = runner.clone(); + for (key, value) in change.as_object().unwrap() { + f.runner[key] = value.clone(); + } + f.reject(expected); + } + f.runner = runner.clone(); + f.runner.as_object_mut().unwrap().remove("pcm"); + f.reject("runner.pcm"); + f.runner = runner.clone(); + f.runner["stream"] = json!(false); + f.workload["gates"][0]["metric"] = json!("first_audio_us"); + f.reject("first_audio_us requires a streaming speech server runner"); + for (pointer, value, expected) in [ + ( + "/request/duration_us", + json!(1000), + "request.duration_us is forbidden", + ), + ( + "/request/negative_prompt", + json!("noise"), + "accept only prompt, seeds and sample_rate", + ), + ( + "/outputs", + json!([{"placeholder":"output","container":"wav","expect":{"duration_us":1,"sample_rate":RATE,"channels":1}}]), + "outputs must be empty", + ), + ("/modality", json!("image"), "require audio modality"), + ] { + f.runner = runner.clone(); + f.workload = workload.clone(); + let (parent, key) = pointer.rsplit_once('/').unwrap(); + f.workload.pointer_mut(parent).unwrap()[key] = value; + f.reject(expected); + } + f.workload = workload.clone(); + f.workload["request"] + .as_object_mut() + .unwrap() + .remove("sample_rate"); + f.reject("request: missing field `sample_rate`"); + // The speech input bound counts characters: 4096 two-byte characters fit. + f.workload = workload.clone(); + f.workload["request"]["prompt"] = json!("é".repeat(4096)); + f.report(); + f.workload["cells"][0]["request"] = + json!({"prompt":"é".repeat(4097),"seeds":[7],"sample_rate":RATE}); + f.reject("cell c1.request.prompt must be at most 4096 characters"); +} + +#[test] +fn speech_capture_measures_whole_wav_and_streamed_first_audio() { + let server = speech_server(); + let mut f = speech_fixture(&server, "wav"); + f.runner["response_format"] = json!("wav"); + f.runner["stream"] = json!(false); + f.runner.as_object_mut().unwrap().remove("pcm"); + let report = f.captured(); + assert_eq!( + report["claim"], + "collector-observed-http-reply-time-and-framing-checked-audio-not-attested-execution-or-quality" + ); + assert_eq!(report["execution"], "delegates-to-resident-service"); + assert_eq!(report["cells"][0]["eligible"], 3); + let run = &report["runs"][1]; + let reply = &run["reply"]; + assert!(run.get("process").is_none()); + assert_eq!(reply["http_status"], 200); + assert!(reply["first_audio_us"].is_null()); + assert_eq!( + reply["audio"], + json!({"channels":1,"sample_rate":RATE,"duration_us":100_000}) + ); + assert_eq!(reply["body"]["bytes"], 1644); + assert!(reply["finish_reason"].is_null()); + assert_eq!(report["cells"][0]["finish_reason_unknown"], 3); + let body = fs::read(f.root.path("capture/run-000001/body.bin")).unwrap(); + assert_eq!(reply["body"]["sha256"], super::digest(&body)); + let end_to_end = reply["end_to_end_us"].as_u64().unwrap(); + assert_eq!(run["realtime_factor"], end_to_end as f64 / 100_000.0); + let inspected = f.inspect_capture(); + successful(&inspected); + assert_eq!( + report, + serde_json::from_slice::(&inspected.stdout).unwrap() + ); + + let f = speech_fixture(&server, "pcm"); + let report = f.captured(); + assert_eq!(report["cells"][0]["eligible"], 3); + let reply = &report["runs"][1]["reply"]; + let first = reply["first_audio_us"].as_u64().unwrap(); + // The fixture sends headers at once and the first PCM byte 150 ms later. + assert!(first >= 150_000, "{first}"); + assert!(first <= reply["end_to_end_us"].as_u64().unwrap()); + assert_eq!(reply["audio"]["duration_us"], 100_000); + assert!( + reply["headers"] + .as_array() + .unwrap() + .contains(&json!(["x-finish-reason", "stop"])) + ); + assert_eq!(report["cells"][0]["first_audio_us"]["n"], 3); + assert_eq!(reply["finish_reason"], "stop"); + assert_eq!(report["cells"][0]["finish_reason_unknown"], 0); + successful(&f.inspect_capture()); +} + +#[test] +fn speech_failures_are_retained_ineligible_and_rechecked_offline() { + let server = speech_server(); + let mut f = speech_fixture(&server, "status"); + let cells = ["status", "length", "partial", "slow"]; + f.workload["cells"] = cells + .iter() + .map(|input| json!({"id":input,"warmup_runs":0,"measured_runs":1,"request":{"prompt":input,"seeds":[7],"sample_rate":RATE}})) + .collect(); + f.workload["gates"][0]["cell"] = json!("status"); + let report = f.captured(); + assert_eq!(report["status"], "complete"); + for (index, reason) in [ + "http_status", + "finish_not_stop", + "partial_frame", + "deadline_exceeded", + ] + .into_iter() + .enumerate() + { + let run = &report["runs"][index]; + assert_eq!(run["eligible"], false, "{run}"); + assert_eq!(run["reasons"], json!([reason]), "{run}"); + assert_eq!(report["cells"][index]["ineligible_by_reason"][reason], 1); + } + assert_eq!(report["runs"][0]["reply"]["http_status"], 422); + assert_eq!( + fs::read(f.root.path("capture/run-000000/body.bin")).unwrap(), + br#"{"error":"refused"}"# + ); + assert_eq!(report["runs"][1]["reply"]["audio"]["duration_us"], 100_000); + assert_eq!(report["runs"][1]["reply"]["finish_reason"], "length"); + assert_eq!(report["runs"][2]["reply"]["body"]["bytes"], 3); + let slow = &report["runs"][3]["reply"]; + assert_eq!(slow["failure"], "deadline_exceeded"); + assert!(slow["end_to_end_us"].as_u64().unwrap() >= 500_000); + assert!(slow["http_status"] == 200 && slow["body"]["bytes"] == 0); + let inspected = f.inspect_capture(); + successful(&inspected); + assert_eq!( + report, + serde_json::from_slice::(&inspected.stdout).unwrap() + ); + + let body = f.root.path("capture/run-000000/body.bin"); + let original = fs::read(&body).unwrap(); + fs::write(&body, b"{}").unwrap(); + inspection_rejected(&f, "retained reply body hash mismatch"); + fs::write(&body, original).unwrap(); + rewrite_evidence(&f, "run-000001/run.json", |run| { + run["reply"]["headers"] = json!([["x-finish-reason", "stop"]]); + }); + inspection_rejected(&f, "reply eligibility mismatch"); +} + +#[test] +fn speech_compare_gates_first_audio_and_two_sided_duration_ratio() { + let server = speech_server(); + let mut f = speech_fixture(&server, "pcm"); + f.workload["cells"][0]["warmup_runs"] = json!(0); + f.workload["gates"] = json!([ + {"cell":"c1","metric":"first_audio_us","max_adverse_bps":500,"max_reference_spread_bps":1000000}, + {"cell":"c1","metric":"duration_ratio","max_adverse_bps":500,"max_reference_spread_bps":0} + ]); + // A 300 ms first-audio delay against none keeps the verdict scheduler-proof. + for (out, slot, voice) in [ + ("a", "a", "slow"), + ("b", "b", "fast"), + ("a2", "a2", "slow"), + ("b-short", "b", "short"), + ("a2-again", "a2", "slow"), + ("b-long", "b", "long"), + ("a2-third", "a2", "slow"), + ] { + f.runner["voice"] = json!(voice); + f.save(); + successful( + &cli() + .args(["media", "capture", "--workload"]) + .arg(f.root.path("workload.json")) + .arg("--runner") + .arg(f.root.path("runner.json")) + .arg("--deployment") + .arg(f.root.path("deployment.json")) + .args(["--slot", slot, "--out"]) + .arg(f.root.path(out)) + .output() + .unwrap(), + ); + } + let report = f.comparison(false, 0); + assert_eq!(report["outcome"], "PASS"); + let [first_audio, ratio] = [&report["gates"][0], &report["gates"][1]]; + assert_eq!(first_audio["outcome"], "PASS"); + assert_eq!(first_audio["reference_summary"]["first_audio_us"]["n"], 6); + assert_eq!(first_audio["candidate_summary"]["finish_reason_unknown"], 0); + assert_eq!(ratio["outcome"], "PASS"); + assert_eq!( + ratio["candidate_summary"]["duration_ratio_bps"]["min"], + 10_000 + ); + assert_eq!(ratio["cross_arm_hash_equality"][0]["equal"], true); + + // Shortened and stretched speech both regress, whatever the first-audio gain. + for (b, a2, ratio_bps) in [ + ("b-short", "a2-again", 5_000), + ("b-long", "a2-third", 20_000), + ] { + let output = cli() + .args(["media", "compare", "--a"]) + .arg(f.root.path("a")) + .arg("--b") + .arg(f.root.path(b)) + .arg("--a2") + .arg(f.root.path(a2)) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(3)); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["gates"][0]["outcome"], "PASS"); + let ratio = &report["gates"][1]; + assert_eq!(ratio["outcome"], "REGRESSION"); + assert_eq!( + ratio["candidate_summary"]["duration_ratio_bps"]["max"], + ratio_bps + ); + assert_eq!(ratio["cross_arm_hash_equality"][0]["equal"], false); + } +} diff --git a/docs/performance/MEDIA.md b/docs/performance/MEDIA.md index cad3746..61caadd 100644 --- a/docs/performance/MEDIA.md +++ b/docs/performance/MEDIA.md @@ -3,7 +3,8 @@ `media preflight`, `media capture`, `media inspect` and `media compare` are experimental and support image, audio and video workloads on Linux and macOS. External probes are not supported. -Preflight prints JSON without starting processes or creating directories. +Preflight prints JSON without starting processes, contacting servers or creating +directories. ```text grill-perf media preflight --workload W.json --runner R.json @@ -18,7 +19,8 @@ Exit codes: **0** admitted; **1** rejected, unreadable or CLI usage error. ## Documents -Both documents require their `kind` (`media-workload-v1` or `media-runner-v1`) +Both documents require their `kind` (`media-workload-v1`; for runners +`media-runner-v1` or [`media-speech-server-v1`](#speech-servers)) and `version: 1`. Records are JSON objects; unknown/duplicate fields and positional records are rejected. Documents are bounded to 4 MiB; resolved programs to 1 GiB. Optional fields are omitted, not null; absent optional workload fields are omitted @@ -41,7 +43,8 @@ Cells (1–64) declare `id`, `warmup_runs` (0–20), `measured_runs` (1–100), whole request. Slots declare unique `id` and `arm`: `a`, `b`, `a2`, optionally `b2`. Gates (1–16) name unique `(cell, metric)` pairs with a declared cell, `max_adverse_bps` (0–9999) and `max_reference_spread_bps` (0–1000000). -Metrics are `end_to_end_us`, `generation_us` and `load_to_generation_us`. +Metrics are `end_to_end_us`, `generation_us`, `load_to_generation_us`, +`first_audio_us` and `duration_ratio` ([speech servers](#speech-servers)). Limits are positive `run_deadline_ms`, `setup_deadline_ms`, `output_bytes`, `stdout_bytes` and `stderr_bytes`. @@ -54,13 +57,13 @@ duration. `audio: true` requires an audio track with nonzero samples, sample byt and media duration; `false` requires its absence. Expectations use the request bounds and exact equality. WAV and MP4 media durations round down to whole microseconds. -At least one output is required. The declared set must equal the output placeholders +CLI runners require at least one output. The declared set must equal the output placeholders used in sample argv. Setup arguments may use only declared output placeholders but need not use any. `{output}` occurs at most once in each argv; samples with multiple images require only `{output_dir}`. The file extension comes from the `output` declaration, not the directory declaration. -The per-arm runner owns `id`, `program`, `args`, `env`, `execution`, `cache`, +The per-arm CLI runner owns `id`, `program`, `args`, `env`, `execution`, `cache`, `pins`, optional `cwd` and optional `setup`. `execution` is `in-process`, `delegates-to-resident-service` or `unknown`. Programs must be executable regular files; relative paths resolve against the invoking directory, not `cwd`, without @@ -214,7 +217,7 @@ repeat. These observations are descriptive, not a comparison verdict. Byte-identical MP4 output requires bit-exact muxing, including stable movie-header creation times. Each reported run includes `realtime_factor`: end-to-end time divided by checked -WAV duration, with values below one meaning faster than real time. +audio duration, with values below one meaning faster than real time. `frames_per_second_end_to_end` is checked video frames per second of end-to-end time. Rates are null for other modalities, ineligible runs and zero durations. These descriptive values add no gates. @@ -279,6 +282,90 @@ nondeterministic. Exit codes are **0** PASS, **1** ERROR/INVALID, **2** INCONCLUSIVE and **3** REGRESSION. +## Speech servers + +A `media-speech-server-v1` runner measures an OpenAI-style +`POST /v1/audio/speech` server that is already running; the Grill never starts, +stops or configures it. The capture's `--deployment` +[declaration](README.md#deployment-declarations-and-privacy) identifies the +server build, and receipts record `execution: delegates-to-resident-service`. + +The runner declares `id`, `endpoint`, `model`, optional `voice`, +`response_format` (`wav` or `pcm`), `stream` and, exactly for `pcm`, `pcm` with +positive `channels` and `sample_bytes`; `stream: true` requires `pcm`. The +endpoint uses HTTPS, or plain HTTP to a literal loopback address; no credentials +are sent. The workload has `modality: "audio"` and `outputs: []`, and its +requests accept only `prompt` (at most 4096 characters), `seeds` and the expected +reply `sample_rate`. +Each sample posts JSON with `model`, `input` (the prompt), `response_format`, +`voice` when declared and `"stream": true` when streaming. Seeds are not sent; +they only schedule samples and group determinism. Preflight and each reservation +record the exact body. + +Capture sends samples serially through one reused HTTP/1.1 client without +proxies, redirects or retries. `end_to_end_us` runs from dispatch to the end of +the reply body; for streamed 200 replies `first_audio_us` stops at the first body +byte. `run_deadline_ms` bounds each request and `output_bytes` the retained +`body.bin`; the other process limits are unused but must be positive. Each run +retains the status, every response header, the body hash and any deadline, +transport or cancellation outcome. Audio duration comes from the body: WAV through +the checks above, PCM as whole frames of `channels × sample_bytes` at the request +rate. + +Failed samples are retained, never retried, and ineligible with `http_status` +(not 200), `finish_not_stop` (an `X-Finish-Reason` other than `stop`), +`partial_frame`, `output_invalid` or `sample_rate_mismatch` (WAV), +`output_missing` (no audio), `output_oversized`, `deadline_exceeded`, +`transport_failed` or `cancelled`. Without that header, `finish_reason` is `null` +(unknown), the sample stays eligible and inspect and compare count it as +`finish_reason_unknown`, so servers that send the header, such as Drama, get the +stricter check automatically. Inspect rechecks body hashes and re-derives finish +reason, audio and reasons from the retained status, headers and body. + +`first_audio_us` gates are lower-is-better and admitted only for a streaming +speech runner, so every measured sample of every arm is streamed. +`duration_ratio` divides each eligible measured sample's audio duration by the +median A+A2 duration for the same cell and seed. Its gate, admitted for audio +workloads, applies the envelope rule in both directions with `max_adverse_bps` as +the allowed deviation, so shortened and stretched speech both regress. Arm +summaries report `duration_ratio_bps`, floored for display. Cross-arm hash +equality compares reply bodies. + +For example, streamed 16-bit mono PCM at 44.1 kHz (adjust the endpoint): + +```json +{"kind": "media-speech-server-v1", "version": 1, "id": "drama", + "endpoint": "http://127.0.0.1:8000/v1/audio/speech", + "model": "fishaudio/drama-3", "response_format": "pcm", "stream": true, + "pcm": {"channels": 1, "sample_bytes": 2}} +``` + +```json +{"kind": "media-workload-v1", "version": 1, "name": "speech-first-audio", + "modality": "audio", + "request": {"prompt": "The quick brown fox jumps over the lazy dog.", + "seeds": [1], "sample_rate": 44100}, + "outputs": [], + "cells": [{"id": "short", "warmup_runs": 2, "measured_runs": 10}], + "slots": [{"id": "a", "arm": "a"}, {"id": "b", "arm": "b"}, {"id": "a2", "arm": "a2"}], + "limits": {"run_deadline_ms": 60000, "setup_deadline_ms": 1, + "output_bytes": 67108864, "stdout_bytes": 1, "stderr_bytes": 1}, + "gates": [ + {"cell": "short", "metric": "first_audio_us", "max_adverse_bps": 500, "max_reference_spread_bps": 2000}, + {"cell": "short", "metric": "end_to_end_us", "max_adverse_bps": 500, "max_reference_spread_bps": 2000}, + {"cell": "short", "metric": "duration_ratio", "max_adverse_bps": 1000, "max_reference_spread_bps": 2000}]} +``` + +Capture `a` with the reference server and its deployment declaration, restart the +server as the candidate for `b`, return to the reference for `a2`, then compare: + +```text +grill-perf media capture --workload W.json --runner R.json --deployment A.json --slot a --out a +grill-perf media capture --workload W.json --runner R.json --deployment B.json --slot b --out b +grill-perf media capture --workload W.json --runner R.json --deployment A.json --slot a2 --out a2 +grill-perf media compare --a a --b b --a2 a2 +``` + ## Private evidence Prompts, program paths, declared env values and outputs are private evidence, diff --git a/docs/performance/RELEASE-NOTES.md b/docs/performance/RELEASE-NOTES.md index 783af77..923ca31 100644 --- a/docs/performance/RELEASE-NOTES.md +++ b/docs/performance/RELEASE-NOTES.md @@ -1,5 +1,15 @@ # Performance release notes +## Unreleased + +- Add a `media-speech-server-v1` runner to `grill-perf media` for an + already-running OpenAI-style `POST /v1/audio/speech` server: serial requests, + streamed `first_audio_us`, retained reply bodies and headers rechecked offline, + a `first_audio_us` gate and a two-sided `duration_ratio` gate against the + reference arm. See [Speech servers](MEDIA.md#speech-servers). +- Verification is limited to a loopback fixture server; no live speech server + was measured. + ## Version 0.10.0 — audio, video and runner markers - Add runner-asserted media markers, exact generation interval gates and offline