diff --git a/Cargo.lock b/Cargo.lock index 89ae91d..f67dc19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "async-trait" version = "0.1.92" @@ -949,6 +955,16 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "saphyr-parser" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c6220586c3dacac54bfddec87af86ddb31862d29666e7e35cf4baf70a25d387" +dependencies = [ + "arraydeque", + "thiserror 2.0.21", +] + [[package]] name = "schemars" version = "0.8.22" @@ -1179,11 +1195,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.21", ] [[package]] @@ -1199,9 +1215,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", @@ -1386,7 +1402,7 @@ dependencies = [ "serde", "serde_json", "syn 2.0.119", - "thiserror 2.0.19", + "thiserror 2.0.21", "unicode-ident", ] @@ -1557,12 +1573,13 @@ dependencies = [ "async-trait", "jsonschema", "prettyplease", + "saphyr-parser", "schemars", "serde", "serde_json", "serde_yaml_ng", "syn 2.0.119", - "thiserror 2.0.19", + "thiserror 2.0.21", "time", "time-tz", "tokio", diff --git a/crates/wavekat-flow/Cargo.toml b/crates/wavekat-flow/Cargo.toml index 163adad..f04c307 100644 --- a/crates/wavekat-flow/Cargo.toml +++ b/crates/wavekat-flow/Cargo.toml @@ -21,6 +21,12 @@ anyhow = "1" async-trait = "0.1" serde = { version = "1", features = ["derive"] } serde_json = "1" +# `source` edits a document's text in place, so it needs each value's +# position, which serde_yaml_ng doesn't expose. Only the event parser is +# used (pure Rust, two small deps). Pinned: 0.2.2's markers count chars, +# not the bytes its docs say, and `source` relies on that (its non-ASCII +# tests catch a change). +saphyr-parser = "=0.2.2" serde_yaml_ng = "0.10" thiserror = "2" time = { version = "0.3", features = ["macros", "parsing"] } diff --git a/crates/wavekat-flow/README.md b/crates/wavekat-flow/README.md index 2865012..91c45c9 100644 --- a/crates/wavekat-flow/README.md +++ b/crates/wavekat-flow/README.md @@ -29,6 +29,7 @@ this crate. |--------|--------------| | `model` (re-exported at root) | Generated document types (`Flow`, `Node`, `Prompt`, …) plus hand-written helpers (`Flow::from_yaml`, `Node::exits`, …) | | `validate` | Semantic validation: reachability, caller traps, exit-set exactness, DTMF and prompt rules | +| `source` | Find and rewrite prompts in a document's YAML text without disturbing comments or layout: voice text prompts as clips, swap local recordings for uploaded clips | | `hours` | Business-hours / timezone math (IANA zones, holiday exceptions) | | `engine` | The flow interpreter + the `FlowEffects` async trait your runtime implements | | `trace` | Serializable run traces emitted by the engine | @@ -97,8 +98,8 @@ meantime, which is the safe direction. ## One format, two languages -This crate is the Rust half of the format. Authoring, comment-preserving -edits, and the TypeScript twin of the validator live in +This crate is the Rust half of the format. General-purpose comment-preserving +edits (rename, add or remove a node, rewire an exit) and the TypeScript twin of the validator live in [`@wavekat/flow-schema`](https://www.npmjs.com/package/@wavekat/flow-schema). The schema, the shared conformance corpus, and the full design rationale live in the [`wavekat-flow` repository](https://github.com/wavekat/wavekat-flow). diff --git a/crates/wavekat-flow/src/lib.rs b/crates/wavekat-flow/src/lib.rs index f913c26..aff5f3c 100644 --- a/crates/wavekat-flow/src/lib.rs +++ b/crates/wavekat-flow/src/lib.rs @@ -11,7 +11,9 @@ //! [`trace`] output) alongside the generated model, adapted to the generated //! types and pinned by the shared conformance corpus. The engine owns the //! [`engine::FlowEffects`] trait *definition*; the daemon keeps its live impl -//! in its own codebase. Comment-preserving mutation stays TS-only. +//! in its own codebase. [`source`] finds and rewrites prompts in a document's text without +//! disturbing comments or layout, for Rust authoring tools; the TS side's +//! general-purpose comment-preserving edits stay in `mutate.ts`. /// The generated document model (`Flow`, `Node`, `Prompt`, …), emitted /// from the schema by `typify` into `OUT_DIR/flow_types.rs`. @@ -32,11 +34,15 @@ mod model_ext; pub mod book; pub mod engine; pub mod hours; +pub mod source; pub mod trace; pub mod validate; pub use model::*; -pub use model_ext::{required_assets, NodeId}; +pub use model_ext::{ + is_generated_clip_ref, is_local_audio_ref, prompt_fields, required_assets, NodeId, + PROMPT_FIELDS, +}; /// The normative JSON Schema (draft 2020-12) for format version 1, as a /// string, bundled so consumers can run structural validation without diff --git a/crates/wavekat-flow/src/model_ext.rs b/crates/wavekat-flow/src/model_ext.rs index 31970d5..2357ca3 100644 --- a/crates/wavekat-flow/src/model_ext.rs +++ b/crates/wavekat-flow/src/model_ext.rs @@ -131,6 +131,60 @@ pub fn required_assets(flow: &Flow) -> Vec { refs.into_iter().collect() } +/// Every node field that can hold a prompt, across all kinds. +pub const PROMPT_FIELDS: &[&str] = &["prompt", "confirm_prompt"]; + +/// The fields a node of `kind` speaks a prompt from, in the order +/// [`Node::prompts`] returns them, or `None` for a kind this build doesn't +/// know. For callers working on the source text, where the typed tree isn't +/// available (a draft that doesn't parse as a [`Flow`] yet). Keep in step +/// with [`Node::prompts`]; the corpus test `source_prompts_match_model` +/// fails when they differ. +pub fn prompt_fields(kind: &str) -> Option<&'static [&'static str]> { + match kind { + "greeting" | "menu" | "message" | "hangup" => Some(&PROMPT_FIELDS[..1]), + "book" => Some(PROMPT_FIELDS), + "hours" | "ring" | "transfer" => Some(&[]), + _ => None, + } +} + +/// A generated-clip ref is a `voice_prompts` id (`vprompt_`), a clip +/// the platform produced and owns. Anything else in an `audio:` field is a +/// ref the platform does *not* own. Twin: `model.ts` `isGeneratedClipRef`. +pub fn is_generated_clip_ref(reference: &str) -> bool { + let Some(rest) = reference + .get(..8) + .filter(|p| p.eq_ignore_ascii_case("vprompt_")) + .map(|_| &reference[8..]) + else { + return false; + }; + !rest.is_empty() && rest.chars().all(|c| c.is_ascii_alphanumeric()) +} + +/// Audio file extensions an author may point an `audio:` at. +const LOCAL_AUDIO_EXTENSIONS: &[&str] = &[ + ".wav", ".wave", ".mp3", ".ogg", ".oga", ".opus", ".flac", ".m4a", ".aac", ".mp4", ".webm", +]; + +/// Does an `audio:` value name a file on the author's machine, to be +/// uploaded before the flow is saved? A path separator or an audio file +/// extension makes it a file; a generated clip ref never is. Any other ref +/// (a clip shipped with the device, say) is neither, and is left alone. +pub fn is_local_audio_ref(reference: &str) -> bool { + let r = reference.trim(); + if r.is_empty() || is_generated_clip_ref(r) { + return false; + } + let lower = r.to_ascii_lowercase(); + r.contains('/') + || r.contains('\\') + || LOCAL_AUDIO_EXTENSIONS + .iter() + .any(|ext| lower.ends_with(ext)) +} + impl Prompt { /// The *synthesizable* text: `Some` for a bare-string prompt the engine /// speaks with TTS, `None` for an audio-asset prompt it plays as a clip. @@ -188,6 +242,32 @@ impl Flow { mod tests { use crate::model::Prompt; + #[test] + fn generated_clip_refs() { + use super::is_generated_clip_ref as clip; + assert!(clip("vprompt_ab12cd34")); + assert!(clip("VPROMPT_AB12")); + assert!(!clip("vprompt_")); + assert!(!clip("vprompt_ab-12")); + assert!(!clip("vprompt_ab12.wav")); + assert!(!clip("shipped_ref")); + assert!(!clip("vprompt")); + assert!(!clip("vpromp€_x")); + } + + #[test] + fn local_audio_refs() { + use super::is_local_audio_ref as local; + assert!(local("./a.wav")); + assert!(local("a.WAV")); + assert!(local("dir/a")); + assert!(local("C:\\clips\\a")); + assert!(local("song.mp3")); + assert!(!local("vprompt_12ab")); + assert!(!local("shipped_ref")); + assert!(!local(" ")); + } + #[test] fn as_text_is_synthesizable_text_only() { // A bare-string prompt is TTS the engine speaks. diff --git a/crates/wavekat-flow/src/source.rs b/crates/wavekat-flow/src/source.rs new file mode 100644 index 0000000..6324d8a --- /dev/null +++ b/crates/wavekat-flow/src/source.rs @@ -0,0 +1,1092 @@ +//! Finding and rewriting the spoken prompts in a flow document's *source +//! text*, keeping comments and layout as they are. +//! +//! The typed tree ([`crate::Flow`]) is lossy: serializing it back drops +//! comments, key order and quoting. An authoring tool that voices a flow +//! (each text prompt swapped for the clip generated from it) or uploads +//! the recordings a flow points at has to edit the author's text instead. +//! This module does that for Rust callers. The TypeScript side does the +//! same through the `yaml` CST in `mutate.ts`. +//! +//! An edit is a splice at the value's position: everything else stays +//! byte-for-byte as it was. Positions come from saphyr-parser's events; the +//! end of each value is found here. After every splice the result is +//! re-parsed and compared event by event against the original with only +//! the intended values changed, so a splice that went wrong is refused +//! ([`SourceError::Mismatch`]) rather than returned. +//! +//! Which fields hold a prompt comes from [`crate::prompt_fields`], the same +//! table [`crate::Node::prompts`] is checked against, so a new prompt field +//! is found here the moment the model knows it. + +use std::collections::HashMap; + +use saphyr_parser::{Event, Parser, ScalarStyle, Span, SpannedEventReceiver}; + +use crate::model_ext::{is_generated_clip_ref, is_local_audio_ref, prompt_fields, PROMPT_FIELDS}; + +/// Why a source read or edit failed. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum SourceError { + /// The text isn't valid YAML. + #[error("the flow isn't valid YAML: {0}")] + Parse(String), + /// A value's extent couldn't be worked out from the text. + #[error("couldn't find the {field} prompt in node {node}")] + Locate { node: String, field: String }, + /// A replacement clip id isn't a generated clip ref. + #[error("`{0}` isn't a generated clip ref (vprompt_…)")] + NotAClipRef(String), + /// The edited text no longer parses. + #[error("couldn't edit the prompts in place: the result didn't parse")] + Reparse, + /// The edited text parses, but changed more than the swapped values. + /// Usually an unusual layout; writing the value on one line, or quoted, + /// avoids it. + #[error("couldn't edit the prompts in place without changing other parts of the document")] + Mismatch, +} + +/// One text prompt, still waiting for audio. +#[derive(Debug, Clone, PartialEq)] +pub struct TextPrompt { + pub node: String, + pub field: String, + /// The node's `kind:`, when it has one. + pub kind: Option, + /// The words, trimmed: what gets synthesized, and the transcript. + pub text: String, + /// Byte range of the value in the source. + start: usize, + end: usize, + /// Indentation for a block-style replacement; `None` inside `{…}`, + /// where the replacement has to be flow style. + indent: Option, + /// Index of the value's event in the document's event stream. + event: usize, +} + +/// One prompt already written as a mapping (`{ audio: …, transcript: … }`). +#[derive(Debug, Clone, PartialEq)] +pub struct AudioPrompt { + pub node: String, + pub field: String, + /// The node's `kind:`, when it has one. + pub kind: Option, + /// The `audio:` value, trimmed, when it's a plain string. + pub audio: Option, + /// The `transcript:` value, trimmed, when present and non-empty. + pub transcript: Option, + /// Where the `audio:` value sits, for [`replace_audio_refs`]. + audio_at: Option, +} + +impl AudioPrompt { + /// Does `audio:` name a file on the author's machine? See + /// [`crate::is_local_audio_ref`]. + pub fn is_local(&self) -> bool { + self.audio.as_deref().is_some_and(is_local_audio_ref) + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +struct Located { + start: usize, + end: usize, + event: usize, +} + +/// Every prompt in a document, in document order, split by how it's voiced. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct Prompts { + /// Non-empty text prompts. + pub text: Vec, + /// Prompts written as a mapping (normally `{ audio: … }`). + pub audio: Vec, +} + +/// A simplified event, for comparing two documents' structure. +#[derive(Debug, Clone, PartialEq)] +enum Ev { + MapStart, + MapEnd, + SeqStart, + SeqEnd, + Scalar(String), + Alias, +} + +#[derive(Clone, Copy, PartialEq)] +enum Ctx { + Block, + Flow, +} + +enum Frame { + Map { + key: Option, + awaiting_key: bool, + ctx: Ctx, + }, + Seq, +} + +/// A scalar value's start, before its end is worked out. +#[derive(Clone)] +struct Found { + node: String, + field: String, + value: String, + style: ScalarStyle, + start: usize, + key_start: usize, + flow: bool, + event: usize, +} + +/// A prompt mapping as it's walked. +struct FoundMap { + node: String, + field: String, + audio: Option, + transcript: Option, +} + +struct Walker<'s> { + src: &'s str, + /// saphyr-parser 0.2.2's markers count chars (not bytes, whatever its + /// docs say); this maps a char index to a byte offset. + char_to_byte: Vec, + stack: Vec, + events: Vec, + /// Start of the key whose value comes next, per open mapping. + last_key_start: Vec, + /// Scalar prompt values, by field in [`PROMPT_FIELDS`]. + scalars: Vec, + /// Mapping prompt values. + maps: Vec, + /// The document's top-level `name:`. + name: Option, + /// Top-level keys, and whether the top-level mapping is block style. + root_keys: Vec, + root_block: bool, + /// `kind:` per node id. + kinds: HashMap, +} + +impl<'s> Walker<'s> { + fn new(src: &'s str) -> Self { + let mut char_to_byte: Vec = src.char_indices().map(|(b, _)| b).collect(); + char_to_byte.push(src.len()); + Self { + src, + char_to_byte, + stack: Vec::new(), + events: Vec::new(), + last_key_start: Vec::new(), + scalars: Vec::new(), + maps: Vec::new(), + name: None, + root_keys: Vec::new(), + root_block: false, + kinds: HashMap::new(), + } + } + + fn byte(&self, char_index: usize) -> usize { + *self.char_to_byte.get(char_index).unwrap_or(&self.src.len()) + } + + /// The keys leading to the value about to arrive, when every open + /// container is a mapping keyed by a plain string. + fn path(&self) -> Option> { + self.stack + .iter() + .map(|f| match f { + Frame::Map { + key: Some(k), + awaiting_key: false, + .. + } => Some(k.as_str()), + _ => None, + }) + .collect() + } + + /// `(node, field)` when the value about to arrive is a node's prompt. + fn prompt_slot(&self) -> Option<(String, String)> { + match self.path()?.as_slice() { + ["nodes", node, field] if PROMPT_FIELDS.contains(field) => { + Some((node.to_string(), field.to_string())) + } + _ => None, + } + } + + /// Called for every node event; true when it's a mapping key. + fn as_key(&mut self, scalar: Option<&str>, start: usize) -> bool { + if let Some(Frame::Map { + key, awaiting_key, .. + }) = self.stack.last_mut() + { + if *awaiting_key { + *key = scalar.map(str::to_string); + *awaiting_key = false; + if self.stack.len() == 1 { + if let Some(k) = scalar { + self.root_keys.push(k.to_string()); + } + } + if let Some(slot) = self.last_key_start.last_mut() { + *slot = start; + } + return true; + } + } + false + } + + fn value_done(&mut self) { + if let Some(Frame::Map { awaiting_key, .. }) = self.stack.last_mut() { + *awaiting_key = true; + } + } + + fn in_flow(&self) -> bool { + self.stack + .iter() + .any(|f| matches!(f, Frame::Map { ctx: Ctx::Flow, .. })) + } + + fn opens_flow(&self, start: usize) -> bool { + matches!(self.src.as_bytes().get(start), Some(b'{') | Some(b'[')) + } + + fn on_scalar(&mut self, value: &str, style: ScalarStyle, start: usize, event: usize) { + let key_start = self.last_key_start.last().copied().unwrap_or(start); + let found = |node: &str, field: &str| Found { + node: node.to_string(), + field: field.to_string(), + value: value.to_string(), + style, + start, + key_start, + flow: self.in_flow(), + event, + }; + let Some(owned) = self + .path() + .map(|p| p.into_iter().map(str::to_string).collect::>()) + else { + return; + }; + let path: Vec<&str> = owned.iter().map(String::as_str).collect(); + match path.as_slice() { + ["name"] => self.name = Some(value.to_string()), + ["nodes", node, "kind"] => { + self.kinds.insert(node.to_string(), value.to_string()); + } + ["nodes", node, field] if PROMPT_FIELDS.contains(field) => { + let f = found(node, field); + self.scalars.push(f); + } + ["nodes", node, field, leaf] if PROMPT_FIELDS.contains(field) => { + let (node, field, leaf) = (node.to_string(), field.to_string(), *leaf); + let f = found(&node, &field); + if let Some(m) = self + .maps + .iter_mut() + .rev() + .find(|m| m.node == node && m.field == field) + { + match leaf { + "audio" => m.audio = Some(f), + "transcript" => m.transcript = Some(value.to_string()), + _ => {} + } + } + } + _ => {} + } + } +} + +impl<'input> SpannedEventReceiver<'input> for Walker<'_> { + fn on_event(&mut self, ev: Event<'input>, span: Span) { + let start = self.byte(span.start.index()); + let index = self.events.len(); + match ev { + Event::Scalar(value, style, _, _) => { + self.events.push(Ev::Scalar(value.to_string())); + if self.as_key(Some(&value), start) { + return; + } + self.on_scalar(&value, style, start, index); + self.value_done(); + } + Event::MappingStart(..) => { + self.events.push(Ev::MapStart); + if !self.as_key(None, start) { + if let Some((node, field)) = self.prompt_slot() { + self.maps.push(FoundMap { + node, + field, + audio: None, + transcript: None, + }); + } + } + let ctx = if self.opens_flow(start) { + Ctx::Flow + } else { + Ctx::Block + }; + if self.stack.is_empty() { + self.root_block = ctx == Ctx::Block; + } + self.stack.push(Frame::Map { + key: None, + awaiting_key: true, + ctx, + }); + self.last_key_start.push(start); + } + Event::MappingEnd => { + self.events.push(Ev::MapEnd); + self.stack.pop(); + self.last_key_start.pop(); + self.value_done(); + } + Event::SequenceStart(..) => { + self.events.push(Ev::SeqStart); + self.as_key(None, start); + self.stack.push(Frame::Seq); + } + Event::SequenceEnd => { + self.events.push(Ev::SeqEnd); + self.stack.pop(); + self.value_done(); + } + Event::Alias(_) => { + self.events.push(Ev::Alias); + if !self.as_key(None, start) { + self.value_done(); + } + } + _ => {} + } + } +} + +fn walk(src: &str) -> Result, SourceError> { + let mut w = Walker::new(src); + Parser::new_from_str(src) + .load(&mut w, false) + .map_err(|e| SourceError::Parse(e.to_string()))?; + Ok(w) +} + +impl Walker<'_> { + /// Is `field` a prompt of `node`, given its kind? A node with no kind, + /// or one this build doesn't know, keeps every prompt field, so nothing + /// is silently skipped; validation reports the kind. + fn speaks(&self, node: &str, field: &str) -> bool { + match self.kinds.get(node).and_then(|k| prompt_fields(k)) { + Some(fields) => fields.contains(&field), + None => true, + } + } +} + +/// Every prompt in `src`, in document order. +pub fn prompts(src: &str) -> Result { + let w = walk(src)?; + let mut out = Prompts::default(); + for f in &w.scalars { + if !w.speaks(&f.node, &f.field) { + continue; + } + // `prompt:` with nothing after it (a hangup with no goodbye) parses + // as an empty plain scalar; there's nothing to say. + let text = f.value.trim(); + if text.is_empty() || (f.style == ScalarStyle::Plain && matches!(text, "~" | "null")) { + continue; + } + let (mut start, end) = value_range(src, f)?; + let indent = (!f.flow).then(|| indent_of_line(src, f.key_start) + 2); + if indent.is_some() { + // A block replacement starts on the next line, so take the + // spaces after the colon with it. + while start > 0 && matches!(src.as_bytes()[start - 1], b' ' | b'\t') { + start -= 1; + } + } + out.text.push(TextPrompt { + node: f.node.clone(), + field: f.field.clone(), + kind: w.kinds.get(&f.node).cloned(), + text: text.to_string(), + start, + end, + indent, + event: f.event, + }); + } + for m in &w.maps { + if !w.speaks(&m.node, &m.field) { + continue; + } + let audio_at = match &m.audio { + Some(f) => { + let (start, end) = value_range(src, f)?; + Some(Located { + start, + end, + event: f.event, + }) + } + None => None, + }; + out.audio.push(AudioPrompt { + node: m.node.clone(), + field: m.field.clone(), + kind: w.kinds.get(&m.node).cloned(), + audio: m.audio.as_ref().map(|f| f.value.trim().to_string()), + transcript: m + .transcript + .as_deref() + .map(str::trim) + .filter(|t| !t.is_empty()) + .map(str::to_string), + audio_at, + }); + } + // Both lists are in document order already; the source order of text + // vs. audio prompts isn't needed by anything. + Ok(out) +} + +/// Every prompt in `src` whose `audio:` names a local file, in document +/// order. +pub fn local_audio(src: &str) -> Result, SourceError> { + Ok(prompts(src)? + .audio + .into_iter() + .filter(AudioPrompt::is_local) + .collect()) +} + +/// The document's top-level `name:`, if it has a non-blank one. +pub fn top_level_name(src: &str) -> Option { + walk(src).ok()?.name.filter(|n| !n.trim().is_empty()) +} + +/// `src` with `key: "value"` appended for each of `fields` whose key isn't +/// already at the top level, so a draft that leaves platform-assigned +/// fields (`id`, `name`) out can still be validated. Appended at the end so +/// issue positions in the original text stay right. Unchanged when the +/// document doesn't parse or isn't a block mapping. +pub fn with_placeholder_fields(src: &str, fields: &[(&str, &str)]) -> String { + let Ok(w) = walk(src) else { + return src.to_string(); + }; + if !w.root_block { + return src.to_string(); + } + let mut out = src.to_string(); + for (key, value) in fields { + if w.root_keys.iter().any(|k| k == key) { + continue; + } + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + out.push_str(&format!("{key}: {}\n", quoted(value))); + } + out +} + +/// A YAML double-quoted scalar (a JSON string is one). +fn quoted(s: &str) -> String { + serde_json::to_string(s).expect("a string always serializes") +} + +/// The YAML written in place of a text prompt: block style at `indent`, as +/// the web editor writes it, or flow style (`{ audio: …, … }`) when `indent` +/// is `None`. +fn audio_prompt_yaml(clip_ref: &str, transcript: &str, indent: Option) -> String { + let transcript = quoted(transcript); + match indent { + Some(n) => { + let pad = " ".repeat(n); + format!("\n{pad}audio: {clip_ref}\n{pad}transcript: {transcript}") + } + None => format!("{{ audio: {clip_ref}, transcript: {transcript} }}"), + } +} + +fn check_clip_refs<'a>(refs: impl IntoIterator) -> Result<(), SourceError> { + match refs.into_iter().find(|r| !is_generated_clip_ref(r)) { + Some(bad) => Err(SourceError::NotAClipRef(bad.clone())), + None => Ok(()), + } +} + +/// Voice text prompts: swap each `(prompt, clip ref)` for +/// `{ audio: , transcript: }`. Nothing else in +/// the document changes. `prompt`s must come from [`prompts`] on this same +/// `src`. +pub fn voice_text_prompts( + src: &str, + swaps: &[(TextPrompt, String)], +) -> Result { + check_clip_refs(swaps.iter().map(|(_, id)| id))?; + let mut sorted: Vec<&(TextPrompt, String)> = swaps.iter().collect(); + sorted.sort_by_key(|(p, _)| std::cmp::Reverse(p.start)); + let mut out = src.to_string(); + for (p, id) in &sorted { + out.replace_range(p.start..p.end, &audio_prompt_yaml(id, &p.text, p.indent)); + } + + // The original event stream with each swapped value expanded into its + // audio mapping must equal the new document's stream. + let before = walk(src)?.events; + let mut by_event: Vec<&(TextPrompt, String)> = swaps.iter().collect(); + by_event.sort_by_key(|(p, _)| p.event); + let mut expected = Vec::with_capacity(before.len() + swaps.len() * 5); + let mut next = by_event.iter().peekable(); + for (i, ev) in before.into_iter().enumerate() { + match next.peek() { + Some((p, id)) if p.event == i => { + expected.extend([ + Ev::MapStart, + Ev::Scalar("audio".into()), + Ev::Scalar(id.clone()), + Ev::Scalar("transcript".into()), + Ev::Scalar(p.text.clone()), + Ev::MapEnd, + ]); + next.next(); + } + _ => expected.push(ev), + } + } + let after = walk(&out).map_err(|_| SourceError::Reparse)?.events; + if after != expected { + return Err(SourceError::Mismatch); + } + Ok(out) +} + +/// Swap each prompt's `audio:` value for a clip ref (a local file's path +/// for the clip uploaded from it, say). Only those values change. Prompts +/// must come from [`prompts`] or [`local_audio`] on this same `src`; one +/// with no `audio:` value is refused. +pub fn replace_audio_refs( + src: &str, + swaps: &[(AudioPrompt, String)], +) -> Result { + check_clip_refs(swaps.iter().map(|(_, id)| id))?; + let mut located = Vec::with_capacity(swaps.len()); + for (p, id) in swaps { + let at = p.audio_at.ok_or_else(|| SourceError::Locate { + node: p.node.clone(), + field: p.field.clone(), + })?; + located.push((at, id)); + } + located.sort_by_key(|(at, _)| std::cmp::Reverse(at.start)); + let mut out = src.to_string(); + for (at, id) in &located { + out.replace_range(at.start..at.end, id); + } + let mut expected = walk(src)?.events; + for (at, id) in &located { + match expected.get_mut(at.event) { + Some(ev @ Ev::Scalar(_)) => *ev = Ev::Scalar((*id).clone()), + _ => return Err(SourceError::Mismatch), + } + } + let after = walk(&out).map_err(|_| SourceError::Reparse)?.events; + if after != expected { + return Err(SourceError::Mismatch); + } + Ok(out) +} + +/// The byte range of a scalar value in the source. +fn value_range(src: &str, f: &Found) -> Result<(usize, usize), SourceError> { + let lost = || SourceError::Locate { + node: f.node.clone(), + field: f.field.clone(), + }; + let bytes = src.as_bytes(); + let line_indent = indent_of_line(src, f.key_start); + match f.style { + ScalarStyle::SingleQuoted | ScalarStyle::DoubleQuoted => { + let quote = if f.style == ScalarStyle::SingleQuoted { + b'\'' + } else { + b'"' + }; + // The marker sits on the opening quote; scan to its partner. + let open = (f.start..bytes.len()) + .find(|&i| bytes[i] == quote) + .ok_or_else(lost)?; + let mut i = open + 1; + while i < bytes.len() { + match bytes[i] { + b'\\' if quote == b'"' => i += 2, + b'\'' if quote == b'\'' && bytes.get(i + 1) == Some(&b'\'') => i += 2, + b if b == quote => return Ok((open, i + 1)), + _ => i += 1, + } + } + Err(lost()) + } + ScalarStyle::Literal | ScalarStyle::Folded => { + // The marker sits on the content; the value starts at the + // `|` / `>` indicator after the key's colon. + let colon = src[f.key_start..] + .find(':') + .map(|i| f.key_start + i) + .ok_or_else(lost)?; + let open = (colon + 1..bytes.len()) + .find(|&i| matches!(bytes[i], b'|' | b'>')) + .ok_or_else(lost)?; + // Content is every following line indented past the key, plus + // blank lines between them; the value ends after the last + // content character, so trailing blank lines stay put. + let mut end = line_end(src, open); + let mut pos = end; + while pos < bytes.len() { + let next = line_end(src, pos + 1); + let line = &src[pos + 1..next]; + if line.trim().is_empty() { + pos = next; + continue; + } + if leading_spaces(line) <= line_indent { + break; + } + end = pos + 1 + line.trim_end().len(); + pos = next; + } + Ok((open, end)) + } + ScalarStyle::Plain => { + // Ends at a comment, at the end of the line or, inside `{…}` / + // `[…]`, at the next `,` `}` `]`; may continue on following + // lines indented past the key. + let stop = |i: usize| -> bool { + let b = bytes[i]; + b == b'\n' + || b == b'\r' + || (b == b'#' && i > 0 && matches!(bytes[i - 1], b' ' | b'\t')) + || (f.flow && matches!(b, b',' | b'}' | b']')) + }; + let mut i = f.start; + while i < bytes.len() && !stop(i) { + i += 1; + } + let mut end = f.start + src[f.start..i].trim_end().len(); + if !f.flow { + let mut pos = line_end(src, f.start); + while pos < bytes.len() { + let next = line_end(src, pos + 1); + let line = &src[pos + 1..next]; + let body = line.trim_start(); + if body.is_empty() + || body.starts_with('#') + || leading_spaces(line) <= line_indent + { + break; + } + let cut = body + .find(" #") + .or_else(|| body.find("\t#")) + .unwrap_or(body.len()); + end = pos + 1 + leading_spaces(line) + body[..cut].trim_end().len(); + pos = next; + } + } + Ok((f.start, end)) + } + } +} + +/// Byte offset of the `\n` ending the line that contains `at` (or the end +/// of the source). +fn line_end(src: &str, at: usize) -> usize { + if at >= src.len() { + return src.len(); + } + src[at..].find('\n').map(|i| at + i).unwrap_or(src.len()) +} + +fn leading_spaces(line: &str) -> usize { + line.len() - line.trim_start_matches(' ').len() +} + +fn indent_of_line(src: &str, at: usize) -> usize { + let line_start = src[..at].rfind('\n').map(|i| i + 1).unwrap_or(0); + leading_spaces(&src[line_start..]) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FLOW: &str = "\ +schema_version: 1 +id: flow_x +name: Test # the name +entry: hello +nodes: + # Say hi first. + hello: + kind: greeting + prompt: Thanks for calling, how can we help? # keep it short + exits: { next: menu } + menu: + kind: menu + prompt: 'Press 1 for sales, it''s quick.' + options: { '1': bye } + exits: { '1': bye } + done: + kind: message + prompt: | + Please leave a message + after the tone. + + # trailing comment + exits: { next: bye } + booked: + kind: book + prompt: \"I can book you in.\\n\" + confirm_prompt: { audio: vprompt_abc, transcript: You're booked for } + bye: + kind: hangup + prompt: Goodbye — 再见. + silent: + kind: hangup + prompt: +"; + + fn texts(src: &str) -> Vec<(String, String, String)> { + prompts(src) + .unwrap() + .text + .into_iter() + .map(|p| (p.node, p.field, p.text)) + .collect() + } + + fn clip(i: usize) -> String { + format!("vprompt_{i}") + } + + #[test] + fn finds_text_and_audio_prompts() { + let found = prompts(FLOW).unwrap(); + assert_eq!(found.audio.len(), 1); + assert_eq!(found.audio[0].audio.as_deref(), Some("vprompt_abc")); + assert_eq!( + found.audio[0].transcript.as_deref(), + Some("You're booked for") + ); + assert_eq!(found.audio[0].kind.as_deref(), Some("book")); + assert_eq!( + texts(FLOW), + vec![ + ( + "hello".into(), + "prompt".into(), + "Thanks for calling, how can we help?".into() + ), + ( + "menu".into(), + "prompt".into(), + "Press 1 for sales, it's quick.".into() + ), + ( + "done".into(), + "prompt".into(), + "Please leave a message\nafter the tone.".into() + ), + ( + "booked".into(), + "prompt".into(), + "I can book you in.".into() + ), + ("bye".into(), "prompt".into(), "Goodbye — 再见.".into()), + ] + ); + } + + #[test] + fn voicing_swaps_only_the_prompts() { + let found = prompts(FLOW).unwrap(); + let swaps: Vec<_> = found + .text + .iter() + .enumerate() + .map(|(i, p)| (p.clone(), clip(i))) + .collect(); + let out = voice_text_prompts(FLOW, &swaps).unwrap(); + let block = + |id: &str, t: &str| format!(" prompt:\n audio: {id}\n transcript: {t}"); + for want in [ + format!( + "{} # keep it short\n", + block("vprompt_0", "\"Thanks for calling, how can we help?\"") + ), + format!( + "{}\n options:", + block("vprompt_1", "\"Press 1 for sales, it's quick.\"") + ), + format!( + "{}\n\n # trailing comment\n", + block("vprompt_2", "\"Please leave a message\\nafter the tone.\"") + ), + format!( + "{}\n confirm_prompt: {{ audio: vprompt_abc", + block("vprompt_3", "\"I can book you in.\"") + ), + format!("{}\n", block("vprompt_4", "\"Goodbye — 再见.\"")), + ] { + assert!(out.contains(&want), "missing {want:?} in:\n{out}"); + } + // Comments and everything else survive. + assert!(out.contains("name: Test # the name\n")); + assert!(out.contains(" # Say hi first.\n")); + // Idempotent: nothing left to voice. + let again = prompts(&out).unwrap(); + assert!(again.text.is_empty()); + assert_eq!(again.audio.len(), 6); + } + + #[test] + fn partial_voicing_leaves_the_rest_as_text() { + let found = prompts(FLOW).unwrap(); + let swaps = vec![(found.text[1].clone(), clip(9))]; + let out = voice_text_prompts(FLOW, &swaps).unwrap(); + let left = texts(&out); + assert_eq!(left.len(), 4); + assert!(left.iter().all(|(node, _, _)| node != "menu")); + } + + #[test] + fn voicing_refuses_a_ref_that_isnt_a_clip() { + let found = prompts(FLOW).unwrap(); + let swaps = vec![(found.text[0].clone(), "odd id".to_string())]; + assert_eq!( + voice_text_prompts(FLOW, &swaps), + Err(SourceError::NotAClipRef("odd id".into())) + ); + } + + #[test] + fn flow_style_nodes_and_multiline_plain() { + let src = "\ +nodes: + a: { kind: greeting, prompt: Hello there, exits: { next: b } } + b: + kind: hangup + prompt: Thanks for + calling us # bye +"; + let found = prompts(src).unwrap(); + assert_eq!( + texts(src), + vec![ + ("a".into(), "prompt".into(), "Hello there".into()), + ("b".into(), "prompt".into(), "Thanks for calling us".into()), + ] + ); + let swaps: Vec<_> = found.text.iter().map(|p| (p.clone(), clip(7))).collect(); + let out = voice_text_prompts(src, &swaps).unwrap(); + assert!(out.contains( + "a: { kind: greeting, prompt: { audio: vprompt_7, transcript: \"Hello there\" }, exits: { next: b } }" + ), "{out}"); + assert!( + out.contains( + " prompt:\n audio: vprompt_7\n transcript: \"Thanks for calling us\" # bye\n" + ), + "{out}" + ); + } + + #[test] + fn ignores_prompt_keys_outside_nodes() { + let src = "\ +prompt: not a node +ui: + hello: + prompt: also not +nodes: + hello: + kind: greeting + prompt: Hi + extra: + prompt: nested, not a node field +"; + assert_eq!( + texts(src), + vec![("hello".into(), "prompt".into(), "Hi".into())] + ); + } + + #[test] + fn only_fields_the_kind_speaks() { + // `confirm_prompt` is `book`'s alone; on a greeting it's a stray + // key for validation to report, not something to voice. A node + // with no kind (yet) keeps every prompt field. + let src = "\ +nodes: + a: + confirm_prompt: Not spoken + kind: greeting + prompt: Spoken + b: + confirm_prompt: Kept +"; + assert_eq!( + texts(src), + vec![ + ("a".into(), "prompt".into(), "Spoken".into()), + ("b".into(), "confirm_prompt".into(), "Kept".into()), + ] + ); + } + + #[test] + fn reads_the_document_name() { + assert_eq!(top_level_name(FLOW).as_deref(), Some("Test")); + assert_eq!(top_level_name("nodes: {}\n"), None); + assert_eq!(top_level_name("nodes:\n name: x\n"), None); + } + + #[test] + fn placeholders_fill_only_missing_fields() { + let src = "# c\nname: Mine\nnodes: {}"; + let out = with_placeholder_fields(src, &[("id", "flow_new"), ("name", "x")]); + assert_eq!(out, "# c\nname: Mine\nnodes: {}\nid: \"flow_new\"\n"); + assert_eq!(with_placeholder_fields(&out, &[("id", "y")]), out); + // Not a block mapping, or not YAML: left alone for validation. + assert_eq!(with_placeholder_fields("{a: 1}", &[("id", "x")]), "{a: 1}"); + assert_eq!(with_placeholder_fields("a: [", &[("id", "x")]), "a: ["); + } + + const LOCAL: &str = "\ +nodes: + hello: + kind: greeting + # recorded by the owner + prompt: + audio: ./intro.wav # keep + transcript: Welcome to the shop. + exits: { next: bye } + menu: + kind: menu + prompt: { audio: 'clips/menu 1.wav', transcript: \"Press 1\" } + known: + kind: message + prompt: { audio: vprompt_abc } + other: + kind: greeting + prompt: { audio: shipped_ref } + booked: + kind: book + prompt: { audio: ../book.wav } + confirm_prompt: Booked +"; + + #[test] + fn finds_local_audio_with_transcripts_and_kinds() { + let got = local_audio(LOCAL).unwrap(); + let summary: Vec<_> = got + .iter() + .map(|a| { + ( + a.node.as_str(), + a.audio.as_deref(), + a.transcript.as_deref(), + a.kind.as_deref(), + ) + }) + .collect(); + assert_eq!( + summary, + vec![ + ( + "hello", + Some("./intro.wav"), + Some("Welcome to the shop."), + Some("greeting") + ), + ( + "menu", + Some("clips/menu 1.wav"), + Some("Press 1"), + Some("menu") + ), + ("booked", Some("../book.wav"), None, Some("book")), + ] + ); + // Audio prompts, local or not, aren't text prompts. + assert_eq!( + texts(LOCAL), + vec![("booked".into(), "confirm_prompt".into(), "Booked".into())] + ); + } + + #[test] + fn replacing_audio_swaps_only_the_refs() { + let got = local_audio(LOCAL).unwrap(); + let swaps: Vec<_> = got + .iter() + .enumerate() + .map(|(i, a)| (a.clone(), clip(i))) + .collect(); + let out = replace_audio_refs(LOCAL, &swaps).unwrap(); + assert!( + out.contains( + " audio: vprompt_0 # keep\n transcript: Welcome to the shop.\n" + ), + "{out}" + ); + assert!( + out.contains("prompt: { audio: vprompt_1, transcript: \"Press 1\" }"), + "{out}" + ); + assert!(out.contains("prompt: { audio: vprompt_2 }"), "{out}"); + assert!(out.contains(" # recorded by the owner\n")); + assert!(out.contains("{ audio: shipped_ref }")); + assert!(local_audio(&out).unwrap().is_empty()); + assert_eq!( + replace_audio_refs(LOCAL, &[(got[0].clone(), "not-a-clip".into())]), + Err(SourceError::NotAClipRef("not-a-clip".into())) + ); + } + + #[test] + fn invalid_yaml_is_an_error() { + assert!(matches!( + prompts("nodes: [unclosed"), + Err(SourceError::Parse(_)) + )); + } + + #[test] + fn replacement_yaml_quotes_the_transcript() { + assert_eq!( + audio_prompt_yaml("vprompt_ab12", "Say \"hi\"", None), + r#"{ audio: vprompt_ab12, transcript: "Say \"hi\"" }"# + ); + assert_eq!( + audio_prompt_yaml("vprompt_1", "Hi", Some(4)), + "\n audio: vprompt_1\n transcript: \"Hi\"" + ); + } +} diff --git a/crates/wavekat-flow/tests/source.rs b/crates/wavekat-flow/tests/source.rs new file mode 100644 index 0000000..17623ef --- /dev/null +++ b/crates/wavekat-flow/tests/source.rs @@ -0,0 +1,137 @@ +//! `source` reads prompts from the text, the model from the typed tree. +//! Over every valid document in the corpus, the two must find the same +//! prompts, node for node. A prompt field added to the model but not to +//! `prompt_fields` (or the other way round) fails here. Each document is +//! then voiced in full, and the result must still parse to the same flow, +//! with every text prompt now a clip carrying its text as the transcript. + +use std::{collections::BTreeMap, fs, path::PathBuf}; + +use wavekat_flow::model::{Flow, Prompt}; +use wavekat_flow::source; + +/// `(node, field-order index) → what it says`, as text or audio ref. +type Spoken = BTreeMap>; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +enum Said { + Text(String), + Audio(Option), +} + +fn valid_documents() -> Vec { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../conformance"); + let mut out = Vec::new(); + for version in fs::read_dir(&root).expect("read corpus root").flatten() { + for bucket in ["valid", "outbound/valid"] { + let Ok(entries) = fs::read_dir(version.path().join(bucket)) else { + continue; + }; + for e in entries.flatten() { + let p = e.path(); + if p.extension().is_some_and(|x| x == "yaml") { + out.push(p); + } + } + } + } + out.sort(); + assert!(!out.is_empty(), "no valid documents in the corpus"); + out +} + +fn from_model(flow: &Flow) -> Spoken { + let mut out = Spoken::new(); + for (id, node) in &flow.nodes { + let said: Vec = node + .prompts() + .into_iter() + .filter_map(|p| match p { + Prompt::Text(t) if t.trim().is_empty() => None, + Prompt::Text(t) => Some(Said::Text(t.trim().to_string())), + Prompt::Audio { audio, .. } => Some(Said::Audio(Some(audio.trim().to_string()))), + }) + .collect(); + if !said.is_empty() { + out.insert(id.clone(), said); + } + } + sorted(out) +} + +fn from_source(found: &source::Prompts) -> Spoken { + let mut out = Spoken::new(); + for p in &found.text { + out.entry(p.node.clone()) + .or_default() + .push(Said::Text(p.text.clone())); + } + for p in &found.audio { + out.entry(p.node.clone()) + .or_default() + .push(Said::Audio(p.audio.clone())); + } + sorted(out) +} + +fn sorted(mut s: Spoken) -> Spoken { + for v in s.values_mut() { + v.sort(); + } + s +} + +#[test] +fn source_prompts_match_model() { + for path in valid_documents() { + let src = fs::read_to_string(&path).unwrap(); + let flow = Flow::from_yaml(&src).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + let found = source::prompts(&src).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + assert_eq!( + from_source(&found), + from_model(&flow), + "{}: source and model disagree on the prompts", + path.display() + ); + } +} + +#[test] +fn voicing_every_corpus_document_keeps_the_flow() { + for path in valid_documents() { + let src = fs::read_to_string(&path).unwrap(); + let found = source::prompts(&src).unwrap(); + let swaps: Vec<_> = found + .text + .iter() + .enumerate() + .map(|(i, p)| (p.clone(), format!("vprompt_{i}"))) + .collect(); + let out = source::voice_text_prompts(&src, &swaps) + .unwrap_or_else(|e| panic!("{}: {e}", path.display())); + + let before = Flow::from_yaml(&src).unwrap(); + let after = Flow::from_yaml(&out).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + assert_eq!(before.entry, after.entry, "{}", path.display()); + assert_eq!(before.nodes.len(), after.nodes.len(), "{}", path.display()); + assert!( + source::prompts(&out).unwrap().text.is_empty(), + "{}: text prompts left after voicing", + path.display() + ); + for (i, (p, _)) in swaps.iter().enumerate() { + let node = &after.nodes[&p.node]; + let want = format!("vprompt_{i}"); + assert!( + node.prompts().into_iter().any(|q| matches!( + q, + Prompt::Audio { audio, transcript: Some(t) } if *audio == want && *t == p.text + )), + "{}: {}.{} wasn't voiced as {want}", + path.display(), + p.node, + p.field + ); + } + } +} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 2f41938..4a97401 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -31,8 +31,10 @@ Nothing is migrated in the consumer repos yet (that's Phase 3). ### TypeScript — move from the platform repo's `packages/flow-schema/src` - [x] `parse.ts` — safe-subset YAML decode (anchors/aliases/tags/dup-keys/ implicit-typing rejection, `nodeRanges` source offsets). -- [x] `mutate.ts` — comment-preserving edits (stays TS-only; the daemon never - edits). Works on the `yaml` CST, not the generated types; only import +- [x] `mutate.ts` — comment-preserving edits (TS-only; the daemon never + edits). The narrower prompt edits Rust authoring tools need (voicing + text prompts, swapping local audio for uploaded clips) are in the + crate's `source` module, checked against the corpus. Works on the `yaml` CST, not the generated types; only import from the model is `ComponentKind`. 20-case suite ported. - [x] `validate.ts` — reachability BFS, caller-trap, exit-set exactness, DTMF, prompt length. ⚠️ dangling-target code reconciled to the frozen corpus: