diff --git a/Cargo.lock b/Cargo.lock index 89ae91d..6b8ee02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -479,6 +479,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -1048,6 +1050,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -1555,6 +1558,7 @@ version = "0.0.10" dependencies = [ "anyhow", "async-trait", + "indexmap", "jsonschema", "prettyplease", "schemars", diff --git a/README.md b/README.md index 0455ae0..1e52088 100644 --- a/README.md +++ b/README.md @@ -49,18 +49,21 @@ crates/wavekat-flow/ # wavekat-flow (crate) — generated typ - **The logic lives here too.** Beyond the generated model, each package carries its language's hand-written half of the format: [`@wavekat/flow-schema`](./packages/flow-schema/README.md) — safe-subset - YAML parsing, semantic validation, hours math, comment-preserving source - edits, and flow diffing (authoring side); + YAML parsing, semantic validation, hours math, structured edits, and flow + diffing (authoring side); [`wavekat-flow`](./crates/wavekat-flow/README.md) — semantic validation, hours math, and the flow interpreter with its run trace (execution side). + Both write one **canonical text** for a flow, byte for byte the same, so + editing is parse → change the model → serialize in either language. See + [`docs/01-canonical-document.md`](./docs/01-canonical-document.md). ## What a schema *cannot* own A JSON Schema defines the document **shape** (types, enums, required fields, the `kind` discriminated union, defaults). It cannot express the format's **semantic rules** — graph reachability, exit-set exactness, hours/timezone -math, DTMF digit validity, prompt length, safe-subset YAML parsing, -comment-preserving edits, or the interpreter. Those live here as hand-written +math, DTMF digit validity, prompt length, safe-subset YAML parsing, the +canonical serialization, or the interpreter. Those live here as hand-written code per language — **pinned by the shared corpus**, not generated. See [`schema/README.md`](./schema/README.md) for the exact boundary. @@ -92,7 +95,7 @@ in **[`docs/ROADMAP.md`](./docs/ROADMAP.md)**. corpus + back-compat runner, and both packages generating their model types from the schema, corpus green in TS and Rust. - **Phase 2 — full consolidation. Done.** The hand-written logic moved in - (TS: safe-subset parse, comment-preserving mutation, semantic validation, + (TS: safe-subset parse, structured edits, semantic validation, hours; Rust: validation, hours, engine, trace), each adapted to the generated types, with the corpus extended to pin the semantic outcomes in both languages. diff --git a/conformance/canonical/comments.canonical.yaml b/conformance/canonical/comments.canonical.yaml new file mode 100644 index 0000000..29b0047 --- /dev/null +++ b/conformance/canonical/comments.canonical.yaml @@ -0,0 +1,23 @@ +schema_version: 2 +id: flow_comments +name: "Comments # are not data" +entry: hi +nodes: + hi: + kind: greeting + note: C# is a note, not a comment + prompt: "It's # not a comment" + exits: { next: book } + book: + kind: book + prompt: "# inside a block scalar\nHere are the times.\n" + confirm_prompt: { audio: vprompt_c0, transcript: You're booked for } + schedule: + mon: + - { open: "09:00", close: "17:00" } + timezone: UTC + duration_mins: 30 + exits: { booked: bye, no_slots: bye, no_input: bye, unavailable: bye } + bye: + kind: hangup + prompt: "multi line # still quoted" diff --git a/conformance/canonical/comments.expected.json b/conformance/canonical/comments.expected.json new file mode 100644 index 0000000..866fc12 --- /dev/null +++ b/conformance/canonical/comments.expected.json @@ -0,0 +1,4 @@ +{ + "description": "Comments are found (and counted) the same way in both languages, and dropped by the canonical text. A `#` inside a quoted scalar, a block scalar, or after a non-blank character is not a comment.", + "comments": 5 +} diff --git a/conformance/canonical/comments.input.yaml b/conformance/canonical/comments.input.yaml new file mode 100644 index 0000000..1ccfb35 --- /dev/null +++ b/conformance/canonical/comments.input.yaml @@ -0,0 +1,26 @@ +# A flow with comments: none of them survive a save. +schema_version: 2 +id: flow_comments # platform id +name: "Comments # are not data" +entry: hi +nodes: + # The greeting. + hi: + kind: greeting + prompt: 'It''s # not a comment' + note: C# is a note, not a comment + exits: { next: book } # trailing + book: + kind: book + prompt: | + # inside a block scalar + Here are the times. + # after the block scalar + confirm_prompt: { audio: vprompt_c0, transcript: "You're booked for" } + schedule: + mon: [{ open: "09:00", close: "17:00" }] + timezone: UTC + duration_mins: 30 + exits: { booked: bye, no_slots: bye, no_input: bye, unavailable: bye } + bye: { kind: hangup, prompt: "multi + line # still quoted" } diff --git a/conformance/canonical/defaults.canonical.yaml b/conformance/canonical/defaults.canonical.yaml new file mode 100644 index 0000000..e09e1b4 --- /dev/null +++ b/conformance/canonical/defaults.canonical.yaml @@ -0,0 +1,24 @@ +schema_version: 1 +id: flow_defaults +name: Defaults are left out +entry: welcome +nodes: + welcome: + kind: greeting + prompt: Hello. + exits: { next: menu } + menu: + kind: menu + prompt: Press 1 for sales. + options: { "1": Sales } + exits: { "1": vm, no_input: vm, invalid: vm } + vm: + kind: message + prompt: Leave a message. + open: + kind: hours + schedule: {} + timezone: UTC + exits: { open: welcome, closed: vm } + bye: + kind: hangup diff --git a/conformance/canonical/defaults.expected.json b/conformance/canonical/defaults.expected.json new file mode 100644 index 0000000..f8863fa --- /dev/null +++ b/conformance/canonical/defaults.expected.json @@ -0,0 +1,4 @@ +{ + "description": "Optional fields equal to their schema default, and empty optional maps and lists, are left out; required ones are written even when empty (`schedule: {}`).", + "comments": 0 +} diff --git a/conformance/canonical/defaults.input.yaml b/conformance/canonical/defaults.input.yaml new file mode 100644 index 0000000..f13fdb2 --- /dev/null +++ b/conformance/canonical/defaults.input.yaml @@ -0,0 +1,31 @@ +schema_version: 1 +id: flow_defaults +name: Defaults are left out +version: 1 +entry: welcome +nodes: + welcome: + kind: greeting + prompt: Hello. + exits: { next: menu } + menu: + kind: menu + prompt: Press 1 for sales. + options: { "1": Sales } + retries: 1 + timeout_secs: 5 + exits: { "1": vm, no_input: vm, invalid: vm } + vm: + kind: message + prompt: Leave a message. + max_secs: 120 + tone: beep + exits: {} + open: + kind: hours + timezone: UTC + schedule: {} + exceptions: [] + exits: { open: welcome, closed: vm } + bye: + kind: hangup diff --git a/conformance/canonical/order.canonical.yaml b/conformance/canonical/order.canonical.yaml new file mode 100644 index 0000000..2947ae8 --- /dev/null +++ b/conformance/canonical/order.canonical.yaml @@ -0,0 +1,21 @@ +schema_version: 1 +id: flow_order +name: Order +version: 3 +note: Order of nodes is the author's; integer ids go first. +entry: start +nodes: + "2": + kind: hangup + "10": + kind: hangup + start: + kind: greeting + note: First thing a caller hears. + prompt: Welcome. + exits: { next: pick } + pick: + kind: menu + prompt: Choose. + options: { "0": Operator, "1": Sales, "2": Support, "*": Repeat, "#": Exit } + exits: { "0": start, "1": start, "2": start, invalid: start, "#": start, "*": start, no_input: start } diff --git a/conformance/canonical/order.expected.json b/conformance/canonical/order.expected.json new file mode 100644 index 0000000..e2af877 --- /dev/null +++ b/conformance/canonical/order.expected.json @@ -0,0 +1,4 @@ +{ + "description": "Schema-described objects are written in schema property order (`note` after `kind` and after `version`). Other maps keep document order, except that integer keys come first in ascending order, as JavaScript objects order them.", + "comments": 0 +} diff --git a/conformance/canonical/order.input.yaml b/conformance/canonical/order.input.yaml new file mode 100644 index 0000000..aee29e6 --- /dev/null +++ b/conformance/canonical/order.input.yaml @@ -0,0 +1,21 @@ +nodes: + start: + exits: { next: pick } + prompt: Welcome. + note: First thing a caller hears. + kind: greeting + pick: + options: { "2": Support, "1": Sales, "0": Operator, "*": Repeat, "#": Exit } + exits: { invalid: start, "#": start, "*": start, "2": start, no_input: start, "0": start, "1": start } + kind: menu + prompt: Choose. + "10": + kind: hangup + "2": + kind: hangup +entry: start +note: Order of nodes is the author's; integer ids go first. +name: Order +version: 3 +id: flow_order +schema_version: 1 diff --git a/conformance/canonical/scalars.canonical.yaml b/conformance/canonical/scalars.canonical.yaml new file mode 100644 index 0000000..f278318 --- /dev/null +++ b/conformance/canonical/scalars.canonical.yaml @@ -0,0 +1,32 @@ +schema_version: 1 +id: flow_scalars +name: "Quotes: only where needed" +entry: a +nodes: + a: + kind: greeting + note: "yes" + prompt: Thanks for calling, how can we help? + exits: { next: b } + b: + kind: menu + prompt: "It's 9: press 1 or # for more" + options: { "1": "Sales, new orders", "2": Plain label, "3": "Ext 5:30" } + exits: { "1": c, "2": c, no_input: c, invalid: c } + c: + kind: message + note: " leading space" + prompt: "Please leave a message\nafter the tone.\n" + exits: { next: d } + d: + kind: hours + schedule: + mon: + - { open: "09:00", close: "17:00" } + timezone: America/New_York + exceptions: + - { date: "2026-12-25", closed: true } + exits: { open: e, closed: e } + e: + kind: hangup + prompt: "Goodbye — 再见.\tTab, \"quotes\" and a backslash \\" diff --git a/conformance/canonical/scalars.expected.json b/conformance/canonical/scalars.expected.json new file mode 100644 index 0000000..ee5c91f --- /dev/null +++ b/conformance/canonical/scalars.expected.json @@ -0,0 +1,4 @@ +{ + "description": "Plain where the string can't be misread, double-quoted otherwise: reserved words, times, dates, leading digits, `: ` and ` #`, leading or trailing spaces, flow indicators and `:` in flow style, and anything needing an escape. Multi-line text becomes one double-quoted line.", + "comments": 0 +} diff --git a/conformance/canonical/scalars.input.yaml b/conformance/canonical/scalars.input.yaml new file mode 100644 index 0000000..939729e --- /dev/null +++ b/conformance/canonical/scalars.input.yaml @@ -0,0 +1,33 @@ +schema_version: 1 +id: flow_scalars +name: 'Quotes: only where needed' +entry: a +nodes: + a: + kind: greeting + prompt: Thanks for calling, how can we help? + note: "yes" + exits: { next: b } + b: + kind: menu + prompt: 'It''s 9: press 1 or # for more' + options: { "1": "Sales, new orders", "2": Plain label, "3": "Ext 5:30" } + exits: { "1": c, "2": c, no_input: c, invalid: c } + c: + kind: message + prompt: | + Please leave a message + after the tone. + note: " leading space" + exits: { next: d } + d: + kind: hours + timezone: America/New_York + schedule: + mon: [{ open: "09:00", close: "17:00" }] + exceptions: + - { date: "2026-12-25", closed: true } + exits: { open: e, closed: e } + e: + kind: hangup + prompt: "Goodbye — 再见.\tTab, \"quotes\" and a backslash \\" diff --git a/conformance/canonical/ui.canonical.yaml b/conformance/canonical/ui.canonical.yaml new file mode 100644 index 0000000..0c58dc6 --- /dev/null +++ b/conformance/canonical/ui.canonical.yaml @@ -0,0 +1,22 @@ +schema_version: 2 +id: flow_ui +name: The ui block is opaque +entry: a +nodes: + a: + kind: hangup + prompt: { audio: vprompt_a1, transcript: Bye } +ui: + zoom: 1 + grid: { snap: true, size: 0.5, origin: null } + positions: + "1": [0, 0] + a: [120, 40.5] + tiny: 1e-7 + huge: 1e+21 + layers: + - { name: main, visible: true } + - [1, 2] + - [] + - {} + tags: [] diff --git a/conformance/canonical/ui.expected.json b/conformance/canonical/ui.expected.json new file mode 100644 index 0000000..5f5c525 --- /dev/null +++ b/conformance/canonical/ui.expected.json @@ -0,0 +1,4 @@ +{ + "description": "`ui` is written as it was given, with nothing left out: collections of scalars on one line, numbers as JavaScript writes them, integer keys first.", + "comments": 0 +} diff --git a/conformance/canonical/ui.input.yaml b/conformance/canonical/ui.input.yaml new file mode 100644 index 0000000..7c1048f --- /dev/null +++ b/conformance/canonical/ui.input.yaml @@ -0,0 +1,23 @@ +schema_version: 2 +id: flow_ui +name: The ui block is opaque +entry: a +nodes: + a: + kind: hangup + prompt: { audio: vprompt_a1, transcript: Bye } +ui: + zoom: 1.0 + grid: { snap: true, size: 0.5, origin: null } + positions: + a: [120, 40.5] + "1": [0, 0] + tiny: 0.0000001 + huge: 1.0e+21 + layers: + - name: main + visible: true + - [1, 2] + - [] + - {} + tags: [] diff --git a/crates/wavekat-flow/Cargo.toml b/crates/wavekat-flow/Cargo.toml index 163adad..4721189 100644 --- a/crates/wavekat-flow/Cargo.toml +++ b/crates/wavekat-flow/Cargo.toml @@ -20,7 +20,10 @@ exclude = ["tests/"] anyhow = "1" async-trait = "0.1" serde = { version = "1", features = ["derive"] } -serde_json = "1" +# `preserve_order`: the opaque `ui` block keeps the key order it was +# written in, which the canonical serialization depends on. +serde_json = { version = "1", features = ["preserve_order"] } +indexmap = { version = "2", features = ["serde"] } serde_yaml_ng = "0.10" thiserror = "2" time = { version = "0.3", features = ["macros", "parsing"] } diff --git a/crates/wavekat-flow/README.md b/crates/wavekat-flow/README.md index 2865012..c9d8bcb 100644 --- a/crates/wavekat-flow/README.md +++ b/crates/wavekat-flow/README.md @@ -29,6 +29,8 @@ this crate. |--------|--------------| | `model` (re-exported at root) | Generated document types (`Flow`, `Node`, `Prompt`, …) plus hand-written helpers (`Flow::from_yaml`, `Node::exits`, …) | | `validate` | Semantic validation: reachability, caller traps, exit-set exactness, DTMF and prompt rules | +| `edit` | Load a document to edit it: `edit::load` refuses unknown fields (a save would delete them) and unsupported versions, and returns the YAML comments a save drops; `edit::load_draft` fills a new draft's missing `id`/`name` | +| `Flow::to_yaml` | The canonical text of a flow, byte for byte what the TypeScript `serializeFlow` writes. Edit by parsing, changing the flow, and calling this | | `hours` | Business-hours / timezone math (IANA zones, holiday exceptions) | | `engine` | The flow interpreter + the `FlowEffects` async trait your runtime implements | | `trace` | Serializable run traces emitted by the engine | @@ -97,8 +99,9 @@ meantime, which is the safe direction. ## One format, two languages -This crate is the Rust half of the format. Authoring, comment-preserving -edits, and the TypeScript twin of the validator live in +This crate is the Rust half of the format. The structured edit operations the +web editor uses, and the TypeScript twins of the validator and the canonical +serializer, live in [`@wavekat/flow-schema`](https://www.npmjs.com/package/@wavekat/flow-schema). The schema, the shared conformance corpus, and the full design rationale live in the [`wavekat-flow` repository](https://github.com/wavekat/wavekat-flow). diff --git a/crates/wavekat-flow/build.rs b/crates/wavekat-flow/build.rs index aa61124..b513cec 100644 --- a/crates/wavekat-flow/build.rs +++ b/crates/wavekat-flow/build.rs @@ -71,8 +71,14 @@ fn main() { let schema: schemars::schema::RootSchema = serde_json::from_value(value).expect("schema into RootSchema"); - let mut type_space = - typify::TypeSpace::new(typify::TypeSpaceSettings::default().with_struct_builder(false)); + // Every map is an `IndexMap`: the order nodes, exits and options are + // written in is the author's, and the canonical serialization keeps it + // (docs/01-canonical-document.md §3). + let mut type_space = typify::TypeSpace::new( + typify::TypeSpaceSettings::default() + .with_struct_builder(false) + .with_map_type("::indexmap::IndexMap"), + ); type_space .add_root_schema(schema) .expect("typify add_root_schema"); diff --git a/crates/wavekat-flow/schema/flow.v1.schema.json b/crates/wavekat-flow/schema/flow.v1.schema.json index 2fe2e66..1a5c734 100644 --- a/crates/wavekat-flow/schema/flow.v1.schema.json +++ b/crates/wavekat-flow/schema/flow.v1.schema.json @@ -25,12 +25,16 @@ "minimum": 0, "default": 1 }, + "note": { + "description": "The author's note about the whole flow: free text for people reading it, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "entry": { "description": "Node id where execution begins.", "type": "string" }, "nodes": { - "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution.", + "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution, but it is the author's reading order and is kept on save.", "type": "object", "additionalProperties": { "$ref": "#/$defs/Node" } }, @@ -120,6 +124,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "greeting" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -130,6 +135,7 @@ "required": ["kind", "schedule", "timezone"], "properties": { "kind": { "const": "hours" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "schedule": { "$ref": "#/$defs/WeeklySchedule" }, "timezone": { "description": "IANA zone (America/New_York). Validated to resolve at load.", "type": "string" }, "exceptions": { "type": "array", "items": { "$ref": "#/$defs/HoursException" } }, @@ -142,6 +148,7 @@ "required": ["kind", "prompt", "options"], "properties": { "kind": { "const": "menu" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "options": { "description": "Digit key → human label. The exit for a digit is exits[digit]. Valid digit keys are enforced by the validator.", @@ -159,6 +166,7 @@ "required": ["kind", "timeout_secs"], "properties": { "kind": { "const": "ring" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "timeout_secs": { "type": "integer", "minimum": 0 }, "exits": { "$ref": "#/$defs/Exits" } } @@ -169,6 +177,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "message" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "max_secs": { "type": "integer", "minimum": 0, "default": 120 }, "tone": { "allOf": [{ "$ref": "#/$defs/MessageTone" }], "default": "beep" }, @@ -181,6 +190,7 @@ "required": ["kind", "target"], "properties": { "kind": { "const": "transfer" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "target": { "type": "string" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -191,6 +201,7 @@ "required": ["kind"], "properties": { "kind": { "const": "hangup" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } diff --git a/crates/wavekat-flow/schema/flow.v2.schema.json b/crates/wavekat-flow/schema/flow.v2.schema.json index 9ce22c1..b9101a3 100644 --- a/crates/wavekat-flow/schema/flow.v2.schema.json +++ b/crates/wavekat-flow/schema/flow.v2.schema.json @@ -25,12 +25,16 @@ "minimum": 0, "default": 1 }, + "note": { + "description": "The author's note about the whole flow: free text for people reading it, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "entry": { "description": "Node id where execution begins.", "type": "string" }, "nodes": { - "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution.", + "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution, but it is the author's reading order and is kept on save.", "type": "object", "additionalProperties": { "$ref": "#/$defs/Node" } }, @@ -121,6 +125,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "greeting" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -131,6 +136,7 @@ "required": ["kind", "schedule", "timezone"], "properties": { "kind": { "const": "hours" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "schedule": { "$ref": "#/$defs/WeeklySchedule" }, "timezone": { "description": "IANA zone (America/New_York). Validated to resolve at load.", "type": "string" }, "exceptions": { "type": "array", "items": { "$ref": "#/$defs/HoursException" } }, @@ -143,6 +149,7 @@ "required": ["kind", "prompt", "options"], "properties": { "kind": { "const": "menu" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "options": { "description": "Digit key → human label. The exit for a digit is exits[digit]. Valid digit keys are enforced by the validator.", @@ -160,6 +167,7 @@ "required": ["kind", "timeout_secs"], "properties": { "kind": { "const": "ring" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "timeout_secs": { "type": "integer", "minimum": 0 }, "exits": { "$ref": "#/$defs/Exits" } } @@ -170,6 +178,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "message" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "max_secs": { "type": "integer", "minimum": 0, "default": 120 }, "tone": { "allOf": [{ "$ref": "#/$defs/MessageTone" }], "default": "beep" }, @@ -182,6 +191,7 @@ "required": ["kind", "target"], "properties": { "kind": { "const": "transfer" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "target": { "type": "string" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -192,6 +202,7 @@ "required": ["kind"], "properties": { "kind": { "const": "hangup" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -202,6 +213,7 @@ "required": ["kind", "prompt", "confirm_prompt", "schedule", "timezone", "duration_mins"], "properties": { "kind": { "const": "book" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "confirm_prompt": { "$ref": "#/$defs/Prompt" }, "schedule": { "$ref": "#/$defs/WeeklySchedule" }, diff --git a/crates/wavekat-flow/src/canonical.rs b/crates/wavekat-flow/src/canonical.rs new file mode 100644 index 0000000..88e661e --- /dev/null +++ b/crates/wavekat-flow/src/canonical.rs @@ -0,0 +1,505 @@ +//! The canonical text of a flow (docs/01-canonical-document.md §4). +//! +//! One document has one text, the same in every language: the TypeScript +//! twin is `packages/flow-schema/src/serialize.ts`, and the corpus under +//! `conformance/canonical/` pins both byte for byte. The emitter is written +//! by hand, because YAML libraries choose layout and quoting their own way, +//! and it is guided by the bundled schema: property order, which properties +//! are required, and their defaults all come from there, so a schema change +//! needs no change here. + +use std::sync::OnceLock; + +use serde_json::{Map, Value}; + +use crate::{Flow, FLOW_V2_SCHEMA}; + +/// The newest schema, which the model is generated from. +pub(crate) fn schema() -> &'static Value { + static SCHEMA: OnceLock = OnceLock::new(); + SCHEMA.get_or_init(|| serde_json::from_str(FLOW_V2_SCHEMA).expect("bundled schema is JSON")) +} + +/// Follow `$ref` (and a single-entry `allOf`, the schema's way of putting a +/// default next to a ref) to the schema that describes a value. +fn resolve(mut s: &'static Value) -> &'static Value { + loop { + if let Some(r) = s.get("$ref").and_then(Value::as_str) { + let name = r.strip_prefix("#/$defs/").expect("refs point into $defs"); + s = &schema()["$defs"][name]; + } else if let Some([only]) = s.get("allOf").and_then(Value::as_array).map(Vec::as_slice) { + s = only; + } else { + return s; + } + } +} + +/// The schema for `value`, picking a `oneOf` branch: by `kind` for a node, +/// by JSON type otherwise (a text or audio prompt). +pub(crate) fn describe(s: Option<&'static Value>, value: &Value) -> Option<&'static Value> { + let s = resolve(s?); + let Some(branches) = s.get("oneOf").and_then(Value::as_array) else { + return Some(s); + }; + branches.iter().map(resolve).find(|b| { + if let Some(kind) = b.pointer("/properties/kind/const") { + return value.get("kind") == Some(kind); + } + match b.get("type").and_then(Value::as_str) { + Some("string") => value.is_string(), + Some("object") => value.is_object(), + Some("array") => value.is_array(), + Some("integer") | Some("number") => value.is_number(), + Some("boolean") => value.is_boolean(), + _ => false, + } + }) +} + +/// The canonical text of `flow`. +pub(crate) fn to_yaml(flow: &Flow) -> String { + let value = serde_json::to_value(flow).expect("a flow always serializes"); + let mut out = String::new(); + // The document, `nodes` and each node are block mappings, whatever + // they hold. + write_block_map( + &mut out, + &entries(&value, Some(schema())), + 0, + Level::Document, + false, + ); + out +} + +/// Where a mapping sits, for the "always block" rule. +#[derive(Clone, Copy, PartialEq)] +enum Level { + Document, + Nodes, + Node, + Other, +} + +impl Level { + fn child(self, key: &str) -> Level { + match (self, key) { + (Level::Document, "nodes") => Level::Nodes, + (Level::Nodes, _) => Level::Node, + _ => Level::Other, + } + } +} + +/// One entry to write: its key, its value and the schema describing it. +type Entry<'v> = (&'v str, &'v Value, Option<&'static Value>); + +/// A mapping's entries in canonical order, with omitted ones left out. +fn entries<'v>(value: &'v Value, s: Option<&'static Value>) -> Vec> { + let Value::Object(map) = value else { + unreachable!("entries of a mapping") + }; + let s = describe(s, value); + let props = s + .and_then(|s| s.get("properties")) + .and_then(Value::as_object); + let Some(props) = props else { + // A map the schema gives no fixed order to. + let items = s + .and_then(|s| s.get("additionalProperties")) + .filter(|a| a.is_object()); + return map_order(map) + .into_iter() + .map(|(k, v)| (k, v, items)) + .collect(); + }; + let required: Vec<&str> = s + .and_then(|s| s.get("required")) + .and_then(Value::as_array) + .map(|r| r.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + let mut out = Vec::new(); + for (key, prop) in props { + let Some(value) = map.get(key) else { continue }; + if !required.contains(&key.as_str()) && omitted(value, resolve_default(prop)) { + continue; + } + out.push((key.as_str(), value, Some(prop))); + } + // Keys the schema doesn't know follow, in map order. (The typed model + // never has any; this keeps the output defined for any input.) + let unknown: Map = map + .iter() + .filter(|(k, _)| !props.contains_key(*k)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + for (k, _) in map_order(&unknown) { + let (k, v) = map.get_key_value(k).expect("key from this map"); + out.push((k.as_str(), v, None)); + } + out +} + +/// A property's `default`, looking through a single-entry `allOf`. +fn resolve_default(prop: &Value) -> Option<&Value> { + prop.get("default").or_else(|| { + prop.get("allOf") + .and_then(Value::as_array) + .and_then(|a| a.first()) + .and_then(|f| f.get("default")) + }) +} + +/// Is an optional property left out: absent, its default, or empty? +fn omitted(value: &Value, default: Option<&Value>) -> bool { + match value { + Value::Null => true, + Value::Object(m) if m.is_empty() => true, + Value::Array(a) if a.is_empty() => true, + _ => default.is_some_and(|d| numbers_equal(d, value) || d == value), + } +} + +fn numbers_equal(a: &Value, b: &Value) -> bool { + match (a.as_f64(), b.as_f64()) { + (Some(x), Some(y)) => a.is_number() && b.is_number() && x == y, + _ => false, + } +} + +/// Map order (§3): keys that are canonical array indices first, ascending, +/// then the rest in the order they were written. JavaScript objects order +/// keys this way, so the TypeScript twin can't do otherwise. +fn map_order(map: &Map) -> Vec<(&str, &Value)> { + let mut indices: Vec<(u32, &str, &Value)> = Vec::new(); + let mut rest = Vec::new(); + for (k, v) in map { + match array_index(k) { + Some(i) => indices.push((i, k, v)), + None => rest.push((k.as_str(), v)), + } + } + indices.sort_by_key(|(i, _, _)| *i); + indices + .into_iter() + .map(|(_, k, v)| (k, v)) + .chain(rest) + .collect() +} + +/// `Some(n)` when `key` is a canonical array index: `0`, or digits with no +/// leading zero, below 2^32 - 1. +fn array_index(key: &str) -> Option { + if key.is_empty() || !key.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + if key.len() > 1 && key.starts_with('0') { + return None; + } + key.parse::().ok().filter(|n| *n < u32::MAX) +} + +fn is_scalar(v: &Value) -> bool { + !matches!(v, Value::Object(_) | Value::Array(_)) +} + +/// Is a collection written in flow style? When every entry it writes +/// (after omissions) is a scalar. +fn is_flow(v: &Value, s: Option<&'static Value>) -> bool { + match v { + Value::Object(_) => entries(v, s).iter().all(|(_, x, _)| is_scalar(x)), + Value::Array(a) => a.iter().all(is_scalar), + _ => false, + } +} + +fn pad(out: &mut String, indent: usize) { + out.extend(std::iter::repeat_n(' ', indent)); +} + +/// Write a block mapping at `indent`. When `on_dash_line`, the first entry +/// continues the current `- ` line of a list item. +fn write_block_map( + out: &mut String, + entries: &[Entry<'_>], + indent: usize, + level: Level, + on_dash_line: bool, +) { + for (i, (key, value, s)) in entries.iter().enumerate() { + if i > 0 || !on_dash_line { + pad(out, indent); + } + out.push_str(&scalar_string(key, false)); + out.push(':'); + write_value(out, value, *s, indent, level.child(key)); + } +} + +/// Write `value` after `key:`, then end the line. +fn write_value( + out: &mut String, + value: &Value, + s: Option<&'static Value>, + indent: usize, + level: Level, +) { + let always_block = matches!(level, Level::Nodes | Level::Node); + match value { + Value::Object(m) if m.is_empty() => out.push_str(" {}\n"), + Value::Array(a) if a.is_empty() => out.push_str(" []\n"), + Value::Object(_) if always_block || !is_flow(value, s) => { + out.push('\n'); + write_block_map(out, &entries(value, s), indent + 2, level, false); + } + Value::Array(a) if !is_flow(value, s) => { + out.push('\n'); + write_block_list(out, a, items_of(s), indent + 2); + } + _ => { + out.push(' '); + write_inline(out, value, s); + out.push('\n'); + } + } +} + +/// The schema of a list's items. +fn items_of(s: Option<&'static Value>) -> Option<&'static Value> { + s.map(resolve).and_then(|s| s.get("items")) +} + +fn write_block_list(out: &mut String, items: &[Value], s: Option<&'static Value>, indent: usize) { + for item in items { + pad(out, indent); + match item { + Value::Object(m) if !m.is_empty() && !is_flow(item, s) => { + out.push_str("- "); + write_block_map(out, &entries(item, s), indent + 2, Level::Other, true); + } + Value::Array(a) if !a.is_empty() && !is_flow(item, s) => { + out.push_str("-\n"); + write_block_list(out, a, items_of(s), indent + 2); + } + _ => { + out.push_str("- "); + write_inline(out, item, s); + out.push('\n'); + } + } + } +} + +/// A scalar, or a flow collection of scalars, on one line. +fn write_inline(out: &mut String, value: &Value, s: Option<&'static Value>) { + match value { + Value::Object(m) if m.is_empty() => out.push_str("{}"), + Value::Object(_) => { + out.push_str("{ "); + for (i, (k, v, _)) in entries(value, s).iter().enumerate() { + if i > 0 { + out.push_str(", "); + } + out.push_str(&scalar_string(k, true)); + out.push_str(": "); + write_scalar(out, v, true); + } + out.push_str(" }"); + } + Value::Array(a) if a.is_empty() => out.push_str("[]"), + Value::Array(a) => { + out.push('['); + for (i, v) in a.iter().enumerate() { + if i > 0 { + out.push_str(", "); + } + write_scalar(out, v, true); + } + out.push(']'); + } + _ => write_scalar(out, value, false), + } +} + +fn write_scalar(out: &mut String, value: &Value, flow: bool) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }), + Value::Number(n) => out.push_str(&number_string(n)), + Value::String(s) => out.push_str(&scalar_string(s, flow)), + Value::Object(_) | Value::Array(_) => unreachable!("only scalars are written inline"), + } +} + +/// A number as JavaScript's `String(n)` writes it. +fn number_string(n: &serde_json::Number) -> String { + if n.is_i64() || n.is_u64() { + return n.to_string(); + } + let f = n.as_f64().expect("a JSON number is finite"); + if f == 0.0 { + return "0".into(); + } + let abs = f.abs(); + if abs >= 1e21 { + // Rust writes `1e21`; JavaScript `1e+21`. + return format!("{f:e}").replacen('e', "e+", 1); + } + if abs < 1e-6 { + return format!("{f:e}"); + } + // Shortest round-trip digits, no exponent: the same as JavaScript in + // this range. + format!("{f}") +} + +/// Words some YAML readers take for booleans, null or numbers. +const RESERVED: &[&str] = &[ + "true", "false", "yes", "no", "on", "off", "y", "n", "null", "inf", "nan", "infinity", +]; + +/// Characters a double-quoted string writes as `\uXXXX`. +fn needs_unicode_escape(c: char) -> bool { + matches!(c, '\0'..='\u{1f}' | '\u{7f}'..='\u{9f}' | '\u{2028}' | '\u{2029}' | '\u{feff}') +} + +/// Can `s` be written as a plain scalar (§4)? +pub(crate) fn is_plain(s: &str, flow: bool) -> bool { + let Some(first) = s.chars().next() else { + return false; + }; + if !(first.is_ascii_alphabetic() || first == '_' || first == '/' || !first.is_ascii()) { + return false; + } + if s.ends_with(' ') || s.ends_with(':') { + return false; + } + if s.contains(": ") || s.contains(" #") { + return false; + } + if s.chars().any(|c| c == '\t' || needs_unicode_escape(c)) { + return false; + } + if flow && s.contains([',', '[', ']', '{', '}', ':']) { + return false; + } + !RESERVED.contains(&s.to_ascii_lowercase().as_str()) +} + +/// `s` as a plain scalar when it can be one, double-quoted otherwise. +fn scalar_string(s: &str, flow: bool) -> String { + if is_plain(s, flow) { + return s.to_string(); + } + let mut out = String::with_capacity(s.len() + 2); + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\t' => out.push_str("\\t"), + '\r' => out.push_str("\\r"), + '\u{8}' => out.push_str("\\b"), + '\u{c}' => out.push_str("\\f"), + c if needs_unicode_escape(c) => out.push_str(&format!("\\u{:04x}", c as u32)), + c => out.push(c), + } + } + out.push('"'); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plain_or_quoted() { + for plain in [ + "Hello", + "Thanks for calling, how can we help?", + "再见", + "_x", + "/a/b", + "C#", + "it's", + ] { + assert!(is_plain(plain, false), "{plain}"); + } + for quoted in [ + "", + "09:00", + "2026-12-25", + "1", + "-x", + ".inf", + "yes", + "No", + "a: b", + "a #b", + "end ", + "key:", + "line\nbreak", + "tab\there", + "'q'", + "\"q\"", + "[x]", + "*a", + "&a", + "!t", + "%d", + "@x", + "`x", + "|x", + ">x", + "#x", + "?x", + "~", + ] { + assert!(!is_plain(quoted, false), "{quoted:?}"); + } + assert!(is_plain("a, b", false)); + assert!(!is_plain("a, b", true)); + assert!(!is_plain("a{b", true)); + // libyaml refuses a `:` inside a plain scalar in flow style. + assert!(is_plain("a:b", false)); + assert!(!is_plain("a:b", true)); + } + + #[test] + fn escapes() { + // Quotes and backslashes are fine in a plain scalar; escaped once quoted. + assert_eq!(scalar_string("say \"hi\"\\", false), "say \"hi\"\\"); + assert_eq!(scalar_string("\"hi\" \\", false), r#""\"hi\" \\""#); + assert_eq!(scalar_string("a\nb\tc", false), r#""a\nb\tc""#); + assert_eq!( + scalar_string("\u{7f}\u{1}\u{2028}", false), + r#""\u007f\u0001\u2028""# + ); + assert_eq!(scalar_string("1", true), r#""1""#); + } + + #[test] + fn numbers_like_javascript() { + let n = |s: &str| number_string(&serde_json::from_str(s).unwrap()); + assert_eq!(n("120"), "120"); + assert_eq!(n("120.0"), "120"); + assert_eq!(n("-0.0"), "0"); + assert_eq!(n("0.5"), "0.5"); + assert_eq!(n("1e21"), "1e+21"); + assert_eq!(n("1.5e21"), "1.5e+21"); + assert_eq!(n("1e20"), "100000000000000000000"); + assert_eq!(n("1e-7"), "1e-7"); + assert_eq!(n("0.000001"), "0.000001"); + } + + #[test] + fn array_indices_first() { + let m: Map = + serde_json::from_str(r#"{"b":1,"2":1,"a":1,"0":1,"01":1,"10":1}"#).unwrap(); + let keys: Vec<&str> = map_order(&m).into_iter().map(|(k, _)| k).collect(); + assert_eq!(keys, ["0", "2", "10", "b", "a", "01"]); + } +} diff --git a/crates/wavekat-flow/src/edit.rs b/crates/wavekat-flow/src/edit.rs new file mode 100644 index 0000000..67c6880 --- /dev/null +++ b/crates/wavekat-flow/src/edit.rs @@ -0,0 +1,399 @@ +//! Loading a document to edit it (docs/01-canonical-document.md §5). +//! +//! Editing is [`Flow::from_yaml`], change the flow, [`Flow::to_yaml`]. Two +//! things in a source can't survive that round trip, and an editor has to +//! know about them before it saves: +//! +//! - **Fields this build doesn't know.** The model drops them, so saving +//! would delete them. [`load`] refuses such a document. +//! - **YAML comments.** They carry no meaning (the words belong in +//! `note`), but the author should be told a save drops them. [`load`] +//! returns them; [`comments`] finds them in any text. The TypeScript +//! parser reports the same comments as `comment_ignored`, and the corpus +//! under `conformance/canonical/` pins the count for both. + +use serde_json::Value; + +use crate::canonical::{describe, schema}; +use crate::{Flow, SUPPORTED_SCHEMA_VERSIONS}; + +/// A YAML comment in a source. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Comment { + /// 1-based line number. + pub line: usize, + /// The words after `#`, trimmed. + pub text: String, +} + +/// A document ready to edit. +#[derive(Debug, Clone)] +pub struct Loaded { + pub flow: Flow, + /// Comments in the source, which saving drops. + pub comments: Vec, +} + +/// Why a document can't be edited. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum LoadError { + /// It isn't a flow document. + #[error("the flow doesn't parse: {0}")] + Parse(String), + /// It declares a version this build can't write back. + #[error( + "the flow declares schema_version {0}, which this version of the tool doesn't support" + )] + UnsupportedVersion(i64), + /// It has fields this build doesn't know; saving would delete them. + #[error("unknown fields would be lost on save: {}", .0.join(", "))] + UnknownFields(Vec), +} + +/// Parse `src` for editing: refuse what a save would lose, and return the +/// comments it would drop. +pub fn load(src: &str) -> Result { + load_value(src, None) +} + +/// The platform-assigned fields a new draft may leave out. A draft is +/// complete enough to edit once they're filled; the platform stamps the +/// real ones when the draft is saved there. Twin: `mutate.ts` +/// `stampIdentity`, which does the same for a draft without an `id`. +#[derive(Debug, Clone, Copy)] +pub struct Missing<'a> { + /// Used when the draft has no `id`. + pub id: &'a str, + /// Used when the draft has no `name`. + pub name: &'a str, +} + +/// [`load`] for a draft that may leave out `id` and `name`: each one +/// missing is filled from `missing`, then the draft is loaded like any +/// other document. +pub fn load_draft(src: &str, missing: Missing<'_>) -> Result { + load_value(src, Some(missing)) +} + +fn load_value(src: &str, missing: Option>) -> Result { + let parse = |e: &dyn std::fmt::Display| LoadError::Parse(e.to_string()); + // Through `serde_yaml_ng::Value` first: its mapping rejects duplicate + // keys at any depth, as `Flow::from_yaml` does. + let yaml: serde_yaml_ng::Value = serde_yaml_ng::from_str(src).map_err(|e| parse(&e))?; + let mut value = serde_json::to_value(&yaml).map_err(|e| parse(&e))?; + if let (Some(missing), Value::Object(map)) = (missing, &mut value) { + for (key, fill) in [("id", missing.id), ("name", missing.name)] { + map.entry(key) + .or_insert_with(|| Value::String(fill.to_string())); + } + } + let flow: Flow = serde_json::from_value(value.clone()).map_err(|e| parse(&e))?; + let supported = + u32::try_from(flow.schema_version).is_ok_and(|v| SUPPORTED_SCHEMA_VERSIONS.contains(&v)); + if !supported { + return Err(LoadError::UnsupportedVersion(flow.schema_version)); + } + let unknown = unknown_in(&value); + if !unknown.is_empty() { + return Err(LoadError::UnknownFields(unknown)); + } + Ok(Loaded { + flow, + comments: comments(src), + }) +} + +/// The dotted paths (`nodes.menu.colour`) of fields in `src` the schema +/// doesn't describe. Nothing inside the opaque `ui` block counts. +pub fn unknown_fields(src: &str) -> Result, LoadError> { + let value: Value = serde_yaml_ng::from_str(src).map_err(|e| LoadError::Parse(e.to_string()))?; + Ok(unknown_in(&value)) +} + +fn unknown_in(value: &Value) -> Vec { + let mut out = Vec::new(); + walk(value, Some(schema()), "", &mut out); + out +} + +fn walk(value: &Value, s: Option<&'static Value>, path: &str, out: &mut Vec) { + let Some(s) = describe(s, value) else { return }; + let join = |key: &str| { + if path.is_empty() { + key.to_string() + } else { + format!("{path}.{key}") + } + }; + match value { + Value::Object(map) => { + if let Some(props) = s.get("properties").and_then(Value::as_object) { + for (k, v) in map { + match props.get(k) { + Some(p) => walk(v, Some(p), &join(k), out), + None => out.push(join(k)), + } + } + } else if let Some(items) = s.get("additionalProperties").filter(|a| a.is_object()) { + for (k, v) in map { + walk(v, Some(items), &join(k), out); + } + } + } + Value::Array(items) => { + if let Some(item) = s.get("items") { + for (i, v) in items.iter().enumerate() { + walk(v, Some(item), &join(&i.to_string()), out); + } + } + } + _ => {} + } +} + +#[derive(Clone, Copy, PartialEq)] +enum Quote { + Single, + Double, +} + +/// Every comment in `src`, in order. +/// +/// A `#` starts a comment at the start of a line's content or after a +/// space or tab, unless it's inside a quoted scalar or a block scalar's +/// content (`|` / `>`). Quotes only open a scalar where a value or key +/// starts (after `: `, `- `, `? `, `[`, `{`, `,` or at the start of a +/// line), so the apostrophe in `It's open` is just a letter. A `#` right +/// after a closing quote (`"x"# c`) also starts one, as in YAML parsers +/// that read the scalar first. A leading byte order mark is skipped. +pub fn comments(src: &str) -> Vec { + let src = src.strip_prefix('\u{feff}').unwrap_or(src); + let mut out = Vec::new(); + let mut quote: Option = None; + // Indent of the line that opened a block scalar still being read. + let mut block: Option = None; + + for (n, line) in src.lines().enumerate() { + let indent = line.len() - line.trim_start_matches(' ').len(); + if let Some(opener) = block { + if line.trim().is_empty() || indent > opener { + continue; + } + block = None; + } + + let chars: Vec<(usize, char)> = line.char_indices().collect(); + // The last non-blank character before the current one on this line. + let mut last: Option = None; + // The index just after a quoted scalar that closed on this line. + let mut closed_at: Option = None; + let mut i = 0; + while i < chars.len() { + let (at, c) = chars[i]; + match quote { + Some(Quote::Double) => { + match c { + '\\' => i += 1, + '"' => { + quote = None; + closed_at = Some(i + 1); + } + _ => {} + } + last = Some(c); + i += 1; + continue; + } + Some(Quote::Single) => { + if c == '\'' { + if chars.get(i + 1).map(|(_, c)| *c) == Some('\'') { + i += 1; + } else { + quote = None; + closed_at = Some(i + 1); + } + } + last = Some(c); + i += 1; + continue; + } + None => {} + } + let after_blank = i == 0 || matches!(chars[i - 1].1, ' ' | '\t'); + let value_start = match last { + None => true, + Some(':' | '-' | '?') => after_blank, + Some('[' | '{' | ',') => true, + _ => false, + }; + match c { + '#' if after_blank || last.is_none() || closed_at == Some(i) => { + out.push(Comment { + line: n + 1, + text: line[at + 1..].trim().to_string(), + }); + break; + } + '"' if value_start => quote = Some(Quote::Double), + '\'' if value_start => quote = Some(Quote::Single), + '|' | '>' if value_start && last.is_some() => block = Some(indent), + _ => {} + } + if !matches!(c, ' ' | '\t') { + last = Some(c); + } + i += 1; + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lines(src: &str) -> Vec<(usize, String)> { + comments(src) + .into_iter() + .map(|c| (c.line, c.text)) + .collect() + } + + #[test] + fn finds_comments_and_only_comments() { + let src = "\ +# top +name: Test # the name +nodes: + # before a node + a: + kind: greeting + prompt: \"Press # for more\" # trailing + note: It's C# day + b: + kind: message + prompt: 'it''s # not a comment' + c: + kind: message + prompt: | + # inside a block scalar + still inside + # after the block + exits: { next: a } # flow + d: { kind: hangup, prompt: \"multi + line # still quoted\" } +"; + assert_eq!( + lines(src), + vec![ + (1, "top".into()), + (2, "the name".into()), + (4, "before a node".into()), + (7, "trailing".into()), + (17, "after the block".into()), + (18, "flow".into()), + ] + ); + } + + #[test] + fn comments_edge_cases() { + // A byte order mark doesn't hide the first line's comment. + assert_eq!(lines("\u{feff}# bom\na: 1\n"), [(1, "bom".into())]); + // A `#` straight after a closing quote is a comment, as in TS. + assert_eq!( + lines("a: \"x\"# c\nb: 'y'# d\n"), + [(1, "c".into()), (2, "d".into())] + ); + // Line numbers count CRLF endings once, and non-ASCII text is fine. + assert_eq!( + lines("a: 1\r\n# crlf\r\nb: 再 # 见\r\n"), + [(2, "crlf".into()), (3, "见".into())] + ); + // A plain `#` with no space before it is part of the scalar. + assert!(comments("a: b#c\nd:# e\n").is_empty()); + } + + #[test] + fn load_refuses_unknown_fields() { + let src = "\ +schema_version: 2 +id: f +name: F +entry: a +colour: blue +nodes: + a: + kind: hangup + shape: round + prompt: { audio: vprompt_a, transcript: Bye, speed: 2 } +ui: + anything: goes +"; + assert_eq!( + load(src).unwrap_err(), + LoadError::UnknownFields(vec![ + "colour".into(), + "nodes.a.shape".into(), + "nodes.a.prompt.speed".into(), + ]) + ); + } + + #[test] + fn load_refuses_unknown_fields_inside_hours() { + // The same four documents the TypeScript decoder warns about. + let doc = |node: &str| { + format!("schema_version: 2\nid: f\nname: F\nentry: a\nnodes:\n a:\n kind: hours\n timezone: UTC\n{node}") + }; + for (node, path) in [ + ( + " schedule:\n mon: [{ open: \"09:00\", close: \"17:00\", x: 1 }]\n", + "nodes.a.schedule.mon.0.x", + ), + ( + " schedule:\n funday: [{ open: \"09:00\", close: \"17:00\" }]\n", + "nodes.a.schedule.funday", + ), + ( + " schedule: {}\n exceptions:\n - { date: \"2026-12-25\", closed: true, zzz: 1 }\n", + "nodes.a.exceptions.0.zzz", + ), + ] { + assert_eq!( + load(&doc(node)).unwrap_err(), + LoadError::UnknownFields(vec![path.into()]), + "{node}" + ); + } + } + + #[test] + fn load_draft_fills_only_what_is_missing() { + let draft = "schema_version: 2\nname: Mine\nentry: a\nnodes:\n a: { kind: hangup }\n"; + assert!(matches!(load(draft), Err(LoadError::Parse(_)))); + let missing = Missing { + id: "flow_new", + name: "Untitled", + }; + let flow = load_draft(draft, missing).unwrap().flow; + assert_eq!((flow.id.as_str(), flow.name.as_str()), ("flow_new", "Mine")); + // Duplicate keys are refused here too. + let dup = "schema_version: 2\nname: A\nname: B\nentry: a\nnodes:\n a: { kind: hangup }\n"; + assert!(matches!(load_draft(dup, missing), Err(LoadError::Parse(_)))); + } + + #[test] + fn load_refuses_unsupported_versions() { + let src = "schema_version: 99\nid: f\nname: F\nentry: a\nnodes:\n a: { kind: hangup }\n"; + assert_eq!(load(src).unwrap_err(), LoadError::UnsupportedVersion(99)); + } + + #[test] + fn load_returns_comments() { + let src = "schema_version: 2\nid: f\nname: F # hi\nentry: a\nnodes:\n a:\n kind: hangup\n note: Bye\n"; + let loaded = load(src).unwrap(); + assert_eq!(loaded.comments.len(), 1); + assert_eq!(loaded.flow.to_yaml(), "schema_version: 2\nid: f\nname: F\nentry: a\nnodes:\n a:\n kind: hangup\n note: Bye\n"); + } +} diff --git a/crates/wavekat-flow/src/engine.rs b/crates/wavekat-flow/src/engine.rs index 8d29279..a6efbd6 100644 --- a/crates/wavekat-flow/src/engine.rs +++ b/crates/wavekat-flow/src/engine.rs @@ -610,7 +610,7 @@ async fn run_menu( fx: &mut E, trace: &mut Trace, prompt: &Prompt, - options: &std::collections::HashMap, + options: &indexmap::IndexMap, retries: u64, timeout_secs: u64, ) -> Result { diff --git a/crates/wavekat-flow/src/hours.rs b/crates/wavekat-flow/src/hours.rs index 3685259..01c2d16 100644 --- a/crates/wavekat-flow/src/hours.rs +++ b/crates/wavekat-flow/src/hours.rs @@ -213,6 +213,7 @@ mod tests { timezone: "America/New_York".into(), exceptions, exits: None, + note: None, } } diff --git a/crates/wavekat-flow/src/lib.rs b/crates/wavekat-flow/src/lib.rs index f913c26..feb7984 100644 --- a/crates/wavekat-flow/src/lib.rs +++ b/crates/wavekat-flow/src/lib.rs @@ -11,7 +11,10 @@ //! [`trace`] output) alongside the generated model, adapted to the generated //! types and pinned by the shared conformance corpus. The engine owns the //! [`engine::FlowEffects`] trait *definition*; the daemon keeps its live impl -//! in its own codebase. Comment-preserving mutation stays TS-only. +//! in its own codebase. Documents are edited through the model: +//! [`Flow::from_yaml`], change the flow, [`Flow::to_yaml`] (the canonical +//! text, identical to the TS side's); [`edit::load`] refuses what a save +//! would lose. /// The generated document model (`Flow`, `Node`, `Prompt`, …), emitted /// from the schema by `typify` into `OUT_DIR/flow_types.rs`. @@ -29,14 +32,24 @@ pub mod model { /// Hand-written helpers layered on the generated model (logic, not shape). mod model_ext; +/// The canonical serialization behind [`Flow::to_yaml`]. +mod canonical; + pub mod book; +pub mod edit; pub mod engine; pub mod hours; pub mod trace; pub mod validate; +/// Re-exported so callers name the generated maps' type without a matching +/// `indexmap` dependency of their own. +pub use indexmap; pub use model::*; -pub use model_ext::{required_assets, NodeId}; +pub use model_ext::{ + is_generated_clip_ref, is_local_audio_ref, prompt_fields, required_assets, NodeId, + PROMPT_FIELDS, +}; /// The normative JSON Schema (draft 2020-12) for format version 1, as a /// string, bundled so consumers can run structural validation without diff --git a/crates/wavekat-flow/src/model_ext.rs b/crates/wavekat-flow/src/model_ext.rs index 31970d5..96129f7 100644 --- a/crates/wavekat-flow/src/model_ext.rs +++ b/crates/wavekat-flow/src/model_ext.rs @@ -12,7 +12,7 @@ //! enum; these methods present the same query surface over the generated //! shape so the validator/engine read the same. -use std::collections::HashMap; +use indexmap::IndexMap; use crate::model::{Flow, Node, Prompt}; @@ -76,7 +76,7 @@ impl Node { /// The node's wired exits (exit name → target node id), or `None` when /// the document omits the block. Every generated variant carries an /// `exits: Option`, so this reads them uniformly. - pub fn exits(&self) -> Option<&HashMap> { + pub fn exits(&self) -> Option<&IndexMap> { let exits = match self { Node::Greeting { exits, .. } | Node::Hours { exits, .. } @@ -107,6 +107,44 @@ impl Node { Node::Hours { .. } | Node::Ring { .. } | Node::Transfer { .. } => Vec::new(), } } + + /// [`Node::prompts`] with the field each comes from (`prompt`, + /// `confirm_prompt`), for a tool that reports or edits prompts by name. + pub fn prompt_slots(&self) -> Vec<(&'static str, &Prompt)> { + let fields = prompt_fields(self.kind()).unwrap_or(&[]); + let prompts = self.prompts(); + self.slot_names(fields).into_iter().zip(prompts).collect() + } + + /// [`Node::prompt_slots`], mutable: voice a text prompt as a clip, or + /// swap a local recording for the clip uploaded from it, then write the + /// flow back with [`Flow::to_yaml`]. + pub fn prompt_slots_mut(&mut self) -> Vec<(&'static str, &mut Prompt)> { + let fields = prompt_fields(self.kind()).unwrap_or(&[]); + let names = self.slot_names(fields); + let prompts: Vec<&mut Prompt> = match self { + Node::Greeting { prompt, .. } + | Node::Menu { prompt, .. } + | Node::Message { prompt, .. } => vec![prompt], + Node::Hangup { prompt, .. } => prompt.iter_mut().collect(), + Node::Book { + prompt, + confirm_prompt, + .. + } => vec![prompt, confirm_prompt], + Node::Hours { .. } | Node::Ring { .. } | Node::Transfer { .. } => Vec::new(), + }; + names.into_iter().zip(prompts).collect() + } + + /// The field names of the prompts this node has: all of `fields`, + /// except a `hangup` with no goodbye has none. + fn slot_names(&self, fields: &'static [&'static str]) -> Vec<&'static str> { + match self { + Node::Hangup { prompt: None, .. } => Vec::new(), + _ => fields.to_vec(), + } + } } /// Every audio asset the flow needs on the device, sorted and unique: @@ -131,6 +169,59 @@ pub fn required_assets(flow: &Flow) -> Vec { refs.into_iter().collect() } +/// Every node field that can hold a prompt, across all kinds. +pub const PROMPT_FIELDS: &[&str] = &["prompt", "confirm_prompt"]; + +/// The fields a node of `kind` speaks a prompt from, in the order +/// [`Node::prompts`] returns them, or `None` for a kind this build doesn't +/// know. Lets a tool name a prompt (`menu.prompt`) or set one by field. +/// Kept in step with [`Node::prompts`] by the corpus test +/// `prompt_fields_match_model`. +pub fn prompt_fields(kind: &str) -> Option<&'static [&'static str]> { + match kind { + "greeting" | "menu" | "message" | "hangup" => Some(&PROMPT_FIELDS[..1]), + "book" => Some(PROMPT_FIELDS), + "hours" | "ring" | "transfer" => Some(&[]), + _ => None, + } +} + +/// A generated-clip ref is a `voice_prompts` id (`vprompt_`), a clip +/// the platform produced and owns. Anything else in an `audio:` field is a +/// ref the platform does *not* own. Twin: `model.ts` `isGeneratedClipRef`. +pub fn is_generated_clip_ref(reference: &str) -> bool { + let Some(rest) = reference + .get(..8) + .filter(|p| p.eq_ignore_ascii_case("vprompt_")) + .map(|_| &reference[8..]) + else { + return false; + }; + !rest.is_empty() && rest.chars().all(|c| c.is_ascii_alphanumeric()) +} + +/// Audio file extensions an author may point an `audio:` at. +const LOCAL_AUDIO_EXTENSIONS: &[&str] = &[ + ".wav", ".wave", ".mp3", ".ogg", ".oga", ".opus", ".flac", ".m4a", ".aac", ".mp4", ".webm", +]; + +/// Does an `audio:` value name a file on the author's machine, to be +/// uploaded before the flow is saved? A path separator or an audio file +/// extension makes it a file; a generated clip ref never is. Any other ref +/// (a clip shipped with the device, say) is neither, and is left alone. +pub fn is_local_audio_ref(reference: &str) -> bool { + let r = reference.trim(); + if r.is_empty() || is_generated_clip_ref(r) { + return false; + } + let lower = r.to_ascii_lowercase(); + r.contains('/') + || r.contains('\\') + || LOCAL_AUDIO_EXTENSIONS + .iter() + .any(|ext| lower.ends_with(ext)) +} + impl Prompt { /// The *synthesizable* text: `Some` for a bare-string prompt the engine /// speaks with TTS, `None` for an audio-asset prompt it plays as a clip. @@ -177,10 +268,14 @@ impl Flow { serde_yaml_ng::from_value(value) } - /// Serialize back to YAML (lossless for the model; the opaque `ui` block - /// round-trips). - pub fn to_yaml(&self) -> Result { - serde_yaml_ng::to_string(self) + /// The canonical text of this flow (docs/01-canonical-document.md §4): + /// the same bytes the TypeScript `serializeFlow` writes for the same + /// flow. Lossless for the model, so editing a document is + /// [`Flow::from_yaml`], change the flow, `to_yaml`. Comments in the + /// source are not part of the model and are not written; see + /// [`crate::edit::load`]. + pub fn to_yaml(&self) -> String { + crate::canonical::to_yaml(self) } } @@ -188,6 +283,32 @@ impl Flow { mod tests { use crate::model::Prompt; + #[test] + fn generated_clip_refs() { + use super::is_generated_clip_ref as clip; + assert!(clip("vprompt_ab12cd34")); + assert!(clip("VPROMPT_AB12")); + assert!(!clip("vprompt_")); + assert!(!clip("vprompt_ab-12")); + assert!(!clip("vprompt_ab12.wav")); + assert!(!clip("shipped_ref")); + assert!(!clip("vprompt")); + assert!(!clip("vpromp€_x")); + } + + #[test] + fn local_audio_refs() { + use super::is_local_audio_ref as local; + assert!(local("./a.wav")); + assert!(local("a.WAV")); + assert!(local("dir/a")); + assert!(local("C:\\clips\\a")); + assert!(local("song.mp3")); + assert!(!local("vprompt_12ab")); + assert!(!local("shipped_ref")); + assert!(!local(" ")); + } + #[test] fn as_text_is_synthesizable_text_only() { // A bare-string prompt is TTS the engine speaks. diff --git a/crates/wavekat-flow/tests/canonical.rs b/crates/wavekat-flow/tests/canonical.rs new file mode 100644 index 0000000..92dc629 --- /dev/null +++ b/crates/wavekat-flow/tests/canonical.rs @@ -0,0 +1,169 @@ +//! The Rust half of the canonical-text contract +//! (docs/01-canonical-document.md §4). The TS package runs the same cases +//! (packages/flow-schema/test/canonical.test.ts), so the two serializers +//! agree byte for byte. + +use std::{fs, path::PathBuf}; + +use wavekat_flow::{edit, prompt_fields, Flow}; + +fn corpus() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../conformance") +} + +#[derive(serde::Deserialize)] +struct Expected { + comments: usize, +} + +fn canonical_cases() -> Vec { + let mut cases: Vec = fs::read_dir(corpus().join("canonical")) + .expect("read canonical corpus") + .filter_map(|e| { + let name = e.ok()?.file_name().to_str()?.to_string(); + name.strip_suffix(".input.yaml").map(str::to_string) + }) + .collect(); + cases.sort(); + assert!(!cases.is_empty(), "no canonical cases"); + cases +} + +fn read(case: &str, ext: &str) -> String { + let path = corpus().join("canonical").join(format!("{case}.{ext}")); + fs::read_to_string(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display())) +} + +#[test] +fn canonical_cases_serialize_exactly() { + for case in canonical_cases() { + let input = read(&case, "input.yaml"); + let canonical = read(&case, "canonical.yaml"); + let expected: Expected = serde_json::from_str(&read(&case, "expected.json")).unwrap(); + + let flow = Flow::from_yaml(&input).unwrap_or_else(|e| panic!("{case}: {e}")); + assert_eq!(flow.to_yaml(), canonical, "{case}: input → canonical"); + + let again = Flow::from_yaml(&canonical).unwrap_or_else(|e| panic!("{case}: {e}")); + assert_eq!( + again.to_yaml(), + canonical, + "{case}: canonical is a fixed point" + ); + + assert_eq!( + edit::comments(&input).len(), + expected.comments, + "{case}: comments in input: {:?}", + edit::comments(&input) + ); + assert!( + edit::comments(&canonical).is_empty(), + "{case}: comments in canonical" + ); + } +} + +/// Every valid document in the corpus, any version, inbound or outbound. +fn valid_documents() -> Vec { + let mut out = Vec::new(); + for version in fs::read_dir(corpus()).unwrap().flatten() { + for bucket in ["valid", "outbound/valid"] { + let Ok(entries) = fs::read_dir(version.path().join(bucket)) else { + continue; + }; + for e in entries.flatten() { + let p = e.path(); + if p.extension().is_some_and(|x| x == "yaml") { + out.push(p); + } + } + } + } + out.sort(); + assert!(!out.is_empty()); + out +} + +#[test] +fn serializing_loses_nothing() { + for path in valid_documents() { + let src = fs::read_to_string(&path).unwrap(); + let flow = Flow::from_yaml(&src).unwrap(); + let text = flow.to_yaml(); + let back = + Flow::from_yaml(&text).unwrap_or_else(|e| panic!("{}: {e}\n{text}", path.display())); + assert_eq!( + serde_json::to_value(&back).unwrap(), + serde_json::to_value(&flow).unwrap(), + "{}: the canonical text means something else", + path.display() + ); + assert_eq!( + back.to_yaml(), + text, + "{}: not a fixed point", + path.display() + ); + } +} + +/// `prompt_fields(kind)` names exactly the fields `Node::prompts()` reads, +/// on every node in the corpus. +#[test] +fn prompt_fields_match_model() { + for path in valid_documents() { + let flow = Flow::from_yaml(&fs::read_to_string(&path).unwrap()).unwrap(); + for (id, node) in &flow.nodes { + let fields = prompt_fields(node.kind()).expect("a known kind"); + let value = serde_json::to_value(node).unwrap(); + let present = fields.iter().filter(|f| value.get(**f).is_some()).count(); + assert_eq!( + present, + node.prompts().len(), + "{}: {id} ({}): prompt_fields disagrees with Node::prompts", + path.display(), + node.kind() + ); + } + } +} + +/// Voicing every text prompt through `prompt_slots_mut` and writing the +/// flow back: only the prompts change. +#[test] +fn voicing_through_prompt_slots() { + use wavekat_flow::Prompt; + for path in valid_documents() { + let mut flow = Flow::from_yaml(&fs::read_to_string(&path).unwrap()).unwrap(); + let names: Vec<(String, Vec<&str>)> = flow + .nodes + .iter() + .map(|(id, n)| { + ( + id.clone(), + n.prompt_slots().iter().map(|(f, _)| *f).collect(), + ) + }) + .collect(); + let mut n = 0; + for node in flow.nodes.values_mut() { + for (_, prompt) in node.prompt_slots_mut() { + if let Prompt::Text(text) = prompt { + *prompt = Prompt::Audio { + audio: format!("vprompt_{n}"), + transcript: Some(text.clone()), + }; + n += 1; + } + } + } + let back = Flow::from_yaml(&flow.to_yaml()).unwrap(); + for (id, fields) in names { + let node = &back.nodes[&id]; + let after: Vec<&str> = node.prompt_slots().iter().map(|(f, _)| *f).collect(); + assert_eq!(after, fields, "{}: {id}", path.display()); + assert!(node.prompts().iter().all(|p| p.as_text().is_none())); + } + } +} diff --git a/docs/01-canonical-document.md b/docs/01-canonical-document.md index c7928c0..350df56 100644 --- a/docs/01-canonical-document.md +++ b/docs/01-canonical-document.md @@ -113,8 +113,9 @@ the two libraries don't agree. The rules: **Layout.** - The document and each node are block mappings, indented two spaces. -- Below a node (and inside `ui`), a map or list whose entries are **all - scalars** is written in flow style on one line: +- Any other map or list (below a node, and inside `ui`) whose written + entries, after the omissions above, are **all scalars** is written in + flow style on one line: `exits: { next: bye }`, `options: { "1": Sales }`, `prompt: { audio: vprompt_a1, transcript: Hi }`, `- { open: "09:00", close: "17:00" }`, `welcome: [120, 40]`. An empty @@ -134,7 +135,8 @@ the two libraries don't agree. The rules: - It doesn't end with a space or `:`. - It contains no `: `, ` #`, tab, line break, or any character that needs an escape (below). - - In flow style, it also contains none of `,` `[` `]` `{` `}`. + - In flow style, it also contains none of `,` `[` `]` `{` `}` `:`. (libyaml, + which Rust readers use, rejects a `:` inside a plain scalar there.) - Lowercased, it isn't `true`, `false`, `yes`, `no`, `on`, `off`, `y`, `n`, `null`, `inf`, `nan` or `infinity`. (YAML 1.1 readers still treat some of these as booleans or numbers.) @@ -165,8 +167,10 @@ the two libraries don't agree. The rules: do): `Flow::from_yaml`, change the typed tree, `Flow::to_yaml`. This works offline and needs no YAML handling in the consumer. Format rules a tool needs live in the crate: `is_generated_clip_ref` (twin of - `isGeneratedClipRef`), `prompt_fields(kind)` (kept in step with - `Node::prompts()` by a corpus test) and `edit::load` (below). + `isGeneratedClipRef`), `is_local_audio_ref`, `prompt_fields(kind)` (kept + in step with `Node::prompts()` by a corpus test), `Node::prompt_slots` + and `prompt_slots_mut` (each prompt with the field it comes from, to + read or change in place) and `edit::load` (below). - **TS**: `mutate.ts` keeps its public API (source in, source out, `EditResult`), so callers don't change. Inside, each op is `parseFlow → change → serializeFlow` instead of CST surgery. A form edit @@ -207,7 +211,8 @@ the platform waits for the npm release. regenerated; the Rust build generates `IndexMap` maps. - Rust: `serde_json` `preserve_order`; the canonical `to_yaml`; `edit::load` (unknown fields, newer version, comments); - `is_generated_clip_ref`, `is_local_audio_ref`, `prompt_fields`. + `is_generated_clip_ref`, `is_local_audio_ref`, `prompt_fields`, + `Node::prompt_slots` and `prompt_slots_mut`. - TS: `serializeFlow`; `note` decoded; `comment_ignored`; `mutate.ts` rebuilt on the model with the `invalid_document` and `unknown_field` refusals. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 2f41938..c27549a 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -31,9 +31,9 @@ Nothing is migrated in the consumer repos yet (that's Phase 3). ### TypeScript — move from the platform repo's `packages/flow-schema/src` - [x] `parse.ts` — safe-subset YAML decode (anchors/aliases/tags/dup-keys/ implicit-typing rejection, `nodeRanges` source offsets). -- [x] `mutate.ts` — comment-preserving edits (stays TS-only; the daemon never - edits). Works on the `yaml` CST, not the generated types; only import - from the model is `ComponentKind`. 20-case suite ported. +- [x] `mutate.ts` — structured edits. Originally comment-preserving edits on + the `yaml` CST; now parse → change the model → `serializeFlow`, the + canonical text shared with Rust (docs/01-canonical-document.md). - [x] `validate.ts` — reachability BFS, caller-trap, exit-set exactness, DTMF, prompt length. ⚠️ dangling-target code reconciled to the frozen corpus: `unknown_exit_target` → `unknown_target`. diff --git a/packages/flow-schema/README.md b/packages/flow-schema/README.md index b9d3244..7f76984 100644 --- a/packages/flow-schema/README.md +++ b/packages/flow-schema/README.md @@ -32,7 +32,9 @@ that executes it. | `validateOutbound` | `validateFlow` plus the rules for a call the app places itself (no `ring`, `book` or `transfer`; no loop that never waits for the callee) — call it instead of `validateFlow` for an outbound flow | | `validateStructure` / `flowV1Schema` | Structural (JSON Schema) validation via `ajv`, and the bundled schema itself | | `validateHours`, `isValidTimezone`, … | Business-hours / timezone checks | -| `addNode`, `setExit`, `renameNode`, … | Comment-preserving YAML source edits (TS-only — the Rust side never edits) | +| `serializeFlow` | The canonical text of a flow, byte for byte what the Rust crate's `Flow::to_yaml` writes | +| `findComments` | The YAML comments in a source (reported by `parseFlow` as `comment_ignored`): they carry no meaning, and saving drops them; the words belong in `note` | +| `addNode`, `setExit`, `renameNode`, … | Structured edits: parse → change the model → `serializeFlow`. Refuse a document that doesn't decode, has unknown fields, or declares an unsupported version | ## Quick start diff --git a/packages/flow-schema/src/generated/flow.v1.schema.json b/packages/flow-schema/src/generated/flow.v1.schema.json index 2fe2e66..1a5c734 100644 --- a/packages/flow-schema/src/generated/flow.v1.schema.json +++ b/packages/flow-schema/src/generated/flow.v1.schema.json @@ -25,12 +25,16 @@ "minimum": 0, "default": 1 }, + "note": { + "description": "The author's note about the whole flow: free text for people reading it, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "entry": { "description": "Node id where execution begins.", "type": "string" }, "nodes": { - "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution.", + "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution, but it is the author's reading order and is kept on save.", "type": "object", "additionalProperties": { "$ref": "#/$defs/Node" } }, @@ -120,6 +124,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "greeting" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -130,6 +135,7 @@ "required": ["kind", "schedule", "timezone"], "properties": { "kind": { "const": "hours" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "schedule": { "$ref": "#/$defs/WeeklySchedule" }, "timezone": { "description": "IANA zone (America/New_York). Validated to resolve at load.", "type": "string" }, "exceptions": { "type": "array", "items": { "$ref": "#/$defs/HoursException" } }, @@ -142,6 +148,7 @@ "required": ["kind", "prompt", "options"], "properties": { "kind": { "const": "menu" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "options": { "description": "Digit key → human label. The exit for a digit is exits[digit]. Valid digit keys are enforced by the validator.", @@ -159,6 +166,7 @@ "required": ["kind", "timeout_secs"], "properties": { "kind": { "const": "ring" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "timeout_secs": { "type": "integer", "minimum": 0 }, "exits": { "$ref": "#/$defs/Exits" } } @@ -169,6 +177,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "message" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "max_secs": { "type": "integer", "minimum": 0, "default": 120 }, "tone": { "allOf": [{ "$ref": "#/$defs/MessageTone" }], "default": "beep" }, @@ -181,6 +190,7 @@ "required": ["kind", "target"], "properties": { "kind": { "const": "transfer" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "target": { "type": "string" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -191,6 +201,7 @@ "required": ["kind"], "properties": { "kind": { "const": "hangup" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } diff --git a/packages/flow-schema/src/generated/flow.v2.schema.json b/packages/flow-schema/src/generated/flow.v2.schema.json index 9ce22c1..b9101a3 100644 --- a/packages/flow-schema/src/generated/flow.v2.schema.json +++ b/packages/flow-schema/src/generated/flow.v2.schema.json @@ -25,12 +25,16 @@ "minimum": 0, "default": 1 }, + "note": { + "description": "The author's note about the whole flow: free text for people reading it, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "entry": { "description": "Node id where execution begins.", "type": "string" }, "nodes": { - "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution.", + "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution, but it is the author's reading order and is kept on save.", "type": "object", "additionalProperties": { "$ref": "#/$defs/Node" } }, @@ -121,6 +125,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "greeting" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -131,6 +136,7 @@ "required": ["kind", "schedule", "timezone"], "properties": { "kind": { "const": "hours" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "schedule": { "$ref": "#/$defs/WeeklySchedule" }, "timezone": { "description": "IANA zone (America/New_York). Validated to resolve at load.", "type": "string" }, "exceptions": { "type": "array", "items": { "$ref": "#/$defs/HoursException" } }, @@ -143,6 +149,7 @@ "required": ["kind", "prompt", "options"], "properties": { "kind": { "const": "menu" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "options": { "description": "Digit key → human label. The exit for a digit is exits[digit]. Valid digit keys are enforced by the validator.", @@ -160,6 +167,7 @@ "required": ["kind", "timeout_secs"], "properties": { "kind": { "const": "ring" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "timeout_secs": { "type": "integer", "minimum": 0 }, "exits": { "$ref": "#/$defs/Exits" } } @@ -170,6 +178,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "message" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "max_secs": { "type": "integer", "minimum": 0, "default": 120 }, "tone": { "allOf": [{ "$ref": "#/$defs/MessageTone" }], "default": "beep" }, @@ -182,6 +191,7 @@ "required": ["kind", "target"], "properties": { "kind": { "const": "transfer" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "target": { "type": "string" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -192,6 +202,7 @@ "required": ["kind"], "properties": { "kind": { "const": "hangup" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -202,6 +213,7 @@ "required": ["kind", "prompt", "confirm_prompt", "schedule", "timezone", "duration_mins"], "properties": { "kind": { "const": "book" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "confirm_prompt": { "$ref": "#/$defs/Prompt" }, "schedule": { "$ref": "#/$defs/WeeklySchedule" }, diff --git a/packages/flow-schema/src/generated/model.ts b/packages/flow-schema/src/generated/model.ts index 4782906..08d9756 100644 --- a/packages/flow-schema/src/generated/model.ts +++ b/packages/flow-schema/src/generated/model.ts @@ -35,12 +35,16 @@ export interface Flow { * Platform-assigned publish counter, bumped on publish. Defaults to 1 for a hand-written or generated pre-publish document. Distinct from schema_version. */ version?: number; + /** + * The author's note about the whole flow: free text for people reading it, in place of a YAML comment. Never read by the engine. + */ + note?: string; /** * Node id where execution begins. */ entry: string; /** - * The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution. + * The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution, but it is the author's reading order and is kept on save. */ nodes: { [k: string]: Node; @@ -58,6 +62,10 @@ export interface Flow { */ export interface GreetingNode { kind: 'greeting'; + /** + * The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine. + */ + note?: string; prompt: Prompt; exits?: Exits; [k: string]: unknown; @@ -84,6 +92,10 @@ export interface Exits { */ export interface HoursNode { kind: 'hours'; + /** + * The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine. + */ + note?: string; schedule: WeeklySchedule; /** * IANA zone (America/New_York). Validated to resolve at load. @@ -134,6 +146,10 @@ export interface HoursException { */ export interface MenuNode { kind: 'menu'; + /** + * The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine. + */ + note?: string; prompt: Prompt; /** * Digit key → human label. The exit for a digit is exits[digit]. Valid digit keys are enforced by the validator. @@ -151,6 +167,10 @@ export interface MenuNode { */ export interface RingNode { kind: 'ring'; + /** + * The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine. + */ + note?: string; timeout_secs: number; exits?: Exits; [k: string]: unknown; @@ -160,6 +180,10 @@ export interface RingNode { */ export interface MessageNode { kind: 'message'; + /** + * The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine. + */ + note?: string; prompt: Prompt; max_secs?: number; tone?: MessageTone & string; @@ -171,6 +195,10 @@ export interface MessageNode { */ export interface TransferNode { kind: 'transfer'; + /** + * The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine. + */ + note?: string; target: string; exits?: Exits; [k: string]: unknown; @@ -180,6 +208,10 @@ export interface TransferNode { */ export interface HangupNode { kind: 'hangup'; + /** + * The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine. + */ + note?: string; prompt?: Prompt; exits?: Exits; [k: string]: unknown; @@ -189,6 +221,10 @@ export interface HangupNode { */ export interface BookNode { kind: 'book'; + /** + * The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine. + */ + note?: string; prompt: Prompt; confirm_prompt: Prompt; schedule: WeeklySchedule; diff --git a/packages/flow-schema/src/generated/schema.ts b/packages/flow-schema/src/generated/schema.ts index ad96e61..b64be65 100644 --- a/packages/flow-schema/src/generated/schema.ts +++ b/packages/flow-schema/src/generated/schema.ts @@ -34,12 +34,16 @@ export const flowV1Schema = { "minimum": 0, "default": 1 }, + "note": { + "description": "The author's note about the whole flow: free text for people reading it, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "entry": { "description": "Node id where execution begins.", "type": "string" }, "nodes": { - "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution.", + "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution, but it is the author's reading order and is kept on save.", "type": "object", "additionalProperties": { "$ref": "#/$defs/Node" @@ -216,6 +220,10 @@ export const flowV1Schema = { "kind": { "const": "greeting" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "prompt": { "$ref": "#/$defs/Prompt" }, @@ -236,6 +244,10 @@ export const flowV1Schema = { "kind": { "const": "hours" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "schedule": { "$ref": "#/$defs/WeeklySchedule" }, @@ -266,6 +278,10 @@ export const flowV1Schema = { "kind": { "const": "menu" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "prompt": { "$ref": "#/$defs/Prompt" }, @@ -302,6 +318,10 @@ export const flowV1Schema = { "kind": { "const": "ring" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "timeout_secs": { "type": "integer", "minimum": 0 @@ -322,6 +342,10 @@ export const flowV1Schema = { "kind": { "const": "message" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "prompt": { "$ref": "#/$defs/Prompt" }, @@ -354,6 +378,10 @@ export const flowV1Schema = { "kind": { "const": "transfer" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "target": { "type": "string" }, @@ -372,6 +400,10 @@ export const flowV1Schema = { "kind": { "const": "hangup" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "prompt": { "$ref": "#/$defs/Prompt" }, @@ -416,12 +448,16 @@ export const flowV2Schema = { "minimum": 0, "default": 1 }, + "note": { + "description": "The author's note about the whole flow: free text for people reading it, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "entry": { "description": "Node id where execution begins.", "type": "string" }, "nodes": { - "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution.", + "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution, but it is the author's reading order and is kept on save.", "type": "object", "additionalProperties": { "$ref": "#/$defs/Node" @@ -601,6 +637,10 @@ export const flowV2Schema = { "kind": { "const": "greeting" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "prompt": { "$ref": "#/$defs/Prompt" }, @@ -621,6 +661,10 @@ export const flowV2Schema = { "kind": { "const": "hours" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "schedule": { "$ref": "#/$defs/WeeklySchedule" }, @@ -651,6 +695,10 @@ export const flowV2Schema = { "kind": { "const": "menu" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "prompt": { "$ref": "#/$defs/Prompt" }, @@ -687,6 +735,10 @@ export const flowV2Schema = { "kind": { "const": "ring" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "timeout_secs": { "type": "integer", "minimum": 0 @@ -707,6 +759,10 @@ export const flowV2Schema = { "kind": { "const": "message" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "prompt": { "$ref": "#/$defs/Prompt" }, @@ -739,6 +795,10 @@ export const flowV2Schema = { "kind": { "const": "transfer" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "target": { "type": "string" }, @@ -757,6 +817,10 @@ export const flowV2Schema = { "kind": { "const": "hangup" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "prompt": { "$ref": "#/$defs/Prompt" }, @@ -780,6 +844,10 @@ export const flowV2Schema = { "kind": { "const": "book" }, + "note": { + "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "prompt": { "$ref": "#/$defs/Prompt" }, diff --git a/packages/flow-schema/src/index.ts b/packages/flow-schema/src/index.ts index 9d7c522..9d1e3ff 100644 --- a/packages/flow-schema/src/index.ts +++ b/packages/flow-schema/src/index.ts @@ -4,8 +4,9 @@ // hand-written authoring logic consolidated here in Phase 2: safe-subset // YAML parsing/decoding, semantic validation (reachability, exit wiring, // hours math, DTMF/prompt rules), the one-call `checkFlow` gate, and -// comment-preserving source edits (`mutate.ts`, TS-only — the daemon never -// edits). The model *helpers* (`requiredExits`, `isTerminal`, …) are +// structured edits (`mutate.ts`: parse → change the model → the canonical +// text from `serializeFlow`, identical to the Rust crate's `Flow::to_yaml`). +// The model *helpers* (`requiredExits`, `isTerminal`, …) are // hand-written twins layered on the generated types — logic, not shape // (see `src/model.ts`). Import paths are kept identical to the platform's // in-repo copy so the Phase 3 swap is a dependency change, not a code @@ -37,13 +38,15 @@ export { diffFlows } from './diff.js'; export type { Issue } from './issues.js'; export { MISSING_ASSET, hasErrors } from './issues.js'; export type { ParseResult } from './parse.js'; -export { parseFlow } from './parse.js'; +export type { Comment } from './parse.js'; +export { findComments, parseFlow } from './parse.js'; +export { serializeFlow } from './serialize.js'; export { OUTBOUND_KINDS, validateFlow, validateOutbound } from './validate.js'; export type { FlowCheck } from './check.js'; export { checkFlow } from './check.js'; export { isValidDate, isValidTimezone, parseTimeMinutes, validateHours } from './hours.js'; -// Comment-preserving source edits (TS-only). +// Structured edits: parse → change the model → serialize (canonical text). export type { EditError, EditResult } from './mutate.js'; export { NODE_ID_PATTERN, diff --git a/packages/flow-schema/src/mutate.ts b/packages/flow-schema/src/mutate.ts index 7b3b591..90c388f 100644 --- a/packages/flow-schema/src/mutate.ts +++ b/packages/flow-schema/src/mutate.ts @@ -1,22 +1,43 @@ -// Structured, comment-preserving edits to a flow document's *source -// text*. The YAML text is the single source of truth (doc 48: "one -// document, many faces") — the form face and the API don't keep a -// parallel model, they rewrite the text through these ops, which go -// through the yaml Document API so comments, key order, and the -// engine-opaque `ui` block survive untouched. +// Structured edits to a flow document. The YAML text is the single source +// of truth, and everything in it that matters is in the model (doc 01: +// "the document is the whole truth"), so every op is // -// TS-only: the daemon never edits documents, so there is no Rust twin. -// These ops work on the `yaml` CST, not the generated model types. +// parseFlow → change the decoded flow → serializeFlow // -// Every op is stateless: source in, source out. Documents are a few KB -// at most, so re-parsing per edit is well inside interaction budgets. +// and the result is the canonical text, the same bytes the Rust crate's +// `Flow::to_yaml` writes. Comments carry no meaning and are not kept (the +// words belong in `note`); `parseFlow` reports them as `comment_ignored` +// so an editor can warn first. +// +// Every op is stateless: source in, source out. Documents are a few KB at +// most, so re-parsing per edit is well inside interaction budgets. -import { isMap, isScalar, parseDocument } from 'yaml'; -import type { Document, Pair, Scalar, YAMLMap, YAMLSeq } from 'yaml'; +import { parseDocument } from 'yaml'; -import type { ComponentKind } from './model.js'; +import type { ComponentKind, Flow, FlowNode } from './model.js'; +import { SUPPORTED_SCHEMA_VERSIONS } from './model.js'; +import { parseFlow } from './parse.js'; +import { serializeFlow } from './serialize.js'; -export type EditError = 'parse_failed' | 'invalid_node_id' | 'duplicate_node_id' | 'unknown_node'; +/** + * Why an edit was refused: + * - `parse_failed`: the text isn't YAML. + * - `invalid_document`: it's YAML, but doesn't decode as a flow (a menu + * with no `options`, say); fix the YAML first. Validation problems, like + * a missing exit, don't count: those documents decode and can be edited. + * - `unknown_field`: it has fields this version doesn't know, which a save + * would delete. + * - `unsupported_version`: it declares a `schema_version` this version + * can't write back. + */ +export type EditError = + | 'parse_failed' + | 'invalid_document' + | 'unknown_field' + | 'unsupported_version' + | 'invalid_node_id' + | 'duplicate_node_id' + | 'unknown_node'; export type EditResult = { ok: true; source: string } | { ok: false; error: EditError }; @@ -27,74 +48,88 @@ export type EditResult = { ok: true; source: string } | { ok: false; error: Edit */ export const NODE_ID_PATTERN = /^[A-Za-z0-9_][A-Za-z0-9_-]*$/; -function withDoc( - source: string, - edit: (doc: Document) => EditError | undefined, -): EditResult { - const doc = parseDocument(source); - if (doc.errors.length > 0) return { ok: false, error: 'parse_failed' }; - const error = edit(doc); - if (error) return { ok: false, error }; - return { ok: true, source: doc.toString() }; -} +type Json = { [key: string]: unknown }; -function nodesMap(doc: Document): YAMLMap | undefined { - const nodes = doc.get('nodes', true); - return isMap(nodes) ? nodes : undefined; +function isObject(v: unknown): v is Json { + return typeof v === 'object' && v !== null && !Array.isArray(v); } -function findPair(map: YAMLMap, key: string): Pair | undefined { - return (map.items as Pair[]).find( - (pair) => isScalar(pair.key) && pair.key.value === key, - ); +function withFlow(source: string, edit: (flow: Flow) => EditError | undefined): EditResult { + const parsed = parseFlow(source); + if (!parsed.flow) { + const yaml = parsed.issues.some( + (i) => i.severity === 'error' && (i.code === 'yaml_error' || i.code === 'yaml_unsupported'), + ); + return { ok: false, error: yaml ? 'parse_failed' : 'invalid_document' }; + } + // The version first: a newer document has fields this build can't know, + // and "unsupported version" is the better answer (as in Rust's `load`). + if (!SUPPORTED_SCHEMA_VERSIONS.includes(parsed.flow.schema_version)) { + return { ok: false, error: 'unsupported_version' }; + } + if (parsed.issues.some((i) => i.code === 'unknown_field')) { + return { ok: false, error: 'unknown_field' }; + } + const flow = structuredClone(parsed.flow); + const error = edit(flow); + if (error) return { ok: false, error }; + return { ok: true, source: serializeFlow(flow) }; } -/** Rename a key in place (preserves the pair's comments and position). */ -function renameKey(map: YAMLMap | undefined, from: string, to: string): void { - if (!map) return; - const pair = findPair(map, from); - if (pair) (pair.key as Scalar).value = to; +/** `nodes` with one key renamed in place, keeping every other key's order. */ +function renameKey(map: Record, from: string, to: string): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(map)) out[k === from ? to : k] = v; + return out; } /** The `ui` block spots that may be keyed by node id. Opaque to the * engine, but a rename/delete that leaves stale ids behind would leak * one node's position onto another once the id is reused. */ -function uiMaps(doc: Document): YAMLMap[] { - const ui = doc.get('ui', true); - if (!isMap(ui)) return []; +function uiMaps(flow: Flow): Json[] { + const ui = flow.ui as unknown; + if (!isObject(ui)) return []; const maps = [ui]; for (const key of ['nodes', 'positions']) { - const sub = ui.get(key, true); - if (isMap(sub)) maps.push(sub); + const sub = ui[key]; + if (isObject(sub)) maps.push(sub); } return maps; } +/** Replace the contents of `target` with `next`, in `next`'s order. */ +function replaceContents(target: Json, next: Json): void { + for (const k of Object.keys(target)) delete target[k]; + Object.assign(target, next); +} + +function exitsOf(node: FlowNode): Record { + if (!node.exits) node.exits = {}; + return node.exits as Record; +} + /** * Rename a node and every reference to it: the `nodes` key, `entry`, * every exit target, and `ui` position keys. */ export function renameNode(source: string, from: string, to: string): EditResult { if (!NODE_ID_PATTERN.test(to)) return { ok: false, error: 'invalid_node_id' }; - return withDoc(source, (doc) => { - const nodes = nodesMap(doc); - const pair = nodes && findPair(nodes, from); - if (!nodes || !pair) return 'unknown_node'; - if (from !== to && findPair(nodes, to)) return 'duplicate_node_id'; + return withFlow(source, (flow) => { + if (!Object.hasOwn(flow.nodes, from)) return 'unknown_node'; + if (from !== to && Object.hasOwn(flow.nodes, to)) return 'duplicate_node_id'; - (pair.key as Scalar).value = to; - if (doc.get('entry') === from) doc.set('entry', to); - for (const nodePair of nodes.items as Pair[]) { - if (!isMap(nodePair.value)) continue; - const exits = nodePair.value.get('exits', true); - if (!isMap(exits)) continue; - for (const exitPair of exits.items as Pair[]) { - if (isScalar(exitPair.value) && exitPair.value.value === from) { - exitPair.value.value = to; - } + flow.nodes = renameKey(flow.nodes, from, to); + if (flow.entry === from) flow.entry = to; + for (const node of Object.values(flow.nodes)) { + const exits = node.exits as Record | undefined; + if (!exits) continue; + for (const [name, target] of Object.entries(exits)) { + if (target === from) exits[name] = to; } } - for (const map of uiMaps(doc)) renameKey(map, from, to); + for (const map of uiMaps(flow)) { + if (Object.hasOwn(map, from)) replaceContents(map, renameKey(map, from, to)); + } return undefined; }); } @@ -106,21 +141,17 @@ export function renameNode(source: string, from: string, to: string): EditResult * `entry` is left for validation to flag. */ export function removeNode(source: string, id: string): EditResult { - return withDoc(source, (doc) => { - const nodes = nodesMap(doc); - if (!nodes || !findPair(nodes, id)) return 'unknown_node'; - nodes.delete(id); - for (const nodePair of nodes.items as Pair[]) { - if (!isMap(nodePair.value)) continue; - const exits = nodePair.value.get('exits', true); - if (!isMap(exits)) continue; - const stale = (exits.items as Pair[]) - .filter((exitPair) => isScalar(exitPair.value) && exitPair.value.value === id) - .map((exitPair) => (exitPair.key as Scalar).value); - for (const name of stale) exits.delete(name); - if (exits.items.length === 0) nodePair.value.delete('exits'); + return withFlow(source, (flow) => { + if (!Object.hasOwn(flow.nodes, id)) return 'unknown_node'; + delete flow.nodes[id]; + for (const node of Object.values(flow.nodes)) { + const exits = node.exits as Record | undefined; + if (!exits) continue; + for (const [name, target] of Object.entries(exits)) { + if (target === id) delete exits[name]; + } } - for (const map of uiMaps(doc)) map.delete(id); + for (const map of uiMaps(flow)) delete map[id]; return undefined; }); } @@ -128,54 +159,38 @@ export function removeNode(source: string, id: string): EditResult { /** Insert a new node built from a plain JS shape (see `defaultNode`). */ export function addNode(source: string, id: string, node: Record): EditResult { if (!NODE_ID_PATTERN.test(id)) return { ok: false, error: 'invalid_node_id' }; - return withDoc(source, (doc) => { - const nodes = nodesMap(doc); - if (nodes && findPair(nodes, id)) return 'duplicate_node_id'; - const created = doc.createNode(node); - // Inline the small maps the same way the documented examples write - // them: `exits: { next: bye }`, `prompt: { audio: welcome }`. - if (isMap(created)) { - for (const key of ['exits', 'options', 'prompt']) { - const sub = created.get(key, true); - if (isMap(sub)) sub.flow = true; - } - } - doc.setIn(['nodes', id], created); + return withFlow(source, (flow) => { + if (Object.hasOwn(flow.nodes, id)) return 'duplicate_node_id'; + flow.nodes[id] = structuredClone(node) as unknown as FlowNode; return undefined; }); } /** Point the flow's `entry` at an existing node. */ export function setEntry(source: string, id: string): EditResult { - return withDoc(source, (doc) => { - const nodes = nodesMap(doc); - if (!nodes || !findPair(nodes, id)) return 'unknown_node'; - doc.set('entry', id); + return withFlow(source, (flow) => { + if (!Object.hasOwn(flow.nodes, id)) return 'unknown_node'; + flow.entry = id; return undefined; }); } /** Wire (or rewire) one exit of a node. */ export function setExit(source: string, id: string, exit: string, target: string): EditResult { - return withDoc(source, (doc) => { - const nodes = nodesMap(doc); - if (!nodes || !findPair(nodes, id)) return 'unknown_node'; - doc.setIn(['nodes', id, 'exits', exit], target); + return withFlow(source, (flow) => { + const node = flow.nodes[id]; + if (!node) return 'unknown_node'; + exitsOf(node)[exit] = target; return undefined; }); } -/** Remove one exit of a node (dropping an empty `exits` map entirely). */ +/** Remove one exit of a node. */ export function removeExit(source: string, id: string, exit: string): EditResult { - return withDoc(source, (doc) => { - const nodes = nodesMap(doc); - const pair = nodes && findPair(nodes, id); - if (!nodes || !pair) return 'unknown_node'; - doc.deleteIn(['nodes', id, 'exits', exit]); - if (isMap(pair.value)) { - const exits = pair.value.get('exits', true); - if (isMap(exits) && exits.items.length === 0) pair.value.delete('exits'); - } + return withFlow(source, (flow) => { + const node = flow.nodes[id]; + if (!node) return 'unknown_node'; + delete exitsOf(node)[exit]; return undefined; }); } @@ -183,6 +198,8 @@ export function removeExit(source: string, id: string, exit: string): EditResult /** * Set (or with `value === undefined`, delete) one config field of a * node — `['prompt']`, `['options', '2']`, `['schedule', 'mon']`, … + * Missing containers on the way are created: a map for a string key, a + * list for a number. */ export function setNodeValue( source: string, @@ -190,15 +207,26 @@ export function setNodeValue( path: (string | number)[], value: unknown, ): EditResult { - return withDoc(source, (doc) => { - const nodes = nodesMap(doc); - if (!nodes || !findPair(nodes, id)) return 'unknown_node'; + return withFlow(source, (flow) => { + const node = flow.nodes[id]; + if (!node) return 'unknown_node'; + if (path.length === 0) return undefined; + let container: unknown = node; + for (let i = 0; i < path.length - 1; i += 1) { + const key = path[i] as string | number; + const holder = container as Record; + if (holder[key] === null || typeof holder[key] !== 'object') { + if (value === undefined) return undefined; + holder[key] = typeof path[i + 1] === 'number' ? [] : {}; + } + container = holder[key]; + } + const last = path[path.length - 1] as string | number; if (value === undefined) { - doc.deleteIn(['nodes', id, ...path]); + if (Array.isArray(container) && typeof last === 'number') container.splice(last, 1); + else delete (container as Record)[last]; } else { - const wrapped = - value !== null && typeof value === 'object' ? doc.createNode(value) : value; - doc.setIn(['nodes', id, ...path], wrapped); + (container as Record)[last] = structuredClone(value); } return undefined; }); @@ -212,22 +240,24 @@ export function setNodeValue( * keys in sync. */ export function setNodePosition(source: string, id: string, x: number, y: number): EditResult { - return withDoc(source, (doc) => { - const nodes = nodesMap(doc); - if (!nodes || !findPair(nodes, id)) return 'unknown_node'; - const position = doc.createNode([Math.round(x), Math.round(y)]) as YAMLSeq; - position.flow = true; - doc.setIn(['ui', 'positions', id], position); + return withFlow(source, (flow) => { + if (!Object.hasOwn(flow.nodes, id)) return 'unknown_node'; + const ui: Json = isObject(flow.ui) ? (flow.ui as Json) : {}; + const positions: Json = isObject(ui['positions']) ? (ui['positions'] as Json) : {}; + positions[id] = [Math.round(x), Math.round(y)]; + ui['positions'] = positions; + flow.ui = ui as Flow['ui']; return undefined; }); } /** Drop every stored position — the map falls back to auto layout. */ export function clearNodePositions(source: string): EditResult { - return withDoc(source, (doc) => { - doc.deleteIn(['ui', 'positions']); - const ui = doc.get('ui', true); - if (isMap(ui) && ui.items.length === 0) doc.delete('ui'); + return withFlow(source, (flow) => { + const ui = flow.ui as unknown; + if (!isObject(ui)) return undefined; + delete ui['positions']; + if (Object.keys(ui).length === 0) delete flow.ui; return undefined; }); } @@ -256,17 +286,23 @@ export function nodePositions(ui: unknown): Record { - if (identity.id !== undefined) doc.set('id', identity.id); - if (identity.version !== undefined) doc.set('version', identity.version); - if (identity.name !== undefined) doc.set('name', identity.name); - return undefined; - }); + const doc = parseDocument(source); + if (doc.errors.length > 0) return { ok: false, error: 'parse_failed' }; + if (!isObject(doc.toJS())) return { ok: false, error: 'invalid_document' }; + if (identity.id !== undefined) doc.set('id', identity.id); + if (identity.version !== undefined) doc.set('version', identity.version); + if (identity.name !== undefined) doc.set('name', identity.name); + return withFlow(doc.toString(), () => undefined); } /** @@ -283,8 +319,8 @@ export function stampIdentity( * document keeps working is untouched. */ export function setSchemaVersion(source: string, version: number): EditResult { - return withDoc(source, (doc) => { - doc.set('schema_version', version); + return withFlow(source, (flow) => { + flow.schema_version = version as Flow['schema_version']; return undefined; }); } diff --git a/packages/flow-schema/src/parse.ts b/packages/flow-schema/src/parse.ts index f5b501c..011c2aa 100644 --- a/packages/flow-schema/src/parse.ts +++ b/packages/flow-schema/src/parse.ts @@ -13,7 +13,7 @@ // at; warnings don't block publish, so both sides still accept the // same set of documents. -import { isMap, isScalar, isSeq, parseDocument, visit } from 'yaml'; +import { Lexer, isMap, isScalar, isSeq, parseDocument, visit } from 'yaml'; import type { Pair, Scalar, YAMLMap } from 'yaml'; import type { @@ -62,6 +62,47 @@ function rangeOf(node: unknown): [number, number] | undefined { return r ? [r[0], r[1]] : undefined; } +/** A YAML comment in a source. */ +export type Comment = { + /** 1-based line number. */ + line: number; + /** The words after `#`, trimmed. */ + text: string; + /** Source range of the whole comment, `#` included. */ + range: [number, number]; +}; + +/** The `yaml` lexer's structure markers: DOCUMENT, FLOW_END and SCALAR. */ +const LEXER_MARKERS = new Set(['\u0002', '\u0018', '\u001f']); + +/** + * Every comment in `source`, in order. Comments carry no meaning in a flow + * document (docs/01-canonical-document.md §2): the words belong in `note`, + * and saving drops them. Twin: `edit.rs` `comments`; the corpus under + * `conformance/canonical/` pins the count for both. + */ +export function findComments(source: string): Comment[] { + const out: Comment[] = []; + let offset = 0; + let line = 1; + for (const token of new Lexer().lex(source)) { + // The lexer also yields one-character control tokens (document start, + // scalar start, flow end) that mark structure and are not in the + // source, so they take no space in the offset. + if (LEXER_MARKERS.has(token) && !source.startsWith(token, offset)) continue; + if (token.startsWith('#')) { + out.push({ + line, + text: token.slice(1).trim(), + range: [offset, offset + token.length], + }); + } + offset += token.length; + for (const ch of token) if (ch === '\n') line += 1; + } + return out; +} + function error(issues: Issue[], code: string, message: string, at?: unknown): void { issues.push({ severity: 'error', code, message, range: rangeOf(at) }); } @@ -490,10 +531,13 @@ function decodeNode(id: string, value: unknown, issues: Issue[]): FlowNode | und warnUnknownFields( fields, - ['kind', 'exits', ...(NODE_CONFIG_FIELDS[kind] ?? [])], + ['kind', 'note', 'exits', ...(NODE_CONFIG_FIELDS[kind] ?? [])], context, issues, ); + const noteField = fields.get('note'); + const note = noteField ? asString(noteField.value, `${context} note`, issues) : undefined; + if (noteField && note === undefined) return undefined; const exits: Record = {}; const exitsField = fields.get('exits'); @@ -509,7 +553,8 @@ function decodeNode(id: string, value: unknown, issues: Issue[]): FlowNode | und } const component = decodeComponent(kind, id, fields, value, issues); - return component === undefined ? undefined : { ...component, exits }; + if (component === undefined) return undefined; + return note === undefined ? { ...component, exits } : { ...component, note, exits }; } /** @@ -557,6 +602,14 @@ export function parseFlow(source: string): ParseResult { Seq: (_key, node) => rejectUnsafe(node), }); if (hasErrors(issues)) return { flow: null, issues, nodeRanges }; + for (const comment of findComments(source)) { + issues.push({ + severity: 'warning', + code: 'comment_ignored', + message: 'comments are not part of the flow and are dropped on save; put the words in a `note`', + range: comment.range, + }); + } const root = doc.contents; if (!isMap(root)) { @@ -568,7 +621,7 @@ export function parseFlow(source: string): ParseResult { const fields = fieldsOf(root, 'flow document', issues); warnUnknownFields( fields, - ['schema_version', 'id', 'name', 'version', 'entry', 'nodes', 'ui'], + ['schema_version', 'id', 'name', 'version', 'note', 'entry', 'nodes', 'ui'], 'flow document', issues, ); @@ -582,6 +635,8 @@ export function parseFlow(source: string): ParseResult { const name = nameField && asString(nameField.value, 'name', issues); const versionField = fields.get('version'); const version = versionField ? asU32(versionField.value, 'version', issues) : 1; + const noteField = fields.get('note'); + const note = noteField ? asString(noteField.value, 'note', issues) : undefined; const entryField = required(fields, 'entry', 'flow document', issues, root); const entry = entryField && asString(entryField.value, 'entry', issues); @@ -629,6 +684,7 @@ export function parseFlow(source: string): ParseResult { id, name, version, + ...(note === undefined ? {} : { note }), entry, nodes, ui: ui as Flow['ui'], diff --git a/packages/flow-schema/src/serialize.ts b/packages/flow-schema/src/serialize.ts new file mode 100644 index 0000000..bd23955 --- /dev/null +++ b/packages/flow-schema/src/serialize.ts @@ -0,0 +1,310 @@ +// The canonical text of a flow (docs/01-canonical-document.md §4). +// +// One document has one text, the same in every language: the Rust twin is +// `crates/wavekat-flow/src/canonical.rs`, and the corpus under +// `conformance/canonical/` pins both byte for byte. The emitter is written +// by hand, because YAML libraries choose layout and quoting their own way, +// and it is guided by the bundled schema: property order, which properties +// are required, and their defaults all come from there, so a schema change +// needs no change here. + +import { flowV2Schema } from './generated/schema.js'; +import type { Flow } from './model.js'; + +type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; +type Schema = { [key: string]: unknown }; + +const ROOT = flowV2Schema as unknown as Schema; + +/** Follow `$ref` (and a single-entry `allOf`, the schema's way of putting + * a default next to a ref) to the schema that describes a value. */ +function resolve(s: Schema): Schema { + for (;;) { + const ref = s['$ref']; + if (typeof ref === 'string') { + const name = ref.replace(/^#\/\$defs\//, ''); + s = (ROOT['$defs'] as Record)[name] as Schema; + continue; + } + const allOf = s['allOf']; + if (Array.isArray(allOf) && allOf.length === 1) { + s = allOf[0] as Schema; + continue; + } + return s; + } +} + +function isObject(v: unknown): v is { [key: string]: Json } { + return typeof v === 'object' && v !== null && !Array.isArray(v); +} + +/** The schema for `value`, picking a `oneOf` branch: by `kind` for a node, + * by JSON type otherwise (a text or audio prompt). */ +export function describe(s: Schema | undefined, value: unknown): Schema | undefined { + if (!s) return undefined; + const resolved = resolve(s); + const branches = resolved['oneOf']; + if (!Array.isArray(branches)) return resolved; + return (branches as Schema[]).map(resolve).find((b) => { + const kind = (b['properties'] as Record | undefined)?.['kind']?.['const']; + if (kind !== undefined) return isObject(value) && value['kind'] === kind; + switch (b['type']) { + case 'string': + return typeof value === 'string'; + case 'object': + return isObject(value); + case 'array': + return Array.isArray(value); + case 'integer': + case 'number': + return typeof value === 'number'; + case 'boolean': + return typeof value === 'boolean'; + default: + return false; + } + }); +} + +/** The root schema, for walkers outside this module. */ +export function flowSchemaRoot(): Schema { + return ROOT; +} + +/** The canonical text of `flow`. */ +export function serializeFlow(flow: Flow): string { + const value = JSON.parse(JSON.stringify(flow)) as Json; + const out: string[] = []; + // The document, `nodes` and each node are block mappings, whatever they + // hold. + writeBlockMap(out, entries(value, ROOT), 0, 'document', false); + return out.join(''); +} + +type Level = 'document' | 'nodes' | 'node' | 'other'; + +function child(level: Level, key: string): Level { + if (level === 'document' && key === 'nodes') return 'nodes'; + if (level === 'nodes') return 'node'; + return 'other'; +} + +type Entry = [key: string, value: Json, schema: Schema | undefined]; + +/** A mapping's entries in canonical order, with omitted ones left out. + * Object key order is already the canonical map order (§3): integer keys + * first, ascending, then the rest as written. */ +function entries(value: Json, s: Schema | undefined): Entry[] { + const map = value as { [key: string]: Json }; + const described = describe(s, value); + const props = described?.['properties'] as Record | undefined; + if (!props) { + const extra = described?.['additionalProperties']; + const items = isObject(extra) ? (extra as Schema) : undefined; + return Object.entries(map).map(([k, v]) => [k, v, items]); + } + const required = (described?.['required'] as string[] | undefined) ?? []; + const out: Entry[] = []; + for (const [key, prop] of Object.entries(props)) { + if (!Object.hasOwn(map, key)) continue; + const v = map[key] as Json; + if (!required.includes(key) && omitted(v, defaultOf(prop))) continue; + out.push([key, v, prop]); + } + // Keys the schema doesn't know follow, in map order. (A decoded flow + // never has any; this keeps the output defined for any input.) + for (const [k, v] of Object.entries(map)) { + if (!Object.hasOwn(props, k)) out.push([k, v, undefined]); + } + return out; +} + +/** A property's `default`, looking through a single-entry `allOf`. */ +function defaultOf(prop: Schema): unknown { + if ('default' in prop) return prop['default']; + const allOf = prop['allOf']; + if (Array.isArray(allOf) && allOf.length > 0) return (allOf[0] as Schema)['default']; + return undefined; +} + +/** Is an optional property left out: absent, its default, or empty? */ +function omitted(value: Json, fallback: unknown): boolean { + if (value === null) return true; + if (Array.isArray(value)) return value.length === 0; + if (isObject(value)) return Object.keys(value).length === 0; + return fallback !== undefined && value === fallback; +} + +function isScalar(v: Json): boolean { + return v === null || typeof v !== 'object'; +} + +/** Is a collection written in flow style? When every entry it writes + * (after omissions) is a scalar. */ +function isFlow(v: Json, s: Schema | undefined): boolean { + if (Array.isArray(v)) return v.every(isScalar); + if (isObject(v)) return entries(v, s).every(([, x]) => isScalar(x)); + return false; +} + +function isEmpty(v: Json): boolean { + if (Array.isArray(v)) return v.length === 0; + if (isObject(v)) return Object.keys(v).length === 0; + return false; +} + +/** Write a block mapping at `indent`. When `onDashLine`, the first entry + * continues the current `- ` line of a list item. */ +function writeBlockMap( + out: string[], + list: Entry[], + indent: number, + level: Level, + onDashLine: boolean, +): void { + list.forEach(([key, value, s], i) => { + if (i > 0 || !onDashLine) out.push(' '.repeat(indent)); + out.push(scalarString(key, false), ':'); + writeValue(out, value, s, indent, child(level, key)); + }); +} + +/** Write `value` after `key:`, then end the line. */ +function writeValue( + out: string[], + value: Json, + s: Schema | undefined, + indent: number, + level: Level, +): void { + const alwaysBlock = level === 'nodes' || level === 'node'; + if (isObject(value) && isEmpty(value)) { + out.push(' {}\n'); + } else if (Array.isArray(value) && value.length === 0) { + out.push(' []\n'); + } else if (isObject(value) && (alwaysBlock || !isFlow(value, s))) { + out.push('\n'); + writeBlockMap(out, entries(value, s), indent + 2, level, false); + } else if (Array.isArray(value) && !isFlow(value, s)) { + out.push('\n'); + writeBlockList(out, value, itemsOf(s), indent + 2); + } else { + out.push(' '); + writeInline(out, value, s); + out.push('\n'); + } +} + +/** The schema of a list's items. */ +function itemsOf(s: Schema | undefined): Schema | undefined { + return s ? (resolve(s)['items'] as Schema | undefined) : undefined; +} + +function writeBlockList(out: string[], items: Json[], s: Schema | undefined, indent: number): void { + for (const item of items) { + out.push(' '.repeat(indent)); + if (isObject(item) && !isEmpty(item) && !isFlow(item, s)) { + out.push('- '); + writeBlockMap(out, entries(item, s), indent + 2, 'other', true); + } else if (Array.isArray(item) && item.length > 0 && !isFlow(item, s)) { + out.push('-\n'); + writeBlockList(out, item, itemsOf(s), indent + 2); + } else { + out.push('- '); + writeInline(out, item, s); + out.push('\n'); + } + } +} + +/** A scalar, or a flow collection of scalars, on one line. */ +function writeInline(out: string[], value: Json, s: Schema | undefined): void { + if (isObject(value)) { + if (isEmpty(value)) { + out.push('{}'); + return; + } + out.push('{ '); + entries(value, s).forEach(([k, v], i) => { + if (i > 0) out.push(', '); + out.push(scalarString(k, true), ': ', scalarText(v, true)); + }); + out.push(' }'); + } else if (Array.isArray(value)) { + out.push('[', value.map((v) => scalarText(v, true)).join(', '), ']'); + } else { + out.push(scalarText(value, false)); + } +} + +function scalarText(value: Json, flow: boolean): string { + if (value === null) return 'null'; + if (typeof value === 'boolean') return value ? 'true' : 'false'; + if (typeof value === 'number') return String(value); + if (typeof value === 'string') return scalarString(value, flow); + throw new Error('only scalars are written inline'); +} + +/** Words some YAML readers take for booleans, null or numbers. */ +const RESERVED = new Set([ + 'true', 'false', 'yes', 'no', 'on', 'off', 'y', 'n', 'null', 'inf', 'nan', 'infinity', +]); + +/** Code points a double-quoted string writes as `\uXXXX`. Lone surrogates + * (possible only in JavaScript strings) are escaped too. */ +function needsUnicodeEscape(cp: number): boolean { + return ( + cp <= 0x1f || + (cp >= 0x7f && cp <= 0x9f) || + cp === 0x2028 || + cp === 0x2029 || + cp === 0xfeff || + (cp >= 0xd800 && cp <= 0xdfff) + ); +} + +/** Can `s` be written as a plain scalar (§4)? */ +export function isPlain(s: string, flow: boolean): boolean { + const first = s.codePointAt(0); + if (first === undefined) return false; + const firstOk = + (first >= 0x41 && first <= 0x5a) || + (first >= 0x61 && first <= 0x7a) || + first === 0x5f || // _ + first === 0x2f || // / + first > 0x7f; + if (!firstOk) return false; + if (s.endsWith(' ') || s.endsWith(':')) return false; + if (s.includes(': ') || s.includes(' #')) return false; + for (const ch of s) { + const cp = ch.codePointAt(0) as number; + if (cp === 0x09 || needsUnicodeEscape(cp)) return false; + } + if (flow && /[,[\]{}:]/.test(s)) return false; + return !RESERVED.has(s.replace(/[A-Z]/g, (c) => c.toLowerCase())); +} + +const ESCAPES: Record = { + '"': '\\"', + '\\': '\\\\', + '\n': '\\n', + '\t': '\\t', + '\r': '\\r', + '\b': '\\b', + '\f': '\\f', +}; + +/** `s` as a plain scalar when it can be one, double-quoted otherwise. */ +export function scalarString(s: string, flow: boolean): string { + if (isPlain(s, flow)) return s; + let out = '"'; + for (const ch of s) { + const cp = ch.codePointAt(0) as number; + const named = ESCAPES[ch]; + if (named !== undefined) out += named; + else if (needsUnicodeEscape(cp)) out += `\\u${cp.toString(16).padStart(4, '0')}`; + else out += ch; + } + return out + '"'; +} diff --git a/packages/flow-schema/test/canonical.test.ts b/packages/flow-schema/test/canonical.test.ts new file mode 100644 index 0000000..7af9007 --- /dev/null +++ b/packages/flow-schema/test/canonical.test.ts @@ -0,0 +1,69 @@ +// The TypeScript half of the canonical-text contract +// (docs/01-canonical-document.md §4). The Rust crate runs the same cases +// (crates/wavekat-flow/tests/canonical.rs), so the two serializers agree +// byte for byte. + +import { readFileSync, readdirSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, it } from 'vitest'; + +import { findComments, parseFlow } from '../src/parse.js'; +import { serializeFlow } from '../src/serialize.js'; + +const corpus = fileURLToPath(new URL('../../../conformance', import.meta.url)); +const canonicalDir = join(corpus, 'canonical'); + +const cases = readdirSync(canonicalDir) + .filter((f) => f.endsWith('.input.yaml')) + .map((f) => f.slice(0, -'.input.yaml'.length)) + .sort(); + +function decode(source: string, label: string) { + const { flow, issues } = parseFlow(source); + if (!flow) throw new Error(`${label}: ${JSON.stringify(issues)}`); + return flow; +} + +describe('canonical corpus', () => { + it('has cases', () => expect(cases.length).toBeGreaterThan(0)); + + for (const name of cases) { + const read = (ext: string) => readFileSync(join(canonicalDir, `${name}.${ext}`), 'utf8'); + it(name, () => { + const input = read('input.yaml'); + const canonical = read('canonical.yaml'); + const expected = JSON.parse(read('expected.json')) as { comments: number }; + + expect(serializeFlow(decode(input, name))).toBe(canonical); + expect(serializeFlow(decode(canonical, name))).toBe(canonical); + expect(findComments(input)).toHaveLength(expected.comments); + expect(findComments(canonical)).toHaveLength(0); + }); + } +}); + +function validDocuments(): string[] { + const out: string[] = []; + for (const version of readdirSync(corpus)) { + for (const bucket of ['valid', 'outbound/valid']) { + const dir = join(corpus, version, bucket); + if (!existsSync(dir)) continue; + for (const f of readdirSync(dir)) if (f.endsWith('.yaml')) out.push(join(dir, f)); + } + } + return out.sort(); +} + +describe('serializing loses nothing', () => { + for (const path of validDocuments()) { + it(path.slice(corpus.length + 1), () => { + const flow = decode(readFileSync(path, 'utf8'), path); + const text = serializeFlow(flow); + const back = decode(text, path); + expect(back).toEqual(flow); + expect(serializeFlow(back)).toBe(text); + }); + } +}); diff --git a/packages/flow-schema/test/mutate.test.ts b/packages/flow-schema/test/mutate.test.ts index e32905f..d5e565c 100644 --- a/packages/flow-schema/test/mutate.test.ts +++ b/packages/flow-schema/test/mutate.test.ts @@ -1,8 +1,9 @@ -// The structured edit ops must preserve what the engine ignores but -// humans rely on — comments, key order, the `ui` block — while keeping -// every reference (entry, exits, ui positions) consistent. A rename -// that misses one exit target silently reroutes a caller; these tests -// are the fence. +// The structured edit ops go through the model (parse → change → +// serialize) and keep every reference (entry, exits, ui positions) +// consistent. A rename that misses one exit target silently reroutes a +// caller; these tests are the fence. What the engine ignores but people +// rely on is data — `note`, node order, the `ui` block — and survives; +// YAML comments aren't, and don't (docs/01-canonical-document.md). // // Migrated from the platform repo's packages/flow-schema/src/mutate.test.ts. @@ -25,6 +26,8 @@ import { stampIdentity, } from '../src/mutate.js'; import { COMPONENT_KINDS } from '../src/model.js'; +import { serializeFlow } from '../src/serialize.js'; +import { parseFlow } from '../src/parse.js'; const SOURCE = `schema_version: 1 id: flow_1 @@ -65,9 +68,14 @@ describe('renameNode', () => { expect(checkFlow(source).valid).toBe(true); }); - it('preserves comments', () => { - const source = edited(renameNode(SOURCE, 'welcome', 'greet')); - expect(source).toContain('swap this for your own greeting'); + it('writes the canonical text: comments go, notes and node order stay', () => { + const noted = SOURCE.replace(' kind: greeting\n', ' kind: greeting\n note: Swap this for your own greeting.\n'); + const source = edited(renameNode(noted, 'welcome', 'greet')); + expect(source).not.toContain('#'); + expect(source).toContain('note: Swap this for your own greeting.'); + expect(Object.keys(checkFlow(source).flow?.nodes ?? {})).toEqual(['greet', 'menu', 'bye']); + const flow = parseFlow(source).flow; + expect(flow && serializeFlow(flow)).toBe(source); }); it('refuses a duplicate or invalid id', () => { @@ -91,7 +99,9 @@ describe('removeNode', () => { // The menu's "1" exit pointed at welcome — gone, so validation now // reports the menu's missing exit instead of a dangling target. expect(flow?.nodes['menu']?.exits?.['1']).toBeUndefined(); - expect(flow?.ui).toEqual({}); + // The ui block held only welcome's position; an empty optional map is + // left out of the canonical text. + expect(flow?.ui).toBeUndefined(); const issueCodes = checkFlow(source).issues.map((issue) => issue.code); expect(issueCodes).toContain('missing_exits'); expect(issueCodes).toContain('missing_entry'); @@ -173,10 +183,9 @@ describe('setNodeValue', () => { }); describe('node positions', () => { - it('stores a rounded [x, y] pair under ui.positions, preserving comments', () => { + it('stores a rounded [x, y] pair under ui.positions', () => { const source = edited(setNodePosition(SOURCE, 'menu', 220.6, 96.2)); - expect(source).toContain('menu: [ 221, 96 ]'); - expect(source).toContain('swap this for your own greeting'); + expect(source).toContain(' menu: [221, 96]\n'); const { flow } = checkFlow(source); expect(nodePositions(flow?.ui)).toEqual({ menu: { x: 221, y: 96 } }); expect(checkFlow(source).valid).toBe(true); @@ -231,7 +240,16 @@ describe('stampIdentity', () => { const { flow } = checkFlow(source); expect(flow?.id).toBe('flow_abc123'); expect(flow?.version).toBe(4); - expect(source).toContain('swap this for your own greeting'); + }); + + it('completes a new draft that has no id or name yet', () => { + const draft = SOURCE.replace('id: flow_1\nname: Test\n', ''); + expect(setEntry(draft, 'menu')).toEqual({ ok: false, error: 'invalid_document' }); + const source = edited(stampIdentity(draft, { id: 'flow_new', name: 'New flow' })); + const { flow } = checkFlow(source); + expect(flow?.id).toBe('flow_new'); + expect(flow?.name).toBe('New flow'); + expect(source.startsWith('schema_version: 1\nid: flow_new\nname: New flow\n')).toBe(true); }); it('fails cleanly on unparseable text', () => { @@ -245,7 +263,6 @@ describe('setSchemaVersion', () => { const { flow } = checkFlow(source); expect(flow?.schema_version).toBe(2); expect(flow?.entry).toBe('welcome'); - expect(source).toContain('swap this for your own greeting'); expect(source).toContain('ui:'); }); @@ -260,3 +277,37 @@ describe('setSchemaVersion', () => { expect(setSchemaVersion('nodes: [', 2)).toEqual({ ok: false, error: 'parse_failed' }); }); }); + +describe('what an edit refuses', () => { + it('refuses a document with fields a save would delete', () => { + const extra = SOURCE.replace(' kind: hangup\n', ' kind: hangup\n colour: blue\n'); + expect(setEntry(extra, 'menu')).toEqual({ ok: false, error: 'unknown_field' }); + }); + + it('refuses a document that does not decode, and says so apart from bad YAML', () => { + const broken = SOURCE.replace(' options: { "1": Loop }\n', ''); + expect(setEntry(broken, 'menu')).toEqual({ ok: false, error: 'invalid_document' }); + expect(setEntry('nodes: [', 'menu')).toEqual({ ok: false, error: 'parse_failed' }); + }); + + it('refuses a version it cannot write back', () => { + const future = SOURCE.replace('schema_version: 1', 'schema_version: 99'); + expect(setEntry(future, 'menu')).toEqual({ ok: false, error: 'unsupported_version' }); + }); + + it('names an unsupported version before its unknown fields', () => { + // A newer document usually has fields this build doesn't know. + const future = SOURCE.replace('schema_version: 1', 'schema_version: 99').replace( + ' kind: hangup\n', + ' kind: hangup\n colour: blue\n', + ); + expect(setEntry(future, 'menu')).toEqual({ ok: false, error: 'unsupported_version' }); + }); + + it('still edits a document that only fails validation', () => { + // A missing exit is a validation error, not a decode error. + const unwired = SOURCE.replace(' exits: { next: menu }\n', ''); + expect(checkFlow(unwired).valid).toBe(false); + expect(setEntry(unwired, 'menu').ok).toBe(true); + }); +}); diff --git a/packages/flow-schema/test/parse.test.ts b/packages/flow-schema/test/parse.test.ts index 6cea7e9..a63e71b 100644 --- a/packages/flow-schema/test/parse.test.ts +++ b/packages/flow-schema/test/parse.test.ts @@ -7,7 +7,7 @@ import { describe, expect, it } from 'vitest'; -import { parseFlow } from '../src/parse.js'; +import { findComments, parseFlow } from '../src/parse.js'; import { promptTranscript, requiredAssets } from '../src/model.js'; // The doc 48 "flow document, sketched" example — kept in sync with the @@ -294,3 +294,85 @@ nodes: expect(source.slice(range![0], range![0] + 'night_menu'.length)).toBe('night_menu'); }); }); + +describe('notes and comments', () => { + const SRC = `# about this flow +schema_version: 1 +id: flow_n +name: Notes +note: Reviewed in May. +entry: hi +nodes: + hi: + kind: hangup + note: Owner recorded this. # trailing + prompt: Bye. +`; + + it('decodes note on the flow and on a node, with no unknown-field warning', () => { + const { flow, issues } = parseFlow(SRC); + expect(flow?.note).toBe('Reviewed in May.'); + expect(flow?.nodes['hi']?.note).toBe('Owner recorded this.'); + expect(issues.map((i) => i.code)).not.toContain('unknown_field'); + }); + + it('reports each comment as comment_ignored, with its range', () => { + const comments = parseFlow(SRC).issues.filter((i) => i.code === 'comment_ignored'); + expect(comments).toHaveLength(2); + expect(comments.every((c) => c.severity === 'warning')).toBe(true); + const [first] = comments; + expect(first && SRC.slice(first.range?.[0], first.range?.[1])).toBe('# about this flow'); + }); + + it('comment ranges are source offsets, however much structure comes before', () => { + // The yaml lexer yields control tokens that are not source text; a range + // must not count them (it used to drift by one per scalar). + const sources = [ + SRC, + 'a: 1 # one\nb: "x # y" # two\nc: |\n # in\n t\n# three\nd: [1, # four\n 2]\n', + '\ufeff--- # doc\na: 1\n... # end\n', + 'a: 1\r\n# crlf\r\nb: 再 # 见\r\n', + 'k: v\n'.repeat(50) + '# late\n', + ]; + for (const source of sources) { + const found = findComments(source); + expect(found.length).toBeGreaterThan(0); + for (const c of found) { + const text = source.slice(c.range[0], c.range[1]); + expect(text.startsWith('#')).toBe(true); + expect(text.slice(1).trim()).toBe(c.text); + expect(source.slice(0, c.range[0]).split('\n').length).toBe(c.line); + } + } + // The issue carries the same range. + const text = SRC.replace('# about this flow\n', ''); + const [trailing] = parseFlow(text).issues.filter((i) => i.code === 'comment_ignored'); + expect(trailing && text.slice(trailing.range?.[0], trailing.range?.[1])).toBe('# trailing'); + }); + + it('a note must be text', () => { + const { flow, issues } = parseFlow(SRC.replace('note: Reviewed in May.', 'note: [x]')); + expect(flow).toBeNull(); + expect(issues.map((i) => i.code)).toContain('expected_string'); + }); +}); + +describe('known fields follow the schema', () => { + // The decoder lists each kind's fields by hand; the schema is the source + // of truth. A field added to the schema but not here would decode as + // unknown and block every edit. + it('every node kind knows exactly its schema properties', async () => { + const { flowV2Schema } = await import('../src/generated/schema.js'); + const defs = (flowV2Schema as unknown as { $defs: Record }> }).$defs; + for (const def of Object.values(defs)) { + const kind = def.properties?.['kind']?.const; + if (!kind) continue; + const src = `schema_version: 2\nid: f\nname: F\nentry: a\nnodes:\n a:\n${Object.keys(def.properties ?? {}) + .filter((p) => p !== 'kind') + .map((p) => ` ${p}: x\n`) + .join('')} kind: ${kind}\n`; + const unknown = parseFlow(src).issues.filter((i) => i.code === 'unknown_field'); + expect(unknown, kind).toEqual([]); + } + }); +}); diff --git a/schema/flow.v1.schema.json b/schema/flow.v1.schema.json index 2fe2e66..1a5c734 100644 --- a/schema/flow.v1.schema.json +++ b/schema/flow.v1.schema.json @@ -25,12 +25,16 @@ "minimum": 0, "default": 1 }, + "note": { + "description": "The author's note about the whole flow: free text for people reading it, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "entry": { "description": "Node id where execution begins.", "type": "string" }, "nodes": { - "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution.", + "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution, but it is the author's reading order and is kept on save.", "type": "object", "additionalProperties": { "$ref": "#/$defs/Node" } }, @@ -120,6 +124,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "greeting" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -130,6 +135,7 @@ "required": ["kind", "schedule", "timezone"], "properties": { "kind": { "const": "hours" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "schedule": { "$ref": "#/$defs/WeeklySchedule" }, "timezone": { "description": "IANA zone (America/New_York). Validated to resolve at load.", "type": "string" }, "exceptions": { "type": "array", "items": { "$ref": "#/$defs/HoursException" } }, @@ -142,6 +148,7 @@ "required": ["kind", "prompt", "options"], "properties": { "kind": { "const": "menu" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "options": { "description": "Digit key → human label. The exit for a digit is exits[digit]. Valid digit keys are enforced by the validator.", @@ -159,6 +166,7 @@ "required": ["kind", "timeout_secs"], "properties": { "kind": { "const": "ring" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "timeout_secs": { "type": "integer", "minimum": 0 }, "exits": { "$ref": "#/$defs/Exits" } } @@ -169,6 +177,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "message" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "max_secs": { "type": "integer", "minimum": 0, "default": 120 }, "tone": { "allOf": [{ "$ref": "#/$defs/MessageTone" }], "default": "beep" }, @@ -181,6 +190,7 @@ "required": ["kind", "target"], "properties": { "kind": { "const": "transfer" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "target": { "type": "string" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -191,6 +201,7 @@ "required": ["kind"], "properties": { "kind": { "const": "hangup" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } diff --git a/schema/flow.v2.schema.json b/schema/flow.v2.schema.json index 9ce22c1..b9101a3 100644 --- a/schema/flow.v2.schema.json +++ b/schema/flow.v2.schema.json @@ -25,12 +25,16 @@ "minimum": 0, "default": 1 }, + "note": { + "description": "The author's note about the whole flow: free text for people reading it, in place of a YAML comment. Never read by the engine.", + "type": "string" + }, "entry": { "description": "Node id where execution begins.", "type": "string" }, "nodes": { - "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution.", + "description": "The flat node set, keyed by human-meaningful node id and string-referenced by exits. Order is irrelevant to execution, but it is the author's reading order and is kept on save.", "type": "object", "additionalProperties": { "$ref": "#/$defs/Node" } }, @@ -121,6 +125,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "greeting" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -131,6 +136,7 @@ "required": ["kind", "schedule", "timezone"], "properties": { "kind": { "const": "hours" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "schedule": { "$ref": "#/$defs/WeeklySchedule" }, "timezone": { "description": "IANA zone (America/New_York). Validated to resolve at load.", "type": "string" }, "exceptions": { "type": "array", "items": { "$ref": "#/$defs/HoursException" } }, @@ -143,6 +149,7 @@ "required": ["kind", "prompt", "options"], "properties": { "kind": { "const": "menu" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "options": { "description": "Digit key → human label. The exit for a digit is exits[digit]. Valid digit keys are enforced by the validator.", @@ -160,6 +167,7 @@ "required": ["kind", "timeout_secs"], "properties": { "kind": { "const": "ring" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "timeout_secs": { "type": "integer", "minimum": 0 }, "exits": { "$ref": "#/$defs/Exits" } } @@ -170,6 +178,7 @@ "required": ["kind", "prompt"], "properties": { "kind": { "const": "message" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "max_secs": { "type": "integer", "minimum": 0, "default": 120 }, "tone": { "allOf": [{ "$ref": "#/$defs/MessageTone" }], "default": "beep" }, @@ -182,6 +191,7 @@ "required": ["kind", "target"], "properties": { "kind": { "const": "transfer" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "target": { "type": "string" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -192,6 +202,7 @@ "required": ["kind"], "properties": { "kind": { "const": "hangup" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "exits": { "$ref": "#/$defs/Exits" } } @@ -202,6 +213,7 @@ "required": ["kind", "prompt", "confirm_prompt", "schedule", "timezone", "duration_mins"], "properties": { "kind": { "const": "book" }, + "note": { "description": "The author's note about this step: free text for people reading the flow, in place of a YAML comment. Never read by the engine.", "type": "string" }, "prompt": { "$ref": "#/$defs/Prompt" }, "confirm_prompt": { "$ref": "#/$defs/Prompt" }, "schedule": { "$ref": "#/$defs/WeeklySchedule" },