From 51bf4e1c6d0c7932414b48344ecb021afe92ed51 Mon Sep 17 00:00:00 2001 From: harehare Date: Mon, 19 Oct 2026 22:09:40 +0900 Subject: [PATCH] feat: let vars take their value from an mq query over a file vars = { VERSION = { mq = "nodes | filter(is_h2) | first() | to_text()", from = "CHANGELOG.md" } } runs the query the way the mq command does and uses the results, one per line. The input is Markdown, MDX, HTML or text by the file extension. ${NAME} in the query and the path refers to other vars, the result is shared within a run, a dry run shows the query, and a missing file or failing query names the variable. --- docs/books/src/guide/vars.md | 24 ++++ docs/books/src/reference/meta.md | 2 +- src/summary.rs | 1 + src/vars.rs | 218 +++++++++++++++++++++++++++++-- 4 files changed, 231 insertions(+), 14 deletions(-) diff --git a/docs/books/src/guide/vars.md b/docs/books/src/guide/vars.md index d602345..309c8bf 100644 --- a/docs/books/src/guide/vars.md +++ b/docs/books/src/guide/vars.md @@ -23,8 +23,32 @@ A `meta` block before the first heading defines variables for every task; a task | `NAME = "app"` | the string; numbers and booleans are turned into strings | | `VERSION = { sh = "git describe" }` | the output of the command, without trailing newlines | +| `VERSION = { mq = "...", from = "CHANGELOG.md" }` | the result of an [mq](https://mqlang.org) query over a file, one result per line | + Commands run with `sh -c` (`cmd /C` on Windows) in the task's working directory. If one fails, the task does not run and the error shows the command, its exit code and the first line of its error output. +## Values from Markdown, HTML or text files + +`{ mq = "query", from = "file" }` runs an mq query over a file and uses the result, so a variable can come from a document that already holds the information, without a `grep` or `sed`: + +````markdown +```meta +vars = { VERSION = { mq = "nodes | filter(is_h2) | first() | to_text()", from = "CHANGELOG.md" }, TAG = "v${VERSION}" } +``` + +## release + +```bash +git tag "$TAG" +``` +```` + +- The query is evaluated the way the `mq` command does it, so the same text works with `mq '' CHANGELOG.md`. A selector such as `.h2` runs on every node and yields the headings. To pick one result over the whole document, start from `nodes`: `nodes | filter(is_h2) | first()`. +- Results are printed one per line, and nodes a selector does not match are left out. A heading is turned into text with `to_text()`; without it you get the Markdown of the node. +- The kind of input follows the extension of `from`: `.html`/`.htm` as HTML, `.mdx` as MDX, `.txt` as plain text, anything else as Markdown. `from` is relative to the task's working directory. +- `${NAME}` in `from` refers to other variables, and so does the query (a name that is not a variable stays as it is, since mq has its own `${...}` in interpolated strings). +- Like `sh`, the query runs once per run, before the task starts, and `--dry-run` shows `` instead. A missing file or a query that fails is an error that names the variable. + ## Referring to other variables `${NAME}` inside a value, or inside an `sh` command, is replaced by that variable: diff --git a/docs/books/src/reference/meta.md b/docs/books/src/reference/meta.md index 6fbb490..9f118b0 100644 --- a/docs/books/src/reference/meta.md +++ b/docs/books/src/reference/meta.md @@ -28,7 +28,7 @@ Error: Invalid task file: task 'flaky': `meta` block: unknown key `retires` on l | `outputs` | array of paths | Files that must exist for the task to be considered up to date | | `params` | array of strings or tables | Named parameters: `"name"` (required), `"name=value"` (default) or `{ name, default, choices, desc }` ([guide](../guide/params-env.md#choices-and-descriptions)) | | `requires` | array of strings | Variables that must be set and not empty ([guide](../guide/params-env.md#required-variables)) | -| `vars` | table | Named values, fixed or `{ sh = "command" }`; `${NAME}` refers to another ([Variables](../guide/vars.md)) | +| `vars` | table | Named values: fixed, `{ sh = "command" }` or `{ mq = "query", from = "file" }`; `${NAME}` refers to another ([Variables](../guide/vars.md)) | | `env` | array of `KEY=VALUE` | Default environment variables | | `dir` | string | Working directory | | `dotenv` | array of paths | `.env` files to load ([guide](../guide/params-env.md#env-files)) | diff --git a/src/summary.rs b/src/summary.rs index 545da58..04f182e 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -43,6 +43,7 @@ pub fn render(section: &Section) -> String { .map(|v| match v.value { VarValue::Literal(_) => v.name.clone(), VarValue::Shell(_) => format!("{} (sh)", v.name), + VarValue::Mq { .. } => format!("{} (mq)", v.name), }) .collect(); rows.push(("vars", vars.join(", "))); diff --git a/src/vars.rs b/src/vars.rs index 7915b79..272155e 100644 --- a/src/vars.rs +++ b/src/vars.rs @@ -28,6 +28,18 @@ pub enum VarValue { Literal(String), /// The output of a shell command, with trailing newlines removed. Shell(String), + /// The result of an mq query over a file, one line per result. + Mq { query: String, from: String }, +} + +impl VarValue { + /// The text that may refer to other vars with `${NAME}`. + fn text(&self) -> String { + match self { + VarValue::Literal(s) | VarValue::Shell(s) => s.clone(), + VarValue::Mq { query, from } => format!("{}\n{}", query, from), + } + } } /// A variable declared in a `vars` table. @@ -80,15 +92,24 @@ pub fn parse_table(context: &str, table: &toml::Table) -> Result> { toml::Value::Integer(i) => VarValue::Literal(i.to_string()), toml::Value::Float(f) => VarValue::Literal(f.to_string()), toml::Value::Boolean(b) => VarValue::Literal(b.to_string()), - toml::Value::Table(t) => match (t.len(), t.get("sh")) { - (1, Some(toml::Value::String(cmd))) if !cmd.trim().is_empty() => { - VarValue::Shell(cmd.clone()) + toml::Value::Table(t) => { + let text = |key: &str| match t.get(key) { + Some(toml::Value::String(s)) if !s.trim().is_empty() => Some(s.clone()), + _ => None, + }; + match (t.len(), text("sh"), text("mq"), text("from")) { + (1, Some(cmd), _, _) => VarValue::Shell(cmd), + (2, _, Some(query), Some(from)) => VarValue::Mq { query, from }, + _ => { + return Err(invalid( + "must be `{ sh = \"command\" }` or `{ mq = \"query\", from = \"file\" }` when it is a table", + )); + } } - _ => return Err(invalid("must be `{ sh = \"command\" }` when it is a table")), - }, + } _ => { return Err(invalid( - "must be a string, number, boolean or `{ sh = \"command\" }`", + "must be a string, number, boolean, `{ sh = \"command\" }` or `{ mq = \"query\", from = \"file\" }`", )); } }; @@ -272,10 +293,8 @@ pub fn depending_on(defs: &[VarDef], roots: &[String]) -> Vec { if found.contains(&def.name) { continue; } - let text = match &def.value { - VarValue::Literal(s) | VarValue::Shell(s) => s, - }; - if references(text) + let text = def.value.text(); + if references(&text) .iter() .any(|r| *r != def.name && (roots.contains(r) || found.contains(r))) { @@ -288,6 +307,45 @@ pub fn depending_on(defs: &[VarDef], roots: &[String]) -> Vec { } } +/// Run an mq `query` over the file `from` (relative to `dir`) and join the +/// results, one per line. The kind of input follows the extension: +/// Markdown (also the fallback), MDX, HTML or plain text. +fn run_mq(name: &str, query: &str, from: &str, dir: Option<&Path>) -> Result { + let fail = |why: String| Error::Variable(name.to_string(), why); + let path = match dir { + Some(dir) if !Path::new(from).is_absolute() => dir.join(from), + _ => PathBuf::from(from), + }; + let content = std::fs::read_to_string(&path) + .map_err(|e| fail(format!("cannot read '{}': {}", path.display(), e)))?; + let extension = path + .extension() + .map(|e| e.to_string_lossy().to_lowercase()) + .unwrap_or_default(); + let input = match extension.as_str() { + "mdx" => mq_lang::parse_mdx_input(&content), + "html" | "htm" => mq_lang::parse_html_input(&content), + "txt" | "text" => mq_lang::parse_text_input(&content), + _ => mq_lang::parse_markdown_input(&content), + } + .map_err(|e| fail(format!("cannot read '{}': {}", path.display(), e)))?; + + let mut engine: mq_lang::Engine = mq_lang::Engine::default(); + engine.load_builtin_module(); + let results = engine + .eval(query, input.into_iter()) + .map_err(|e| fail(format!("mq query `{}` failed: {}", query, e)))?; + let lines: Vec = results + .into_iter() + .filter(|v| !v.is_none()) + .map(|v| v.to_string()) + // The nodes a selector does not match come back empty; the mq + // command line leaves them out too. + .filter(|line| !line.is_empty()) + .collect(); + Ok(lines.join("\n").trim_end_matches(['\n', '\r']).to_string()) +} + /// Wait for the entry of a command another task is running, giving up when /// the run is cancelled or the deadline passes. fn lock_entry<'a>( @@ -332,9 +390,7 @@ pub fn resolve(defs: &[VarDef], ctx: &ResolveContext) -> Result s.clone(), - }; + let text = |d: &VarDef| d.value.text(); let mut order: Vec = Vec::new(); let mut state = vec![0u8; by_name.len()]; // 0 new, 1 visiting, 2 done fn visit( @@ -434,6 +490,39 @@ pub fn resolve(defs: &[VarDef], ctx: &ResolveContext) -> Result { + // As in a command, a name that is not a var stays: mq has its + // own `${...}` in interpolated strings. + let query = interpolate(query, &lookup, false).unwrap_or_else(|_| query.clone()); + let from = interpolate(from, &lookup, true).map_err(|unknown| { + Error::Variable( + def.name.clone(), + format!("unknown variable `${{{}}}`", unknown), + ) + })?; + if ctx.dry_run { + format!("", from, query) + } else { + let key = ( + ctx.dir.map(Path::to_path_buf), + format!("mq {}\n{}", from, query), + Vec::new(), + ); + let entry = { + let mut cache = ctx.cache.lock().unwrap_or_else(|e| e.into_inner()); + cache.entry(key).or_default().clone() + }; + let mut slot = lock_entry(&entry, ctx.io)?; + match &*slot { + Some(hit) => hit.clone(), + None => { + let out = run_mq(&def.name, &query, &from, ctx.dir)?; + *slot = Some(out.clone()); + out + } + } + } + } }; resolved.push((def.name.clone(), value)); } @@ -778,6 +867,109 @@ mod tests { ); } + fn mq(name: &str, query: &str, from: &str) -> VarDef { + VarDef { + name: name.into(), + value: VarValue::Mq { + query: query.into(), + from: from.into(), + }, + } + } + + fn resolve_in( + defs: &[VarDef], + dir: Option<&Path>, + dry_run: bool, + ) -> Result> { + let cache = ShellCache::default(); + resolve( + defs, + &ResolveContext { + base: &[], + fixed: &[], + dir, + dry_run, + cache: &cache, + select: Select::All, + io: &io(&AtomicBool::new(false), None), + }, + ) + } + + #[test] + fn parses_mq_values() { + let table: toml::Table = + toml::from_str("V = { mq = \".h2\", from = \"CHANGELOG.md\" }\n").unwrap(); + let vars = parse_table("ctx", &table).unwrap(); + assert_eq!( + vars[0].value, + VarValue::Mq { + query: ".h2".into(), + from: "CHANGELOG.md".into() + } + ); + for src in [ + "A = { mq = \".h2\" }", + "A = { from = \"x.md\" }", + "A = { mq = \"\", from = \"x.md\" }", + "A = { mq = \".h2\", from = \"x.md\", sh = \"x\" }", + "A = { mq = 1, from = \"x.md\" }", + ] { + let table: toml::Table = toml::from_str(src).unwrap(); + let err = parse_table("ctx", &table).unwrap_err().to_string(); + assert!( + err.contains("`{ mq = \"query\", from = \"file\" }`"), + "{src}: {err}" + ); + } + } + + #[test] + fn an_mq_query_reads_a_file_relative_to_the_directory() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("CHANGELOG.md"), + "# Changelog\n\n## 1.4.0\n\n- a\n\n## 1.3.2\n", + ) + .unwrap(); + let defs = [ + lit("FILE", "CHANGELOG"), + mq( + "LATEST", + "nodes | filter(is_h2) | first() | to_text()", + "${FILE}.md", + ), + mq("ALL", ".h2 | to_text()", "CHANGELOG.md"), + lit("TAG", "v${LATEST}"), + ]; + let vars = resolve_in(&defs, Some(dir.path()), false).unwrap(); + assert_eq!(value(&vars, "LATEST"), "1.4.0"); + // Nodes the selector skips do not leave blank lines. + assert_eq!(value(&vars, "ALL"), "1.4.0\n1.3.2"); + assert_eq!(value(&vars, "TAG"), "v1.4.0"); + } + + #[test] + fn a_missing_file_or_a_bad_query_names_the_variable() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("a.md"), "## x\n").unwrap(); + let err = resolve_in(&[mq("V", ".h2", "nope.md")], Some(dir.path()), false) + .unwrap_err() + .to_string(); + assert!(err.starts_with("Variable 'V': cannot read"), "{err}"); + let err = resolve_in(&[mq("V", ".h9", "a.md")], Some(dir.path()), false) + .unwrap_err() + .to_string(); + assert!(err.contains("Variable 'V': mq query `.h9` failed"), "{err}"); + } + + #[test] + fn a_dry_run_shows_the_query_instead_of_running_it() { + let vars = resolve_in(&[mq("V", ".h2", "nope.md")], None, true).unwrap(); + assert_eq!(value(&vars, "V"), ""); + } + #[test] fn a_dry_run_shows_the_command_instead_of_running_it() { let cache = ShellCache::default();