From 00f1172edff8ebec9362eab4524c337c0a0b431a Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Sat, 24 Oct 2026 05:38:06 +0200 Subject: [PATCH 01/10] An identity source's names get no vectors from the backfill, and turning the flag on drops the ones already stored (#1390) Signed-off-by: Bogdan Szabo --- crates/utopia-server/src/extraction_open.rs | 4 + .../extraction_open_identity_names_tests.rs | 35 +++- crates/utopia-store/src/name_vectors.rs | 69 +++++++- crates/utopia-store/src/resolution.rs | 59 ++++--- crates/utopia-store/src/sources.rs | 5 +- ...dentity_name_attaches_by_its_exact_name.rs | 157 +++++++++++++++++- ...source-may-declare-its-names-identities.md | 6 +- 7 files changed, 290 insertions(+), 45 deletions(-) diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs index 4df943a94..d83faa482 100644 --- a/crates/utopia-server/src/extraction_open.rs +++ b/crates/utopia-server/src/extraction_open.rs @@ -1246,6 +1246,10 @@ pub(crate) async fn run_open( state.emit_review(kb_id); } + // 身份来源（0071 决定 5）：散文先写下的名字向量，这篇给了它身份证据，删掉 + if identity { + utopia_store::name_vectors::drop_for_document(pool, document_id).await?; + } // 名字向量：这篇新写的名字事实，向量补上（0041 决定 3 通道 2）。算不出来只记日志—— // 文档已经抽完了，不能因为召回的辅助数据没算而把它标成 failed if let (Some(settings), Some(client)) = (settings, &embed) { diff --git a/crates/utopia-server/src/extraction_open_identity_names_tests.rs b/crates/utopia-server/src/extraction_open_identity_names_tests.rs index 352f0853f..5e235977d 100644 --- a/crates/utopia-server/src/extraction_open_identity_names_tests.rs +++ b/crates/utopia-server/src/extraction_open_identity_names_tests.rs @@ -1,13 +1,15 @@ //! A statements source with `{"names": "identity"}` (0071) gets no embedding call and no -//! `name_vector|` pair. The fake endpoint returns one vector for every name. +//! `name_vector|` pair, and the name it pushes loses the vector prose gave it first. The fake +//! endpoint returns one vector for every name. use super::*; use axum::{routing::post, Json, Router}; use serde_json::{json, Value}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -/// Push `PUSHED` into a base holding `KNOWN` (with a name vector); answers (embedding calls, review reasons). -async fn push(config: Value) -> anyhow::Result)>> { +/// Push `PUSHED` and `KNOWN` into a base holding `KNOWN` (with a name vector); answers +/// (embedding calls, review reasons, whether `KNOWN` kept its vector). +async fn push(config: Value) -> anyhow::Result, bool)>> { let Some(url) = utopia_store::test_db::url() else { return Ok(None); }; @@ -17,7 +19,7 @@ async fn push(config: Value) -> anyhow::Result)>> { std::array::from_fn(|_| Uuid::now_v7()); let text = serde_json::to_string(&json!({ "external_id": "commit:identity", - "e": [[PUSHED, "SourceFile", true]], + "e": [[PUSHED, "SourceFile", true], [KNOWN, "SourceFile", true]], "s": [[null, PUSHED, "path", null, "src/x.ts", null, null, null]], "n": [] }))?; @@ -90,6 +92,11 @@ async fn push(config: Value) -> anyhow::Result)>> { .bind(kb) .fetch_all(&pool) .await?; + let known_kept: bool = + sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM name_vectors WHERE fact_id = $1)") + .bind(fact) + .fetch_one(&pool) + .await?; server.abort(); sqlx::query("DELETE FROM jobs WHERE payload->>'kb_id' = $1") @@ -100,7 +107,7 @@ async fn push(config: Value) -> anyhow::Result)>> { .bind(org) .execute(&pool) .await?; - Ok(Some((calls.load(Ordering::SeqCst), reasons))) + Ok(Some((calls.load(Ordering::SeqCst), reasons, known_kept))) } const KNOWN: &str = "re-cinq/lore:src/a.ts"; @@ -108,19 +115,29 @@ const PUSHED: &str = "re-cinq/lore:src/b.ts"; #[tokio::test] async fn an_identity_source_adds_no_name_vectors_and_asks_no_neighbours() -> anyhow::Result<()> { - let Some(outcome) = push(json!({"names": "identity"})).await? else { + let Some((calls, reasons, _)) = push(json!({"names": "identity"})).await? else { return Ok(()); }; - assert_eq!(outcome, (0, Vec::::new())); + assert_eq!((calls, reasons), (0, Vec::::new())); + Ok(()) +} + +#[tokio::test] +async fn a_name_prose_gave_a_vector_loses_it_when_an_identity_source_pushes_it( +) -> anyhow::Result<()> { + let Some((_, _, known_kept)) = push(json!({"names": "identity"})).await? else { + return Ok(()); + }; + assert!(!known_kept); Ok(()) } #[tokio::test] async fn a_statements_source_without_the_flag_still_proposes_by_name_vector() -> anyhow::Result<()> { - let Some((calls, reasons)) = push(json!({})).await? else { + let Some((calls, reasons, known_kept)) = push(json!({})).await? else { return Ok(()); }; - assert!(calls > 0 && reasons.iter().any(|r| r.starts_with("name_vector|"))); + assert!(calls > 0 && reasons.iter().any(|r| r.starts_with("name_vector|")) && known_kept); Ok(()) } diff --git a/crates/utopia-store/src/name_vectors.rs b/crates/utopia-store/src/name_vectors.rs index c61f08ab9..14700c80a 100644 --- a/crates/utopia-store/src/name_vectors.rs +++ b/crates/utopia-store/src/name_vectors.rs @@ -25,11 +25,28 @@ pub const TOP_K: usize = 8; /// 0.4 上下的余弦——所以这条线比画像的 `SIM_ATTACH` 高。同样是待测量的临时值 pub const SIM_FLOOR: f32 = 0.60; +/// 名字事实 `$fact` 有身份来源的证据（0071 决定 4、5）：这样的名字不留向量 +macro_rules! has_identity_evidence { + ($fact:literal) => { + concat!( + "EXISTS (SELECT 1 FROM fact_evidence fe + JOIN documents d ON d.id = fe.document_id + JOIN sources s ON s.id = d.source_id + WHERE fe.fact_id = ", + $fact, + " AND s.kind = 'statements' AND s.config->>'names' = 'identity')" + ) + }; +} + /// `known_as` 的 id 用子查询取，不要改回 JOIN：子查询先算出一个值，`predicate_id = 它` 才成了 /// `facts_live_predicate_idx`（0115）的索引条件，扫描只碰这个库的名字事实，且仍按 id 走， /// `ORDER BY ... LIMIT` 不用排序。写成 JOIN，规划器沿 `facts_kb_id_key` 把整个库按 id 读一遍， /// 谓词只能逐行筛（#1296：20 万条事实的库上读 20 万行，这样读 2 万行） -const PENDING_SQL: &str = "SELECT f.id, f.subject_id, f.object_value->>'value' +/// +/// 有身份来源证据的名字不补向量，哪怕散文也提到它（0071 决定 4、5） +const PENDING_SQL: &str = concat!( + "SELECT f.id, f.subject_id, f.object_value->>'value' FROM facts f LEFT JOIN name_vectors v ON v.fact_id = f.id WHERE f.kb_id = $1 @@ -37,10 +54,15 @@ const PENDING_SQL: &str = "SELECT f.id, f.subject_id, f.object_value->>'value' WHERE nr.kb_id = $1 AND nr.builtin AND nr.key = $2) AND f.invalidated_at IS NULL AND f.object_value IS NOT NULL AND v.fact_id IS NULL + AND NOT ", + has_identity_evidence!("f.id"), + " ORDER BY f.id - LIMIT $3"; + LIMIT $3" +); -/// 还没有向量的名字事实：现行的、`known_as` 上的、`name_vectors` 里没有它的。 +/// 还没有向量的名字事实：现行的、`known_as` 上的、`name_vectors` 里没有它的、 +/// 没有身份来源证据的。 /// 按写入先后取（`facts.id` 是 uuid v7，按它排即按写入排；这张表没有 created_at—— /// 端到端跑出来的：按一个不存在的列排，补向量每篇都静默失败），一次取一批 pub async fn pending( @@ -57,19 +79,50 @@ pub async fn pending( Ok(rows) } +/// 来源刚开了身份名（0071 决定 6）：删掉有它文档证据的名字向量，与改配置同一个事务 +pub async fn drop_for_source( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + source_id: Uuid, +) -> AppResult<()> { + sqlx::query( + "DELETE FROM name_vectors v + USING fact_evidence fe JOIN documents d ON d.id = fe.document_id + WHERE fe.fact_id = v.fact_id AND d.source_id = $1", + ) + .bind(source_id) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// 身份来源的文档抽完（0071 决定 5）：散文先写下的名字向量，这篇给了它身份证据就删 +pub async fn drop_for_document(pool: &PgPool, document_id: Uuid) -> AppResult<()> { + sqlx::query( + "DELETE FROM name_vectors v USING fact_evidence fe + WHERE fe.fact_id = v.fact_id AND fe.document_id = $1", + ) + .bind(document_id) + .execute(pool) + .await?; + Ok(()) +} + /// 写一批名字向量：`(fact_id, entity_id, embedding)`。同一条事实重写就覆盖（换了模型 -/// 重算）。第一次写下这个维度的向量，索引就该排上了（0035） +/// 重算）。第一次写下这个维度的向量，索引就该排上了（0035）。 +/// 写的时候再查一次身份证据：补向量算到一半，来源开了身份名或身份文档抽完删过一轮， +/// 这里不能把它写回去 pub async fn set(pool: &PgPool, kb_id: Uuid, items: &[(Uuid, Uuid, Vec)]) -> AppResult<()> { if items.is_empty() { return Ok(()); } let mut tx = pool.begin().await?; for (fact_id, entity_id, emb) in items { - sqlx::query( + sqlx::query(concat!( "INSERT INTO name_vectors (fact_id, kb_id, entity_id, embedding) - VALUES ($1, $2, $3, $4) - ON CONFLICT (fact_id) DO UPDATE SET embedding = EXCLUDED.embedding", - ) + SELECT $1::uuid, $2::uuid, $3::uuid, $4::vector WHERE NOT ", + has_identity_evidence!("$1"), + " ON CONFLICT (fact_id) DO UPDATE SET embedding = EXCLUDED.embedding" + )) .bind(fact_id) .bind(kb_id) .bind(entity_id) diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index 8c7a9e217..b7f34250a 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -402,6 +402,9 @@ pub async fn resolve_identity_name( Some(ctx) => update_profile(pool, holder.id, holder.profile_n, ctx).await?, None => touch_entity(pool, holder.id).await?, } + if holder.type_id.is_none() { + adopt_type(pool, kb_id, holder.id, &holder.canonical_name, type_id).await?; + } Ok(Resolution { entity_id: holder.id, created: false, @@ -1011,8 +1014,6 @@ struct CrossCandidate { type_key: Option, // 同上；`types_are_kin` 要按 id 走类层级 type_id: Option, - // 抽取升格要看它：人说过「就是没有类型」时，那也是一个决定 - type_source: String, profile_embedding: Option, profile_n: i32, } @@ -1125,6 +1126,38 @@ fn confusable_reviews( .collect() } +/// 持有者还没判出类型而这次判出来了 → 升格。 +/// 不是合并（没有第二个实体），不入 entity_merges；本体页可手工改回 +/// +/// **人说过的「没有类型」不算「还没判出来」。** 0009 之后两者都是 +/// NULL，只看 type_id 分不出——于是一个人看过、认为 +/// 本体里没有合适类的实体，会在下一次抽取时被安上一个类型 +async fn adopt_type( + pool: &PgPool, + kb_id: Uuid, + entity_id: Uuid, + canonical_name: &str, + type_id: Option, +) -> AppResult<()> { + let Some(type_id) = type_id else { + return Ok(()); + }; + let adopted = sqlx::query( + "UPDATE entities + SET type_id = $2, type_source = 'extracted', updated_at = now() + WHERE id = $1 AND type_id IS NULL AND type_source <> 'human'", + ) + .bind(entity_id) + .bind(type_id) + .execute(pool) + .await?; + if adopted.rows_affected() > 0 { + // 类型标签兜底的消歧后缀可能已过时 + refresh_disambiguators(pool, kb_id, canonical_name).await?; + } + Ok(()) +} + /// 同类型召回为空时的跨类型处置（类型漂移）。 /// concept 兜底型候选跑既有画像分层：高相似直接 ATTACH（漂移的召回修复，concept /// 侧类型升格为具体类型），灰区/无从判别 → 宁分勿合新建 + 审核对；易混具体类型 @@ -1152,7 +1185,7 @@ async fn resolve_type_drift( None => None, }; let cross: Vec = sqlx::query_as(&format!( - "SELECT e.id, e.canonical_name, t.key AS type_key, e.type_id, e.type_source, + "SELECT e.id, e.canonical_name, t.key AS type_key, e.type_id, e.profile_embedding, e.profile_n FROM entities e LEFT JOIN entity_types t ON t.id = e.type_id -- IS DISTINCT FROM 而不是 <>：后者遇 NULL 返回 NULL，被 WHERE 当假， @@ -1210,24 +1243,8 @@ async fn resolve_type_drift( if let Some((best, sim)) = best { if sim >= SIM_ATTACH { update_profile(pool, best.id, best.profile_n, ctx).await?; - // 候选还没判出类型而这次抽取判出来了 → 升格。 - // 不是合并（没有第二个实体），不入 entity_merges；本体页可手工改回 - // - // **人说过的「没有类型」不算「还没判出来」。** 0009 之后两者都是 - // NULL，只看 type_key.is_none() 分不出——于是一个人看过、认为 - // 本体里没有合适类的实体，会在下一次抽取时被安上一个类型 - if best.type_key.is_none() && best.type_source != "human" && type_id.is_some() { - sqlx::query( - "UPDATE entities - SET type_id = $2, type_source = 'extracted', updated_at = now() - WHERE id = $1", - ) - .bind(best.id) - .bind(type_id) - .execute(pool) - .await?; - // 类型标签兜底的消歧后缀可能已过时 - refresh_disambiguators(pool, kb_id, &best.canonical_name).await?; + if best.type_key.is_none() { + adopt_type(pool, kb_id, best.id, &best.canonical_name, type_id).await?; } // mention 已定居到召回实体；同名易混类型实体的疑点仍在 → 照常入队 let mut reviews = diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rs index 7f596c6ec..a669d0151 100644 --- a/crates/utopia-store/src/sources.rs +++ b/crates/utopia-store/src/sources.rs @@ -281,7 +281,7 @@ pub async fn update( && old_full && new_full && rss_feed_url(&previous.config) != rss_feed_url(&next_config); - let source = sqlx::query_as( + let source: Source = sqlx::query_as( "UPDATE sources SET name = COALESCE($2, name), config = COALESCE($3, config), @@ -303,6 +303,9 @@ pub async fn update( if (!old_full && new_full) || feed_url_changed { crate::rss_full_content::enable_source(&mut tx, id).await?; } + if !previous.names_are_identities() && source.names_are_identities() { + crate::name_vectors::drop_for_source(&mut tx, id).await?; + } tx.commit().await?; opened(source) } diff --git a/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs b/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs index 647606416..d3fc88177 100644 --- a/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs +++ b/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs @@ -1,8 +1,11 @@ -//! 0071 decisions 2 and 3: an identity mention attaches to the exact-name holder whatever the -//! profile says, and is created otherwise. Skipped without `UTOPIA_DATABASE_URL`. +//! 0071 decisions 2 to 6: an identity mention attaches to the exact-name holder whatever the +//! profile says, and is created otherwise; its names get no vectors. Skipped without +//! `UTOPIA_DATABASE_URL`. +use serde_json::json; use sqlx::PgPool; -use utopia_store::resolution::resolve_identity_name; +use utopia_store::names::{record, NameSource}; +use utopia_store::{name_vectors, resolution::resolve_identity_name, sources}; use uuid::Uuid; /// Orthogonal to every holder's profile, so the profile never supports an attach. @@ -86,6 +89,26 @@ async fn one_exact_holder_is_attached_and_otherwise_the_name_is_created() -> any let untyped = resolve_identity_name(&pool, kb, None, "acme/d.ts", Some(&same), None, &[]).await?; assert!(untyped.created && untyped.entity_id != typed); + // an untyped holder takes the declared class + let untyped_holder = holder(&pool, kb, None, "acme/f.ts").await?; + resolve("acme/f.ts").await?; + let class: Option = sqlx::query_scalar("SELECT type_id FROM entities WHERE id = $1") + .bind(untyped_holder) + .fetch_one(&pool) + .await?; + assert_eq!(class, Some(project)); + // a class a person removed stays removed + let cleared = holder(&pool, kb, None, "acme/g.ts").await?; + sqlx::query("UPDATE entities SET type_source = 'human' WHERE id = $1") + .bind(cleared) + .execute(&pool) + .await?; + resolve("acme/g.ts").await?; + let class: Option = sqlx::query_scalar("SELECT type_id FROM entities WHERE id = $1") + .bind(cleared) + .fetch_one(&pool) + .await?; + assert_eq!(class, None); Ok(()) }) .await @@ -121,3 +144,131 @@ async fn a_name_under_another_class_is_created_with_a_drift_review_unless_disjoi }) .await } + +/// A statements source with this config and one chunk in it; `None` is an upload's chunk. +async fn chunk( + pool: &PgPool, + kb: Uuid, + config: Option, +) -> anyhow::Result<(Uuid, Uuid)> { + let (source, document, chunk) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + if let Some(config) = &config { + sqlx::query("INSERT INTO sources (id, kb_id, kind, name, config) VALUES ($1, $2, 'statements', 's', $3)") + .bind(source) + .bind(kb) + .bind(config) + .execute(pool) + .await?; + } + sqlx::query("INSERT INTO documents (id, kb_id, filename, sha256, source_id) VALUES ($1, $2, 'd', $1::text, $3)") + .bind(document) + .bind(kb) + .bind(config.map(|_| source)) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO chunks (id, kb_id, document_id, seq, text) VALUES ($1, $2, $3, 0, 'acme')", + ) + .bind(chunk) + .bind(kb) + .bind(document) + .execute(pool) + .await?; + Ok((source, chunk)) +} + +/// Names `entity` once from each chunk and returns the one name fact. +async fn named( + pool: &PgPool, + kb: Uuid, + entity: Uuid, + name: &str, + chunks: &[Uuid], +) -> anyhow::Result { + let mut fact = None; + for &chunk_id in chunks { + fact = record( + pool, + kb, + entity, + name, + Some(NameSource { + chunk_id, + quote: name, + }), + None, + ) + .await?; + } + fact.ok_or_else(|| anyhow::anyhow!("no name fact")) +} + +async fn vectors(pool: &PgPool, kb: Uuid) -> anyhow::Result> { + Ok( + sqlx::query_scalar("SELECT fact_id FROM name_vectors WHERE kb_id = $1") + .bind(kb) + .fetch_all(pool) + .await?, + ) +} + +async fn pending_entities(pool: &PgPool, kb: Uuid) -> anyhow::Result> { + Ok(name_vectors::pending(pool, kb, 100) + .await? + .into_iter() + .map(|(_, entity, _)| entity) + .collect()) +} + +#[tokio::test] +async fn a_name_with_identity_evidence_gets_no_vector_even_when_prose_names_it_too( +) -> anyhow::Result<()> { + in_kb(|pool, kb, project, _| async move { + let (_, identity) = chunk(&pool, kb, Some(json!({"names": "identity"}))).await?; + let (_, prose) = chunk(&pool, kb, None).await?; + let both = holder(&pool, kb, Some(project), "acme/a.ts").await?; + let both_name = named(&pool, kb, both, "acme/a.ts", &[identity, prose]).await?; + let prose_only = holder(&pool, kb, Some(project), "acme/b.ts").await?; + named(&pool, kb, prose_only, "acme/b.ts", &[prose]).await?; + assert_eq!(pending_entities(&pool, kb).await?, vec![prose_only]); + // a backfill that read it as pending before the identity evidence came writes nothing + name_vectors::set(&pool, kb, &[(both_name, both, vec![1.0, 0.0, 0.0])]).await?; + assert_eq!(vectors(&pool, kb).await?, Vec::::new()); + Ok(()) + }) + .await +} + +#[tokio::test] +async fn turning_the_flag_on_drops_the_sources_name_vectors_and_off_makes_them_pending( +) -> anyhow::Result<()> { + in_kb(|pool, kb, project, _| async move { + let (source, pushed) = chunk(&pool, kb, Some(json!({}))).await?; + let (_, prose) = chunk(&pool, kb, None).await?; + let file = holder(&pool, kb, Some(project), "acme/a.ts").await?; + let file_name = named(&pool, kb, file, "acme/a.ts", &[pushed]).await?; + let other = holder(&pool, kb, Some(project), "acme/b.ts").await?; + let other_name = named(&pool, kb, other, "acme/b.ts", &[prose]).await?; + let vector = vec![1.0, 0.0, 0.0]; + name_vectors::set( + &pool, + kb, + &[ + (file_name, file, vector.clone()), + (other_name, other, vector), + ], + ) + .await?; + let flag = |config: serde_json::Value| { + let pool = pool.clone(); + async move { sources::update(&pool, source, None, Some(&config), None, None).await } + }; + + flag(json!({"names": "identity"})).await?; + assert_eq!(vectors(&pool, kb).await?, vec![other_name]); + flag(json!({})).await?; + assert!(pending_entities(&pool, kb).await?.contains(&file)); + Ok(()) + }) + .await +} diff --git a/docs/decisions/0071-a-source-may-declare-its-names-identities.md b/docs/decisions/0071-a-source-may-declare-its-names-identities.md index c0c76194b..8b9f5f9df 100644 --- a/docs/decisions/0071-a-source-may-declare-its-names-identities.md +++ b/docs/decisions/0071-a-source-may-declare-its-names-identities.md @@ -1,6 +1,6 @@ # 0071 · A source may declare its names identities -- **Status**: In progress · 2026-10-19 · [#1293](https://github.com/deeplethe/utopia/issues/1293) · open: decisions 5, 6 and 7, and the `name_vectors::pending` skip of decision 4 +- **Status**: In progress · 2026-10-19 · [#1293](https://github.com/deeplethe/utopia/issues/1293) · open: decision 7 - **Written**: 2026-10-19 (conventions in the [README](README.md)) - **Related**: [0054](0054-a-source-may-push-statements-in-the-open-contract.md) decision 3 sends pushed statements down the same resolution path as prose, and this record narrows that for one kind of source; [0041](0041-a-name-is-a-claim-about-an-entity.md) owns name facts and name-vector recall; [#1248](https://github.com/deeplethe/utopia/issues/1248) meets the same problem from the merge side and is decided here as decision 7; [#1296](https://github.com/deeplethe/utopia/issues/1296) measured what the recall costs; [#1308](https://github.com/deeplethe/utopia/pull/1308) is the disjointness fix that stands on its own @@ -34,7 +34,7 @@ A mention from an identity source runs no `name_vectors::nearest` query and prop **5. Identity evidence wins.** -One name fact can carry evidence from both an identity source and a prose document, because facts dedupe on subject, predicate and object. Such a fact gets no vector. The identity source has vouched for the name, and a prose mention still reaches it by exact match. This changes recall for prose: a prose mention of a *variant* of that name (an abbreviation, another script) no longer finds the entity through its vector, and the code PR covers that with the test `a_name_with_identity_evidence_gets_no_vector_even_when_prose_names_it_too`. +One name fact can carry evidence from both an identity source and a prose document, because facts dedupe on subject, predicate and object. Such a fact gets no vector. The identity source has vouched for the name, and a prose mention still reaches it by exact match. The order does not matter: when an identity document is extracted, the vectors of the name facts it gave evidence to are deleted, and a backfill writes no vector for a name that has identity evidence by then. This changes recall for prose: a prose mention of a *variant* of that name (an abbreviation, another script) no longer finds the entity through its vector, and the code PR covers that with the test `a_name_with_identity_evidence_gets_no_vector_even_when_prose_names_it_too`. **6. Switching the flag.** @@ -47,7 +47,7 @@ One name fact can carry evidence from both an identity source and a prose docume Why a person and not a stricter threshold: an identity source has already said what its names are. Decisions 2 to 4 remove the pairs that name vectors and the single-candidate split would raise, so what is left are containment and type-drift pairs and the cross-source duplicate (one thing pushed under two names by two producers). Those are exactly the cases where the model's `same` is a guess about a claim the producer made, and a wrong merge of identities is what #1248 measured: on a small local model, 40 automatic merges with governance off, nearly all wrong (a method into its file, two unrelated types). A person sees few of them because the vectors are off. Raising `AUTO_CONF` for these pairs would still let a confident wrong verdict through, and a per-base switch is a dead end (below). -**A NULL-typed holder keeps its class for now.** A typed mention that attaches to a holder with `type_id IS NULL` does not upgrade the holder's class, as the type-drift path does. PR 2 decides that. +**A NULL-typed holder takes the declared class.** *Revised 2026-10-23 (#1390):* this note said a holder with `type_id IS NULL` keeps its class and left the choice to the next PR. A typed identity mention that attaches to such a holder now gives it the declared class, as the type-drift path does and with the same helper, so a class a person removed stays removed. ## Not doing From 292aa66ebc7f80a9ef8a4a6d5d03e3527c0756e6 Mon Sep 17 00:00:00 2001 From: Nash Aschenbath Date: Sat, 24 Oct 2026 11:43:31 +0800 Subject: [PATCH 02/10] changes and entity_facts take an offset and say how to fetch the next page, and find_entities reports its match count in MCP output (#1396) Signed-off-by: Aschenbath Co-authored-by: Aschenbath Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-server/src/api/chat.rs | 10 +- crates/utopia-server/src/api/mcp_tests.rs | 176 ++++++++++++++++++ crates/utopia-server/src/api/tools.rs | 54 +++++- crates/utopia-server/src/api/tools_graph.rs | 25 ++- crates/utopia-store/src/graph.rs | 10 +- .../utopia-store/tests/store/graph_changes.rs | 1 + .../tests/store/no_predicate_still_shows.rs | 1 + .../store/the_second_clock_can_be_rewound.rs | 1 + .../0020-an-auditor-reads-it-without-us.md | 19 +- web/src/docs/mcp.md | 33 ++-- 10 files changed, 293 insertions(+), 37 deletions(-) diff --git a/crates/utopia-server/src/api/chat.rs b/crates/utopia-server/src/api/chat.rs index 47b6a3444..13c0f9a85 100644 --- a/crates/utopia-server/src/api/chat.rs +++ b/crates/utopia-server/src/api/chat.rs @@ -433,8 +433,8 @@ pub(super) fn base_tools() -> serde_json::Value { Conclusions a business rule reached about this entity come back too, \ marked `[rule: ]` with the readings that made them true — take \ those as given rather than re-deriving them from the readings yourself. \ - Output is grouped by predicate and cut at `limit`; narrow with predicate, \ - object_type, since / until, or use timeline / neighbors / paths_between.", + Output is grouped by predicate and cut at `limit`; use `offset` to continue. \ + Narrow with predicate, object_type, since / until, or use timeline / neighbors / paths_between.", "parameters": { "type": "object", "properties": { @@ -444,6 +444,7 @@ pub(super) fn base_tools() -> serde_json::Value { "since": { "type": "string", "description": "Optional WORLD-time window start: facts still holding after it." }, "until": { "type": "string", "description": "Optional WORLD-time window end: facts that had started by it." }, "limit": { "type": "integer", "description": "How many facts to return (default 80, max 300); the reply says when it is cut." }, + "offset": { "type": "integer", "minimum": 0, "description": "Zero-based position in the filtered facts (default 0). Use the reply's next_offset to continue, keeping the entity, filters and time parameters unchanged." }, "at": { "type": "string", "description": "Optional as-of moment on the WORLD axis (YYYY, YYYY-MM, \ @@ -568,7 +569,7 @@ pub(super) fn base_tools() -> serde_json::Value { fractional seconds. \ For 'before a correction arrived', find that event here and pass its \ timestamp, exactly as printed, to entity_facts as `before`. Keep at for the \ - world date asked about.", + world date asked about. Returns up to 40 events; use `offset` to continue.", "parameters": { "type": "object", "properties": { @@ -595,7 +596,8 @@ pub(super) fn base_tools() -> serde_json::Value { "description": "Optional filter. A freshly ingested corpus is nearly \ all 'asserted'; pass [\"corrected\", \"rejected\"] to isolate \ the places we actually changed our mind." - } + }, + "offset": { "type": "integer", "minimum": 0, "description": "Zero-based position in the matching events (default 0). Use the reply's next_offset to continue, keeping the window and filters unchanged." } }, "required": ["since"] } diff --git a/crates/utopia-server/src/api/mcp_tests.rs b/crates/utopia-server/src/api/mcp_tests.rs index ee16cd1b2..173069cf3 100644 --- a/crates/utopia-server/src/api/mcp_tests.rs +++ b/crates/utopia-server/src/api/mcp_tests.rs @@ -1195,6 +1195,109 @@ async fn entity_facts_keeps_identity_values_filters_and_both_clocks() -> anyhow: f.clean().await } +#[tokio::test] +async fn entity_facts_pages_filtered_rows_without_repeating_time_ties() -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + async fn check(f: &Fixture) -> anyhow::Result<()> { + let extra: Vec = (0..4).map(|_| Uuid::new_v4()).collect(); + sqlx::query( + "INSERT INTO facts(id,kb_id,subject_id,predicate_id,object_value, + valid_from,valid_from_precision,recorded_at) + SELECT extra.id,f.kb_id,f.subject_id,f.predicate_id, + jsonb_build_object('value',extra.n),f.valid_from, + f.valid_from_precision,f.recorded_at + FROM facts f CROSS JOIN unnest($2::uuid[]) WITH ORDINALITY extra(id,n) + WHERE f.id=$1", + ) + .bind(f.attribute) + .bind(&extra) + .execute(&f.state.pool) + .await?; + let mut expected = extra; + expected.push(f.attribute); + expected.sort(); + for offset in [0, 2, 3, 4, 5, 99] { + let page = f + .call( + "entity_facts", + json!({"entity_id":f.subject,"predicate":"weight","limit":2, + "offset":offset,"at":"2026-01-01","before":CORRECTION}), + ) + .await?; + let data = &page["structuredContent"]; + assert_eq!(page["isError"], false); + assert_eq!(data["offset"], offset); + assert_eq!(data["matched_facts"], 5); + let ids: Vec = data["facts"] + .as_array() + .unwrap() + .iter() + .map(|fact| uuid(&fact["id"])) + .collect(); + assert_eq!(ids, expected[offset.min(5)..(offset + 2).min(5)]); + let next = (offset + 2 < 5).then_some(offset + 2); + assert_eq!(data["next_offset"], json!(next)); + assert_eq!(data["truncated"], next.is_some()); + if let Some(next) = next { + assert!(page["content"][0]["text"] + .as_str() + .unwrap() + .contains(&format!("offset={next}"))); + } + } + Ok(()) + } + let result = check(&f).await; + f.clean().await?; + result +} + +#[tokio::test] +async fn paged_tools_advertise_offsets_and_refuse_invalid_values() -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let listed = f + .request(f.kb, "tools/list", json!({})) + .await + .map_err(|e| e.0)? + .0; + for name in ["entity_facts", "changes"] { + let tool = listed["result"]["tools"] + .as_array() + .unwrap() + .iter() + .find(|tool| tool["name"] == name) + .unwrap(); + assert_eq!( + tool["inputSchema"]["properties"]["offset"]["type"], + "integer" + ); + assert_eq!(tool["inputSchema"]["properties"]["offset"]["minimum"], 0); + } + for offset in [ + json!(-1), + json!(1.5), + json!("2"), + json!(null), + json!(u64::MAX), + ] { + for tool in ["entity_facts", "changes"] { + let result = f + .call( + tool, + json!({"entity_id":f.subject,"since":"2026","offset":offset}), + ) + .await?; + assert_eq!(result["isError"], true, "{tool}: {result}"); + assert!(result.get("structuredContent").is_none()); + } + } + f.clean().await +} + #[tokio::test] async fn changes_returns_fact_ids_and_a_reusable_correction_timestamp() -> anyhow::Result<()> { let Some(f) = Fixture::new().await? else { @@ -1255,6 +1358,17 @@ async fn changes_returns_fact_ids_and_a_reusable_correction_timestamp() -> anyho .bind(f.attribute) .execute(&f.state.pool) .await?; + let ids: Vec = sqlx::query_scalar( + "SELECT id FROM facts WHERE kb_id=$1 AND recorded_at='2026-03-21' ORDER BY id", + ) + .bind(f.kb) + .fetch_all(&f.state.pool) + .await?; + // 同一事实同一时刻的两种事件恰好跨页；只按时间或事实 id 排都不够。 + sqlx::query("UPDATE facts SET invalidated_at=recorded_at WHERE id=$1") + .bind(ids[39]) + .execute(&f.state.pool) + .await?; let capped = f .call( "changes", @@ -1269,6 +1383,64 @@ async fn changes_returns_fact_ids_and_a_reusable_correction_timestamp() -> anyho 40 ); assert_eq!(capped["structuredContent"]["limit_reached"], true); + assert_eq!(capped["structuredContent"]["offset"], 0); + assert_eq!(capped["structuredContent"]["next_offset"], 40); + assert!(capped["content"][0]["text"] + .as_str() + .unwrap() + .contains("offset=40")); + assert_eq!( + capped["structuredContent"]["changes"] + .as_array() + .unwrap() + .iter() + .map(|event| (uuid(&event["fact_id"]), event["kind"].as_str().unwrap())) + .collect::>(), + ids[..40] + .iter() + .map(|id| (*id, "asserted")) + .collect::>() + ); + let last = f + .call( + "changes", + json!({"since":"2026-03-21","until":"2026-03-21","offset":40}), + ) + .await?; + let events = last["structuredContent"]["changes"].as_array().unwrap(); + assert_eq!(events.len(), 2); + assert_eq!(uuid(&events[0]["fact_id"]), ids[39]); + assert_eq!(events[0]["kind"], "rejected"); + assert_eq!(uuid(&events[1]["fact_id"]), ids[40]); + assert_eq!(last["structuredContent"]["offset"], 40); + assert!(last["structuredContent"]["next_offset"].is_null()); + assert_eq!(last["structuredContent"]["limit_reached"], false); + let exhausted = f + .call( + "changes", + json!({"since":"2026-03-21","until":"2026-03-21","offset":42}), + ) + .await?; + assert_eq!(exhausted["structuredContent"]["changes"], json!([])); + assert!(exhausted["structuredContent"]["next_offset"].is_null()); + assert_eq!(exhausted["isError"], false); + // 满四十条不等于还有下一页；过滤后再偏移，断言事件正好剩四十条。 + let full_last = f + .call( + "changes", + json!({"since":"2026-03-21","until":"2026-03-21","offset":1, + "entity_id":f.subject,"kinds":["asserted"]}), + ) + .await?; + assert_eq!(full_last["structuredContent"]["limit_reached"], true); + assert!(full_last["structuredContent"]["next_offset"].is_null()); + assert_eq!( + full_last["structuredContent"]["changes"] + .as_array() + .unwrap() + .len(), + 40 + ); f.clean().await } @@ -3119,6 +3291,10 @@ async fn a_list_cut_at_its_limit_says_so() -> anyhow::Result<()> { ); let gadgets = f.call("find_entities", json!({"name":"Gadget"})).await?; anyhow::ensure!(!text(&gadgets).contains("were read"), "{gadgets}"); + assert_eq!(widgets["structuredContent"]["matched_entities"], 10); + assert_eq!(widgets["structuredContent"]["truncated"], true); + assert_eq!(gadgets["structuredContent"]["matched_entities"], 3); + assert_eq!(gadgets["structuredContent"]["truncated"], false); step_has( &step_of(f, &[], "find_entities", json!({"name":"Widget"})).await, json!({"status":"ok","count":8,"total":10}), diff --git a/crates/utopia-server/src/api/tools.rs b/crates/utopia-server/src/api/tools.rs index 08b216cf9..b64eda518 100644 --- a/crates/utopia-server/src/api/tools.rs +++ b/crates/utopia-server/src/api/tools.rs @@ -21,7 +21,7 @@ const TOOL_CHUNK_CHARS: usize = 800; const DOCUMENT_CHARS: usize = 24_000; /// 一次 changes 最多回多少条。刚灌完的库里 asserted 是成百上千条，全发出去 /// 只会把上下文填满而不增加信息——有信息量的是 corrected/rejected，那类事件 -/// 本来就稀少。截断时正文末尾写明，模型据此收窄窗口：界面上那一步的 "40+" +/// 本来就稀少。截断时正文末尾给出下一页，模型据此续读：界面上那一步的 "40+" /// 只给人看，模型读不到 const CHANGES_LIMIT: i64 = 40; @@ -768,7 +768,30 @@ pub async fn rule_matches(ctx: &ToolCtx<'_>, args: &serde_json::Value) -> ToolRe ) } +/// 两个按行分页的工具共用边界：数据库 OFFSET 收 i64，内存分页收 usize。 +pub(super) fn page_offset(args: &serde_json::Value, kind: &str) -> Result { + match args.get("offset") { + None => Ok(0), + Some(value) => value + .as_i64() + .and_then(|n| usize::try_from(n).ok()) + .ok_or_else(|| { + ToolResult::new( + "Expected `offset` to be a non-negative integer row position.".into(), + Step::new(kind, "?", "invalid offset") + .invalid("offset") + .json(), + ) + .error() + }), + } +} + pub async fn changes(ctx: &ToolCtx<'_>, args: &serde_json::Value) -> ToolResult { + let offset = match page_offset(args, "changes") { + Ok(offset) => offset, + Err(result) => return result, + }; // 两端与 `at` 同一种写法：YYYY / YYYY-MM / YYYY-MM-DD。`since` 取那一段的第一天， // `until` 取最后一天——「2023 年有什么变化」不该逼模型编一个 12 月 31 日 let since = args["since"] @@ -799,14 +822,15 @@ pub async fn changes(ctx: &ToolCtx<'_>, args: &serde_json::Value) -> ToolResult .collect() }); let kinds = kinds.filter(|k: &Vec| !k.is_empty()); - let rows = match utopia_store::graph::graph_changes( + let mut rows = match utopia_store::graph::graph_changes( &ctx.state.pool, ctx.kb_id, since, until, entity, kinds.as_deref(), - CHANGES_LIMIT, + CHANGES_LIMIT + 1, + offset as i64, ) .await { @@ -822,29 +846,38 @@ pub async fn changes(ctx: &ToolCtx<'_>, args: &serde_json::Value) -> ToolResult .error(); } }; - let text = if rows.is_empty() { + // 满一页不等于还有下一页；多读的一条只用来给出续读位置。 + let more = rows.len() as i64 > CHANGES_LIMIT; + rows.truncate(CHANGES_LIMIT as usize); + let next_offset = more.then(|| offset + rows.len()); + let text = if rows.is_empty() && offset > 0 { + format!("No recorded changes remaining in {window} at offset {offset}.") + } else if rows.is_empty() { format!("No recorded changes in {window}.") } else { let mut lines: Vec = rows.iter().map(change_line).collect(); // 取的是**最近**的四十条，更早的被截掉。不说的话模型会把这四十条当成窗口的 // 全部，而要找的那次更正可能正好在更早的地方 - if rows.len() as i64 == CHANGES_LIMIT { + if let Some(next) = next_offset { + let page = if offset == 0 { + format!("Only the {CHANGES_LIMIT} most recent changes in this window are listed") + } else { + format!("{CHANGES_LIMIT} changes at offset {offset} are listed") + }; lines.push(format!( - "(Only the {CHANGES_LIMIT} most recent changes in this window are listed; it may \ - hold more. Narrow since/until, or pass kinds such as [\"corrected\", \ - \"rejected\"], to see the rest.)" + "({page}; call changes with offset={next} and the same window and filters to continue.)" )); } lines.join("\n") }; - let detail = if rows.len() as i64 == CHANGES_LIMIT { + let detail = if more { format!("{CHANGES_LIMIT}+ changes") } else { format!("{} changes", rows.len()) }; let mut step = Step::new("changes", window.as_str(), detail) .count(rows.len()) - .more(rows.len() as i64 == CHANGES_LIMIT); + .more(more); if let Some(day) = since_day { step = step.field("since", day.to_string()); } @@ -854,6 +887,7 @@ pub async fn changes(ctx: &ToolCtx<'_>, args: &serde_json::Value) -> ToolResult ToolResult::new(text, step.json()).structured(json!({ "kb_id": ctx.kb_id, "since": since, "until": until, "limit": CHANGES_LIMIT, "limit_reached": rows.len() as i64 == CHANGES_LIMIT, + "offset": offset, "next_offset": next_offset, "changes": rows.iter().map(|r| json!({ "fact_id": r.fact_id, "at": r.at, "kind": r.kind, "subject_id": r.subject_id, "subject_name": r.subject_name, diff --git a/crates/utopia-server/src/api/tools_graph.rs b/crates/utopia-server/src/api/tools_graph.rs index eb27a1214..f9870f184 100644 --- a/crates/utopia-server/src/api/tools_graph.rs +++ b/crates/utopia-server/src/api/tools_graph.rs @@ -12,8 +12,8 @@ //! 写在结果最上面，模型不同意就拿 id 再来一次。 use super::tools::{ - entity_facts_detail, fact_line, just_before, literal_text, parse_when, Step, ToolCtx, - ToolResult, ToolSink, + entity_facts_detail, fact_line, just_before, literal_text, page_offset, parse_when, Step, + ToolCtx, ToolResult, ToolSink, }; use serde_json::{json, Value}; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -331,6 +331,7 @@ pub async fn find_entities(ctx: &ToolCtx<'_>, sink: &mut ToolSink, args: &Value) } ToolResult::new(text, step.json()).structured(json!({ "kb_id": ctx.kb_id, + "matched_entities": total, "truncated": total.is_some_and(|t| t > read as i64), "entities": ranked.iter().map(|n| json!({ "id": n.id, "name": n.name, "type_key": n.type_key, "type_label": n.type_label, "disambiguator": n.disambiguator, @@ -679,6 +680,10 @@ impl FactFilter<'_> { } pub async fn entity_facts(ctx: &ToolCtx<'_>, sink: &mut ToolSink, args: &Value) -> ToolResult { + let offset = match page_offset(args, "facts") { + Ok(offset) => offset, + Err(result) => return result, + }; let raw = args["entity_id"] .as_str() .or_else(|| args["entity"].as_str()) @@ -769,7 +774,9 @@ pub async fn entity_facts(ctx: &ToolCtx<'_>, sink: &mut ToolSink, args: &Value) Vec::new() }); let (kept, aligned) = filtered(ctx, sink, &facts, &filter).await; - let shown: Vec<&EntityFact> = kept.iter().copied().take(limit).collect(); + let shown: Vec<&EntityFact> = kept.iter().copied().skip(offset).take(limit).collect(); + let end = offset + shown.len(); + let next_offset = (end < kept.len()).then_some(end); let mut lines: Vec = Vec::new(); if let Some(note) = &who.note { lines.push(note.clone()); @@ -806,11 +813,12 @@ pub async fn entity_facts(ctx: &ToolCtx<'_>, sink: &mut ToolSink, args: &Value) )); } } - if shown.len() < kept.len() { + if offset > 0 || shown.len() < kept.len() { + head.push_str(&format!(", {} shown at offset {offset}", shown.len())); + } + if let Some(next) = next_offset { head.push_str(&format!( - ", {} shown. Narrow with predicate=, object_type=, since/until; timeline for the \ - dated ones; neighbors for the related entities", - shown.len() + ". Call entity_facts with offset={next} and the same entity, filters and time parameters to continue" )); } lines.push(head); @@ -853,7 +861,8 @@ pub async fn entity_facts(ctx: &ToolCtx<'_>, sink: &mut ToolSink, args: &Value) })).collect::>(), "at": m.at, "as_of": m.as_of, "before": m.before, "total_facts": facts.len(), "matched_facts": kept.len(), - "limit": limit, "truncated": shown.len() < kept.len(), + "limit": limit, "offset": offset, "next_offset": next_offset, + "truncated": next_offset.is_some(), "facts": shown.iter().map(|f| json!({ "id": f.id, "direction": f.direction, "predicate_key": f.predicate_key, "predicate_label": f.predicate_label, diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs index 61a39b9ca..e96a58540 100644 --- a/crates/utopia-store/src/graph.rs +++ b/crates/utopia-store/src/graph.rs @@ -1833,7 +1833,7 @@ pub async fn entity_detail( AND NOT {represented} AND ({subject} = $2 OR {object} = $2) AND {not_name} - ORDER BY f.valid_from NULLS LAST, f.recorded_at", + ORDER BY f.valid_from NULLS LAST, f.recorded_at, f.id", targets = TIME_CORRECTED_TARGETS, corrected_ends = corrected_ends_sql("f"), correction_note = correction_note_sql("f"), @@ -2495,6 +2495,7 @@ pub async fn entity_history( /// /// 事件推导与 `entity_history` 同源（见那里的注释）：一条事实行最多产出两个事件， /// 且已被后继修正的死亡不重复记。 +#[allow(clippy::too_many_arguments)] pub async fn graph_changes( pool: &PgPool, kb_id: Uuid, @@ -2503,6 +2504,7 @@ pub async fn graph_changes( entity_id: Option, kinds: Option<&[String]>, limit: i64, + offset: i64, ) -> AppResult> { // 两个分支各自按**自己那根时间列**开窗，而不是先union再过滤： // 一条 2 月写入、8 月被推翻的事实，在"3–4 月"窗口里两个事件都不该出现 @@ -2553,8 +2555,9 @@ pub async fn graph_changes( -- 限量的变更清单会被它挤满。新读到的别名、改名照样列出来 AND NOT (coalesce(r.builtin AND r.key = 'known_as', false) AND lower(ev.object_value->>'value') = lower(s.canonical_name)) - ORDER BY ev.at DESC, ev.id - LIMIT $6" + -- 同一事实可以在同一时刻写入并作废；事件种类也得参与排序才能稳定续页。 + ORDER BY ev.at DESC, ev.id, ev.kind + LIMIT $6 OFFSET $7" )) .bind(kb_id) .bind(since) @@ -2562,6 +2565,7 @@ pub async fn graph_changes( .bind(entity_id) .bind(kinds) .bind(limit) + .bind(offset) .fetch_all(pool) .await?) } diff --git a/crates/utopia-store/tests/store/graph_changes.rs b/crates/utopia-store/tests/store/graph_changes.rs index 9c1300380..c2524f1a5 100644 --- a/crates/utopia-store/tests/store/graph_changes.rs +++ b/crates/utopia-store/tests/store/graph_changes.rs @@ -176,6 +176,7 @@ async fn ledger_events_are_derived_as_specified() -> anyhow::Result<()> { entity, kinds.as_deref(), 100, + 0, ) .await } diff --git a/crates/utopia-store/tests/store/no_predicate_still_shows.rs b/crates/utopia-store/tests/store/no_predicate_still_shows.rs index 490869468..1a10f3ca5 100644 --- a/crates/utopia-store/tests/store/no_predicate_still_shows.rs +++ b/crates/utopia-store/tests/store/no_predicate_still_shows.rs @@ -261,6 +261,7 @@ async fn a_fact_without_a_predicate_is_still_visible_everywhere() -> anyhow::Res None, None, 50, + 0, ) .await?; let c = changes.iter().find(|c| c.fact_id == f.surfaced); diff --git a/crates/utopia-store/tests/store/the_second_clock_can_be_rewound.rs b/crates/utopia-store/tests/store/the_second_clock_can_be_rewound.rs index 725148603..d02f8b71e 100644 --- a/crates/utopia-store/tests/store/the_second_clock_can_be_rewound.rs +++ b/crates/utopia-store/tests/store/the_second_clock_can_be_rewound.rs @@ -286,6 +286,7 @@ async fn the_recording_axis_rewinds_on_every_graph_read() -> anyhow::Result<()> Some(f.zhang), None, 10, + 0, ) .await?; assert_eq!(events.len(), 2); diff --git a/docs/decisions/0020-an-auditor-reads-it-without-us.md b/docs/decisions/0020-an-auditor-reads-it-without-us.md index 625078c65..c1513c441 100644 --- a/docs/decisions/0020-an-auditor-reads-it-without-us.md +++ b/docs/decisions/0020-an-auditor-reads-it-without-us.md @@ -1,6 +1,6 @@ # 0020 · An auditor reads it without us -- **Status**: Implemented · revised 2026-09-25 (#902) · open: SPARQL (#308) +- **Status**: Implemented · revised 2026-10-23 (#1345) · open: SPARQL (#308) - **Written**: 2026-09-05 (conventions in the [README](README.md)) - **Related**: [0001](0001-ontology-import-and-governance.md) kept the imported file verbatim and projected what today's consumers can use — this is the first consumer pointing the other way. [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) and [0019](0019-the-second-clock-can-be-rewound.md) are what makes the export worth reading: the intervals and the lineage are the content, not the triples @@ -29,6 +29,23 @@ shapes are not that boundary. MCP may return the same UUIDs alongside its prose so a caller can join an agent result to this export. The user-facing contract is documented in [Agents over MCP](../../web/src/docs/mcp.md#the-external-read-contract). +**Revision 2026-10-23 ([#1345](https://github.com/deeplethe/utopia/issues/1345)).** +`changes` and `entity_facts` accept a zero-based `offset`, defaulting to zero, +and return `offset` and `next_offset` alongside their existing fields. A null +`next_offset` means the page has no successor. Facts are paged after filtering; +their separate names and derived conclusions keep their existing selection. +Changes read one extra event to establish whether another page exists, while +`limit_reached` continues to mean only that the returned page filled its limit. +Both lists break time ties with ledger identities. Offsets describe the result +of each call, not a retained snapshot: callers keep filters and time parameters +unchanged, and a changing live result can move page boundaries. + +`find_entities` also returns `matched_entities` (the partial-name count, or null +when lookup did not count that set) and `truncated`, carrying the existing text +notice into structured output. It remains a ranked candidate lookup; a more +specific name narrows a truncated result. These additions preserve the current +fact fields; a compact fact representation is a separate proposal. + Classes and relations that came in from an import **keep the IRI they came with**. A base built from the schema.org pack exports `schema:Organization`, not a Utopia mint of it, so the file lines up with the vocabulary the reader already has. That is the payoff for having stored `iri` on `entity_types` and `relation_types` since the first import. Everything else is minted under a base namespace: entities, documents, and one IRI per fact. Facts get their own IRI rather than blank nodes because a statement someone may have to cite has to be addressable, and because `supersedes` needs somewhere to point. A class or relation the ontology grew itself is minted from its **key** rather than its uuid — the key is what the base already uses as its identifier (`UNIQUE (kb_id, key)`, and it is what the extraction prompt and the API speak), so the file stays readable; a rename is the one event that moves such an IRI, and renaming a class is rarer than reading the export. diff --git a/web/src/docs/mcp.md b/web/src/docs/mcp.md index d9c203cfb..aeb0e931e 100644 --- a/web/src/docs/mcp.md +++ b/web/src/docs/mcp.md @@ -39,12 +39,12 @@ Three methods are served: |---|---| | `search_chunks` | Full-text + semantic search over the base's documents. Returns the six best-matching passages, each cut at 800 characters and carrying its `document_id`. Pass `as_of` to search the base as it stood at that moment — earlier versions, documents deleted since; full-text recall stays current, so hits are right but may be incomplete | | `get_document` | Read one document by `document_id`, all sections in order, up to 24,000 body characters per call. When truncated, call again with the optional `offset` given in the notice to continue | -| `find_entities` | Entities by (partial) name: id, type, and a disambiguator when several share a name | -| `entity_facts` | One entity's facts with validity ranges. Pass `at` (a date) to see the world as of that day; this is the tool for "who was X in 2024". Pass `as_of` (a date or an RFC3339 moment) to see the facts **as the base held them then**, before later corrections, retractions and merges — "what did we have on record before the memo arrived". The two combine: `at` for the date asked about, `as_of` for when | +| `find_entities` | Entities by (partial) name: id, type, and a disambiguator when several share a name. A truncated partial-name result reports its match count; use a more specific name to narrow it | +| `entity_facts` | One entity's facts with validity ranges, paged with `limit` and `offset`. Pass `at` (a date) to see the world as of that day; this is the tool for "who was X in 2024". Pass `as_of` (a date or an RFC3339 moment) to see the facts **as the base held them then**, before later corrections, retractions and merges — "what did we have on record before the memo arrived". The two combine: `at` for the date asked about, `as_of` for when | | `neighbors` | The entities linked to one entity, one hop, grouped by predicate; narrow with `predicate` or `object_type`. Takes a name as well as an id | | `timeline` | One entity's dated facts in world-time order; `since` / `until` narrow the window | | `paths_between` | The chains of facts joining two entities, up to three hops, shortest first. With `at`, every edge must hold at that moment; with `as_of`, the chains as the base held them then | -| `changes` | What the graph learned or revised in a window of **record** time: asserted, corrected, rejected, merged. Needs no entity; use it when the question names a period, not a subject | +| `changes` | What the graph learned or revised in a window of **record** time: asserted, corrected, rejected, merged. Needs no entity; use it when the question names a period, not a subject. Returns up to 40 events per call; use `offset` to continue | | `search_docs` | Utopia's own manual, for questions about how the platform works. Never the user's documents | | `remember` | Record one sentence into the base's memory. **Needs a `write` token held by an editor**; a token without it does not see this tool in `tools/list`, and calling it anyway says why | @@ -112,10 +112,10 @@ is a successful read, not an error. | Tool | Structured fields | |---|---| -| `find_entities` | `entities[]`, in the same ranked order as the text: `id`, `name`, `type_key`, `type_label`, `disambiguator`, `fact_count`. This is a candidate list, not an exhaustive entity export | +| `find_entities` | `matched_entities`, `truncated`, `entities[]`, in the same ranked order as the text: `id`, `name`, `type_key`, `type_label`, `disambiguator`, `fact_count`. The count is for the partial-name lookup, or `null` when the fallback combined individual words without counting all matches. `truncated` reports the partial-name cap; this is a candidate list, not an exhaustive entity export | | `search_chunks` | `as_of`, `limit`, `limit_reached`, `chunks[]`. Each chunk has `chunk_id`, `document_id`, zero-based `seq`, `filename`, `text`, `truncated`. The text uses the same 800-character cutoff as the prose | -| `entity_facts` | `entity` (`id`, `name`, `type_key`, `type_label`), effective `at`/`as_of`/`before`, `total_facts`, `matched_facts`, `limit`, `truncated`, `facts[]`, `derived_facts[]` | -| `changes` | Effective `since` (inclusive) and `until` (exclusive), `limit`, `limit_reached`, `changes[]`. Events carry `fact_id`, `at`, `kind`, `subject_id`, `subject_name`, `predicate_label`, `object_name`, `object_value`, `confidence`, `valid_from`, `valid_to`, both validity precisions, `document_id`, `filename`, `quote` | +| `entity_facts` | `entity` (`id`, `name`, `type_key`, `type_label`), effective `at`/`as_of`/`before`, `total_facts`, `matched_facts`, `limit`, `offset`, `next_offset`, `truncated`, `facts[]`, `derived_facts[]` | +| `changes` | Effective `since` (inclusive) and `until` (exclusive), `limit`, `offset`, `next_offset`, `limit_reached`, `changes[]`. Events carry `fact_id`, `at`, `kind`, `subject_id`, `subject_name`, `predicate_label`, `object_name`, `object_value`, `confidence`, `valid_from`, `valid_to`, both validity precisions, `document_id`, `filename`, `quote` | An asserted `facts[]` entry contains `id`, `direction`, `predicate_key`, `predicate_label`, `inferred`, `temporal`, `other_id`, `other_name`, `object_value`, @@ -146,12 +146,23 @@ over `as_of` and reports the effective cutoff `T − 1µs`. A `changes[].at` can passed to `before` without losing precision. Events have no separate event UUID; `fact_id` identifies the affected assertion, and can appear in multiple events. -The structure follows the current text selection: assertion filters and the +The structure follows the text selection: assertion filters, `offset`, and the default 80-fact limit (maximum 300) apply to `facts[]`; derived conclusions only -use `at`/`as_of`/`before` and are listed separately without that limit. `truncated` -reports omitted matching assertions. Search returns at most six chunks and -changes at most 40 events; `limit_reached` means the cap was reached, not that -another result is known to exist. Narrow the query/window to inspect more. +use `at`/`as_of`/`before` and are listed separately without that limit. Names and +derived conclusions accompany every page. `truncated` says there are more matching +assertions after this page. + +For `entity_facts` and `changes`, omit `offset` to start at zero. When `next_offset` +is an integer, call the same tool with that value as `offset` and keep the entity, +filters and time parameters unchanged. A null `next_offset` means there are no +more rows. At or past the end, the page is empty. Facts follow their start time +(undated last), record time and fact ID; changes follow record time newest first, +then fact ID and event kind. These offsets do not hold a snapshot between calls: +new records, corrections or a moving "up to now" window can move page boundaries. + +Search returns at most six chunks and changes at most 40 events; `limit_reached` +means the page filled its cap, not that another result is known to exist. Use +`next_offset` to continue changes; narrow the query to inspect more search hits. Historical full-text search retains the recall limitation described above. ## What an agent records waits for a nod From 9ec06a3aa3c0c839f2929cf311a5baebb4c6fd1e Mon Sep 17 00:00:00 2001 From: WaylandYang <145302500+WaylandYang@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:47:51 +0800 Subject: [PATCH 03/10] An attribute declared on a class reaches the typed graph: a late entity gets its bound class, a dated statement reads as its period, a text attribute takes a named object, an unreadable value is a drop row (#1370) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * An attribute declared on a class reaches the typed graph: a late entity gets its bound class, a dated statement reads as its period, a text attribute takes a named object, an unreadable value is a drop row Measured on four company profiles with one class and six attributes, 18 value statements gave 5 typed facts. Four gates, two of them bugs: - A kind word bound while a document was still extracting never reached the entities that document created afterwards: extraction reads the binding table once, and a decision projects itself only when it is written. `type_bindings::apply_bound` now runs at the end of every extraction and at the start of every kind-word alignment run. - A bound value the attribute's datatype could not read was skipped in silence. Materialisation records it as `attr_datatype` in `extraction_drops`, rewritten as a set on every recompute, shown in the Library's drop panel. - A statement dated at a single moment, bound to a state attribute whose binding marks neither end, computed no row (`MOMENT_UNREAD`), so no annual figure ever typed. It now holds over the period the date names, with `end_after_period` (0053 revision 2026-10-22). - An attribute whose value is a name (法定代表人) never had a candidate because extraction lists the name as a thing. A text attribute with a declared domain is now a candidate for an entity-object signature, forward only; the object's name becomes the value (0044 revision 2026-10-22). After the four: 12 typed attribute facts from the same documents with the hand-written ontology, 23 with the enterprise-profile scenario pack (0073, a separate change). Co-Authored-By: Claude Fable 5.1 Signed-off-by: Wayland Yang * The period reading of a single date applies to attributes only; a relation at a single moment with no marks still computes no row CI's `a_moment_marks_its_state` suite showed what the period reading does to a relation: "joined on 2023-06-01" read as a one-day span closes the open works-for row that starts the same day at the next day, through the write gate's "same start, this observation brings an end, that row is open" rule. That is an ending the document never stated. A dated value is a measurement for its period and has no such neighbour, so the reading stays for attributes (the annual figures that motivated it) and the 2026-09-27 rule stands for relations, including a rule that concludes a relation from a single-date statement. The record, the design file and the suite's header say so; the four relation cases pass unchanged. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Wayland Yang * The typed-to-untyped namesake case expects the aligned class the catch-up projection writes Resolution still gives no entity a class: a chosen untyped namesake records the mention's kind word and leaves the class to alignment. With the kind word bound to Person, the projection that runs when the extraction ends now types it `aligned`, where it used to stay empty only because that projection was missing. The case checks the class and its source instead of an empty type. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Wayland Yang --------- Signed-off-by: Wayland Yang Co-authored-by: Wayland Yang Co-authored-by: Claude Fable 5.1 --- crates/utopia-extract/src/phrase_align.rs | 5 +- crates/utopia-server/src/extraction_open.rs | 6 + .../src/namesake_scripted_cases.rs | 24 +- crates/utopia-server/src/phrase_alignment.rs | 17 +- crates/utopia-server/src/type_alignment.rs | 7 + crates/utopia-store/src/extraction_drops.rs | 45 +++ crates/utopia-store/src/materialize.rs | 284 +++++++++++++----- .../src/materialize_period_tests.rs | 156 ++++++++++ .../src/materialize_unreadable_tests.rs | 168 +++++++++++ crates/utopia-store/src/type_bindings.rs | 33 ++ .../src/type_bindings_catch_up_tests.rs | 135 +++++++++ .../tests/store/a_moment_marks_its_state.rs | 3 +- ...ology-is-a-view-over-what-documents-say.md | 4 +- ...cision-records-the-inputs-it-considered.md | 43 ++- docs/design/ontology.md | 21 +- web/src/i18n/en.ts | 1 + web/src/i18n/zh.ts | 1 + 17 files changed, 865 insertions(+), 88 deletions(-) create mode 100644 crates/utopia-store/src/materialize_period_tests.rs create mode 100644 crates/utopia-store/src/materialize_unreadable_tests.rs create mode 100644 crates/utopia-store/src/type_bindings_catch_up_tests.rs diff --git a/crates/utopia-extract/src/phrase_align.rs b/crates/utopia-extract/src/phrase_align.rs index 69eab3ae7..ccabfecc2 100644 --- a/crates/utopia-extract/src/phrase_align.rs +++ b/crates/utopia-extract/src/phrase_align.rs @@ -165,7 +165,10 @@ phrases may be in any language.\n\ 3. Answer null when no candidate fits; when the sentences show the phrase meaning different \ things under this signature; when the phrase only reports, introduces or evaluates (\"said\", \ \"announced\", \"is expected to\") unless a candidate is about that; or when a value is not \ -what the attribute measures (a share count is not revenue, a date is not an amount).\n\ +what the attribute measures (a share count is not revenue, a date is not an amount). A text \ +attribute may be a candidate for an item whose object is a named thing: choose it when the \ +object is named only to carry that attribute's value for the subject (a legal representative's \ +name, a registered address), and the object's name becomes the value.\n\ 4. The direction follows the definition: for \"X —is a subsidiary of→ Y\" subsidiary_of is \ forward; for \"X —owns→ Y\", if subsidiary_of is the only fitting candidate, it is reverse.\n\ 5. Never invent a key, never answer with a label, never choose for an item a key that is not \ diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs index d83faa482..3a8b378a2 100644 --- a/crates/utopia-server/src/extraction_open.rs +++ b/crates/utopia-server/src/extraction_open.rs @@ -1217,6 +1217,12 @@ pub(crate) async fn run_open( serde_json::json!({ "document_id": document_id }), ) .await?; + // 这篇开抽时还没绑上、抽到一半才绑上的类别词：它的类补到这篇后半建的实体上 + // （`bound_types` 是开抽时读的一份，判定落下时的投影看不见之后建的实体） + let caught_up = utopia_store::type_bindings::apply_bound(pool, kb_id).await?; + if caught_up > 0 { + tracing::info!(%document_id, entities = caught_up, "抽取途中绑上的类补到了后建的实体上"); + } // 新出现的类别词绑到类（0044 对齐第一片）：库级，同库排着就不重复 utopia_store::jobs::enqueue_unless_queued( pool, diff --git a/crates/utopia-server/src/namesake_scripted_cases.rs b/crates/utopia-server/src/namesake_scripted_cases.rs index 4fbcdd9b1..b5be0f345 100644 --- a/crates/utopia-server/src/namesake_scripted_cases.rs +++ b/crates/utopia-server/src/namesake_scripted_cases.rs @@ -563,8 +563,11 @@ impl Fx { } _ => {} } + // 类别词绑到的类，TypedToUntyped 收尾时要核对它 + let mut person_class = None; if matches!(guard, TypedToUntyped | UntypedToTyped | IncompatibleType) { let person = Uuid::now_v7(); + person_class = Some(person); sqlx::query( "INSERT INTO entity_types(id,kb_id,key,label) VALUES ($1,$2,'person','Person')", ) @@ -629,17 +632,24 @@ impl Fx { .bind(self.kb) .fetch_one(&self.pool) .await?; - let type_preserved = if guard == TypedToUntyped { - sqlx::query_scalar::<_, bool>("SELECT type_id IS NULL FROM entities WHERE id=$1") - .bind(expected) - .fetch_one(&self.pool) - .await? + // 消解不给实体定类，类由对齐来定：被选中的无类同名实体记下了提及的类别词 + // （`set_specific_type`），「person」又已绑到 Person，抽完那一刻的补投影 + // （`type_bindings::apply_bound`）就把它写成 aligned 的 Person。消解自己写的类 + // 不会是 aligned；从前它留着空类，是补投影缺席的结果，不是规矩 + let type_from_alignment = if guard == TypedToUntyped { + sqlx::query_scalar::<_, bool>( + "SELECT type_source = 'aligned' AND type_id = $2 FROM entities WHERE id=$1", + ) + .bind(expected) + .bind(person_class) + .fetch_one(&self.pool) + .await? } else { true }; Ok(( - owners == vec![expected] && count == 2 && reviews == 0 && type_preserved, - json!({"expected_owner":expected,"owners":owners,"entities":count,"pending_reviews":reviews,"type_preserved":type_preserved}), + owners == vec![expected] && count == 2 && reviews == 0 && type_from_alignment, + json!({"expected_owner":expected,"owners":owners,"entities":count,"pending_reviews":reviews,"type_from_alignment":type_from_alignment}), )) } diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rs index 6b018a90b..c2fe8012b 100644 --- a/crates/utopia-server/src/phrase_alignment.rs +++ b/crates/utopia-server/src/phrase_alignment.rs @@ -132,6 +132,20 @@ fn fits_direction( via.extend(within(&p.domains, sig.subject_type_id, closure)?); return Some(Fit { via }); } + if p.kind == "attribute" { + // 宾语是实体的签名也能绑文本属性（0044 修订 2026-10-22）：法定代表人、注册地址的值 + // 是个名字，抽取把它列成了东西；只认正向，名字就是值。数、日期、真假不是名字。 + // 没声明定义域的不算（`known_as` 这类）：没声明是「不限」，它会出现在每一条关系签名 + // 的候选里，同 NVDA 上泛泛的 value 属性吞掉现金流量表每一行的教训 + if p.datatype.as_deref() != Some("text") + || direction != Direction::Forward + || p.domains.is_empty() + { + return None; + } + via.extend(within(&p.domains, sig.subject_type_id, closure)?); + return Some(Fit { via }); + } if p.kind != "relation" { return None; } @@ -522,12 +536,13 @@ async fn shortlist( continue; }; let (fitting, _) = &full[&s.key()]; + // 字面值的签名只绑属性；宾语是实体的两种都可能（关系，或文本属性），不按类筛 let near = utopia_store::ontology::nearest_relation_type_ids( pool, kb_id, vector, (fitting.len() * 2) as i64, - Some(property_kind(s)), + s.object_is_value.then_some("attribute"), ) .await?; if near.is_empty() { diff --git a/crates/utopia-server/src/type_alignment.rs b/crates/utopia-server/src/type_alignment.rs index 3f0a5c6e3..88ea0e307 100644 --- a/crates/utopia-server/src/type_alignment.rs +++ b/crates/utopia-server/src/type_alignment.rs @@ -169,6 +169,13 @@ async fn align_types_locked( client: &utopia_llm::LlmClient, ) -> anyhow::Result<()> { let pool = &state.pool; + // 先把已绑上的类补到它的类别词下还没有类的实体上：判定落下时的投影只看得见那一刻 + // 的实体，之后抽出来的（判定期间还在抽的那几篇）落成没有类的；这里每轮补一次， + // 短语对齐排在这之后，签名两端的类才是齐的 + let caught_up = type_bindings::apply_bound(pool, kb_id).await?; + if caught_up > 0 { + tracing::info!(%kb_id, entities = caught_up, "已绑上的类补到了后建的实体上"); + } // 调模型之前在一个快照里读齐输入（#795）：类的定义、类的版本与父边、类别词、已有的 // 判定。分几次读的话，两次读之间提交的编辑会让提示词里的定义和指纹里的版本对不上 let (classes, snapshot, sigs, existing) = { diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rs index de55ec1ce..55cc82971 100644 --- a/crates/utopia-store/src/extraction_drops.rs +++ b/crates/utopia-store/src/extraction_drops.rs @@ -61,6 +61,51 @@ pub mod reason { /// 宾语其实是个值（一个量、一个日期、表格的一格）：挪进值那一格，不建实体 /// （`open::values_are_not_things`）。数的是规则 2 和表格规则还有多少次没管住模型 pub const OBJECT_IS_VALUE: &str = "object_is_value"; + /// 陈述绑到了一条属性，值却按属性声明的 datatype 与单位读不出来（「超过2000人」读不成 + /// 数，「5000万美元」换不成万元）：陈述留在开放图谱，不落类型化行（`materialize`）。 + /// 这一条由物化发出，不是抽取；每轮物化按库重写，detail 是属性的 key + pub const ATTR_DATATYPE: &str = "attr_datatype"; +} + +/// 一轮物化读不出的值：(文档, 属性 key, 一个样例)。 +pub struct Unreadable { + pub document_id: Uuid, + pub detail: String, + pub example: String, +} + +/// 把一个库里某个原因码的行整个换成这一批（事务内）。物化是集合运算、每轮重算， +/// 它发出的信号也按集合写：删掉这个原因码在这个库下的全部行，再按（文档, detail） +/// 数一遍写回。用 `record` 的话每轮重算都会把同一条陈述再数一次 +pub async fn replace_reason_tx( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + kb_id: Uuid, + reason: &str, + rows: &[Unreadable], +) -> AppResult<()> { + // 整库的语句不缓存（#1298）：缓存几次后换成通用计划，看不见这个库有多少行 + sqlx::query("DELETE FROM extraction_drops WHERE kb_id = $1 AND reason = $2") + .persistent(false) + .bind(kb_id) + .bind(reason) + .execute(&mut **tx) + .await?; + for r in rows { + sqlx::query( + "INSERT INTO extraction_drops (kb_id, document_id, reason, detail, example) + VALUES ($1, $2, $3, left($4, 120), left($5, 200)) + ON CONFLICT (kb_id, document_id, reason, detail) + DO UPDATE SET count = extraction_drops.count + 1, updated_at = now()", + ) + .bind(kb_id) + .bind(r.document_id) + .bind(reason) + .bind(&r.detail) + .bind(&r.example) + .execute(&mut **tx) + .await?; + } + Ok(()) } pub async fn record( diff --git a/crates/utopia-store/src/materialize.rs b/crates/utopia-store/src/materialize.rs index 2383072be..c2d263cee 100644 --- a/crates/utopia-store/src/materialize.rs +++ b/crates/utopia-store/src/materialize.rs @@ -33,18 +33,11 @@ const MATCH: &str = "b.kb_id = s.kb_id AND b.object_is_value = (s.object_id IS NULL) AND (s.object_id IS NULL OR b.object_type_id IS NOT DISTINCT FROM oe.type_id)"; -/// 一条只说了一刻的陈述（起止同值）绑在一条状态属性上，而绑定没说那一刻是开始还是 -/// 结束（`marks` 是 none 或未知，#966）：它不算类型化行，留在开放图谱里带着它的日期。 -/// 照抄起止会写成一段任何时刻都不成立的状态。`s` 是陈述，`b` 是绑定。 -/// -/// 每一项都得是真值、不能是 NULL：它在 `NOT` 后面用，`valid_to = valid_from` 在只有起点的 -/// 陈述上是 NULL，`NOT NULL` 还是 NULL，「自那天起」的陈述就整批不算了 -const MOMENT_UNREAD: &str = - "(s.valid_from IS NOT NULL AND s.valid_to IS NOT DISTINCT FROM s.valid_from - AND EXISTS (SELECT 1 FROM relation_types rt - WHERE rt.id = b.relation_type_id AND rt.temporal = 'state') - AND b.marks IS DISTINCT FROM 'start' AND b.marks IS DISTINCT FROM 'end')"; - +/// 一条只说了一刻的陈述（起止同值）绑在一条状态属性上怎么读，见 [`span_of`]：绑定说 +/// `start` 从那一刻起，说 `end` 到那一刻为止，说 `none` 或没说按那个日期命名的时段读 +/// （0053 修订 2026-10-22）。从前 `none` 和没说的不算类型化行（`MOMENT_UNREAD`），四篇公司 +/// 简介上每一条年度数字都这么丢了；照抄起止会写成一段任何时刻都不成立的状态，所以终点 +/// 挪到桶的尽头、标上 `end_after_period`，读的人印得出原文写的 2023 /// 属性的 datatype 与单位，值按它们读（[`attribute_value`]）；`rt` 是绑到或蕴含的那条属性， /// 是关系时 datatype 为 NULL，值原样写 const SHAPE: &str = "CASE WHEN rt.kind = 'attribute' THEN rt.datatype END AS datatype, rt.unit"; @@ -74,6 +67,8 @@ struct Due { direction: String, subject_id: Uuid, object_id: Option, + /// 宾语是实体时它的名字：绑到文本属性上，名字就是值（0044 修订 2026-10-22） + object_name: Option, object_value: Option, valid_from: Option>, valid_from_precision: Option, @@ -86,6 +81,8 @@ struct Due { marks: Option, /// 绑到的属性的时间语义 temporal: Option, + /// 绑到的是关系还是属性 + kind: String, /// 绑到的是属性时它的 datatype 与单位，值按它们读（[`attribute_value`]） datatype: Option, unit: Option, @@ -212,8 +209,7 @@ async fn materialize_in_tx( AND ((b.direction = 'forward' AND t.subject_id = s.subject_id) OR (b.direction = 'reverse' AND t.subject_id = s.object_id)) AND NOT EXISTS (SELECT 1 FROM statement_qualifiers q - WHERE q.fact_id = s.id AND q.role = 'mood') - AND NOT {MOMENT_UNREAD})" + WHERE q.fact_id = s.id AND q.role = 'mood'))" )) .persistent(false) .bind(kb_id) @@ -265,11 +261,11 @@ async fn materialize_in_tx( // 且谓词相同。reverse 只对两样东西之间的关系有意义（字面值当不了主语） let due: Vec = sqlx::query_as(&format!( "SELECT s.id AS statement, b.relation_type_id AS property, b.direction, - s.subject_id, s.object_id, s.object_value, + s.subject_id, s.object_id, oe.canonical_name AS object_name, s.object_value, s.valid_from, s.valid_from_precision, s.valid_from_grade, s.valid_to, s.valid_to_precision, s.attested_from, s.confidence, b.marks, - rt.temporal, {SHAPE} + rt.temporal, rt.kind, {SHAPE} FROM facts s JOIN entities se ON se.id = s.subject_id LEFT JOIN entities oe ON oe.id = s.object_id @@ -278,7 +274,6 @@ async fn materialize_in_tx( WHERE s.kb_id = $1 AND s.layer = 'open' AND s.invalidated_at IS NULL AND b.status = 'bound' AND (b.direction = 'forward' OR s.object_id IS NOT NULL) - AND NOT {MOMENT_UNREAD} AND NOT EXISTS (SELECT 1 FROM statement_qualifiers q WHERE q.fact_id = s.id AND q.role = 'mood') -- 勘误撤过的（陈述, 属性）不再算（0044 决定 7）：撤销要站得住 @@ -296,46 +291,41 @@ async fn materialize_in_tx( .await?; let (mut added, mut merged) = (0u64, 0u64); + // 绑上了却按 datatype 读不出的值：(陈述, 属性, 原文的字)，收尾时记成丢弃行 + let mut unreadable: Vec<(Uuid, Uuid, String)> = Vec::new(); for d in &due { let reverse = d.direction == "reverse"; - // 一刻的陈述在状态属性下按绑定说的读（#966）：开始写成从那一刻起，结束写成到那一刻 - // 为止（走 `insert_fact_on`「某天结束了」那条路，关上开放的那段）。什么都不算的 - // 在查询里已经排掉了（[`MOMENT_UNREAD`]）。陈述不带 `end_after_period`：终点挪没挪到 - // 时段的尽头由写入路径定（#994 第一刀） - let moment = d.temporal.as_deref() == Some("state") - && d.valid_from.is_some() - && d.valid_from == d.valid_to; - let validity = match (moment, d.marks.as_deref()) { - (false, _) => Validity { - from: d.valid_from, - from_precision: d.valid_from_precision.as_deref(), - from_grade: d.valid_from_grade.as_deref(), - to: d.valid_to, - to_precision: d.valid_to_precision.as_deref(), - attested_at: d.attested_from, - end_after_period: false, - }, - (true, Some("start")) => Validity { - from: d.valid_from, - from_precision: d.valid_from_precision.as_deref(), - from_grade: d.valid_from_grade.as_deref(), - to: None, - to_precision: None, - attested_at: d.attested_from, - end_after_period: false, - }, - (true, Some("end")) => Validity { - from: None, - from_precision: None, - from_grade: None, - to: d.valid_from, - to_precision: d.valid_from_precision.as_deref(), - attested_at: d.attested_from, - end_after_period: false, - }, - (true, _) => continue, + // 一刻的陈述在状态属性下按绑定说的读（#966、0053 修订 2026-10-22）：开始写成从那一刻 + // 起，结束写成到那一刻为止（走 `insert_fact_on`「某天结束了」那条路，关上开放的那段）， + // 都不是或没说：属性按那个日期命名的时段读，关系不写（见 `span_of`） + let Some(validity) = span_of( + d.kind == "attribute", + d.temporal.as_deref() == Some("state"), + d.marks.as_deref(), + d.valid_from, + d.valid_from_precision.as_deref(), + d.valid_from_grade.as_deref(), + d.valid_to, + d.valid_to_precision.as_deref(), + d.attested_from, + ) else { + continue; + }; + // 文本属性绑上了一条宾语是实体的陈述：宾语的名字就是值（0044 修订 2026-10-22） + let named_value = (d.kind == "attribute") + .then(|| { + d.object_name + .as_deref() + .map(|n| serde_json::json!({ "value": n })) + }) + .flatten(); + let object_value = named_value.as_ref().or(d.object_value.as_ref()); + let object_id = if d.kind == "attribute" { + None + } else { + d.object_id }; - let (fact, new) = match (reverse, d.object_id, &d.object_value) { + let (fact, new) = match (reverse, object_id, object_value) { (true, Some(object), _) => { insert_fact_on( tx, @@ -361,9 +351,11 @@ async fn materialize_in_tx( .await? } (false, None, Some(value)) => { - // 读不成属性的 datatype：陈述留在开放图谱里，这一对不写 + // 读不成属性的 datatype：陈述留在开放图谱里，这一对不写——但要记一行 + // （`attr_datatype`），从前这里什么都不说，图里少了一个数没人说得出为什么 let Some(value) = attribute_value(value, d.datatype.as_deref(), d.unit.as_deref()) else { + unreadable.push((d.statement, d.property, raw_text(value))); continue; }; insert_fact_on( @@ -443,7 +435,8 @@ async fn materialize_in_tx( } // 3a. 已批准的规则算隐含行（0044 决定 3 第五片）。读数只查缓存：缓存里没有的这一轮 // 不算，`read_phrases` 填上之后再来。短语规则按陈述触发，类别词规则按实体触发 - let implied = imply_in_tx(tx, kb_id, &mut written).await?; + let implied = imply_in_tx(tx, kb_id, &mut written, &mut unreadable).await?; + record_unreadable(tx, kb_id, &unreadable).await?; // 3b. 见证跟着来源陈述走（0064 决定 3、5）：类型化的行自己没有出处。物化与算隐含行 // 用的写入口在陈述没有见证时填的是此刻（那是给人写的事实用的缺省），这里改回陈述的。 // 排在 3a 之后：规则算出的行也是这样写下的 @@ -531,6 +524,10 @@ struct Implied { valid_to_precision: Option, attested_from: Option>, confidence: f32, + /// 结论属性的时间语义：一刻的陈述落到状态属性上按时段读（规则没有 marks） + temporal: String, + /// 结论是属性还是关系：一刻的陈述只在属性上读成时段 + kind: String, datatype: Option, unit: Option, } @@ -539,6 +536,7 @@ async fn imply_in_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, kb_id: Uuid, written: &mut Vec, + unreadable: &mut Vec<(Uuid, Uuid, String)>, ) -> AppResult { // 短语规则：签名下活着的、没 mood 的陈述；宾语是读数的答案（缓存里的实体或值）， // 没有读数时就是陈述的宾语。已经有活着的隐含行以这条陈述为来源的不再算 @@ -548,7 +546,7 @@ async fn imply_in_tx( CASE WHEN r.reading IS NULL THEN s.object_id ELSE pr.entity_id END AS object_id, CASE WHEN r.reading IS NULL THEN s.object_value ELSE pr.value END AS object_value, s.valid_from, s.valid_from_precision, s.valid_from_grade, s.valid_to, s.valid_to_precision, - s.attested_from, s.confidence, {SHAPE} + s.attested_from, s.confidence, rt.temporal, rt.kind, {SHAPE} FROM implication_rules r JOIN relation_types rt ON rt.id = r.conclude_property_id JOIN facts s ON s.kb_id = r.kb_id AND s.layer = 'open' AND s.invalidated_at IS NULL @@ -560,10 +558,6 @@ async fn imply_in_tx( AND {RULE_MATCH} AND NOT EXISTS (SELECT 1 FROM statement_qualifiers q WHERE q.fact_id = s.id AND q.role = 'mood') AND (r.reading IS NULL OR pr.entity_id IS NOT NULL OR pr.value IS NOT NULL) - -- 规则没有 marks（#966）：一刻的陈述蕴含一条状态属性时，读法未知，不算隐含行。 - -- 用 IS NOT DISTINCT FROM：只有起点的陈述上 `=` 是 NULL，NOT 之后照样把它排掉 - AND NOT (s.valid_from IS NOT NULL AND s.valid_to IS NOT DISTINCT FROM s.valid_from - AND rt.temporal = 'state') AND NOT EXISTS (SELECT 1 FROM errata_actions ea WHERE ea.statement_id = s.id AND ea.predicate_id = r.conclude_property_id AND ea.status = 'applied' AND ea.action IN ('retract', 'revise')) @@ -584,7 +578,7 @@ async fn imply_in_tx( pr.entity_id AS object_id, pr.value AS object_value, NULL::timestamptz AS valid_from, NULL::text AS valid_from_precision, NULL::text AS valid_from_grade, NULL::timestamptz AS valid_to, NULL::text AS valid_to_precision, - NULL::timestamptz AS attested_from, 0.9::real AS confidence, {SHAPE} + NULL::timestamptz AS attested_from, 0.9::real AS confidence, rt.temporal, rt.kind, {SHAPE} FROM implication_rules r JOIN relation_types rt ON rt.id = r.conclude_property_id JOIN entities e ON e.kb_id = r.kb_id AND e.merged_into IS NULL AND {KIND} = r.phrase @@ -603,23 +597,34 @@ async fn imply_in_tx( let mut implied = 0u64; for d in by_statement.iter().chain(by_entity.iter()) { - let validity = Validity { - from: d.valid_from, - from_precision: d.valid_from_precision.as_deref(), - from_grade: d.valid_from_grade.as_deref(), - to: d.valid_to, - to_precision: d.valid_to_precision.as_deref(), - attested_at: d.attested_from, - end_after_period: false, + // 规则没有 marks（#966）：一刻的陈述蕴含一条状态属性时，结论是属性的按那个日期命名 + // 的时段读，是关系的不算隐含行（0053 修订 2026-10-22，同物化） + let Some(validity) = span_of( + d.kind == "attribute", + d.temporal == "state", + None, + d.valid_from, + d.valid_from_precision.as_deref(), + d.valid_from_grade.as_deref(), + d.valid_to, + d.valid_to_precision.as_deref(), + d.attested_from, + ) else { + continue; }; let value; let object = match (d.object_id, &d.object_value) { (Some(o), _) => FactObject::Entity(o), (None, Some(v)) => { - let Some(v) = attribute_value(v, d.datatype.as_deref(), d.unit.as_deref()) else { + let Some(read) = attribute_value(v, d.datatype.as_deref(), d.unit.as_deref()) + else { + // 类别词规则的来源是实体，没有文档可记；短语规则的记在触发它的陈述上 + if let Some(statement) = d.statement { + unreadable.push((statement, d.property, raw_text(v))); + } continue; }; - value = v; + value = read; FactObject::Value(&value) } _ => continue, @@ -676,6 +681,133 @@ async fn imply_in_tx( Ok(implied) } +/// 一条陈述落成类型化行时的区间。起止不同的照抄；起止同值的一刻落到状态属性上按绑定说的读 +/// （#966）：`start` 从那一刻起，`end` 到那一刻为止（走「某天结束了」那条路）。`none` 或没说： +/// 落到**属性**上按那个日期命名的时段读（0053 修订 2026-10-22）——「2023年营业收入58亿元」是 +/// 2023 这一年的一个量，起点是它，终点是桶的尽头，标上 `end_after_period`，读的人印得出原文 +/// 写的 2023；落到关系上照旧不算行——「joined on 2023-06-01」读成一天的任职会把同一天起的 +/// 开放行关在次日（写入口「同起点、这次带了终点、那行还开着就关上」），那是一条错的结束。 +/// 没有精度的一刻不是桶，读不出时段，也返回 None +#[allow(clippy::too_many_arguments)] +fn span_of<'a>( + attribute: bool, + state: bool, + marks: Option<&str>, + from: Option>, + from_precision: Option<&'a str>, + from_grade: Option<&'a str>, + to: Option>, + to_precision: Option<&'a str>, + attested_at: Option>, +) -> Option> { + let moment = state && from.is_some() && from == to; + Some(match (moment, marks) { + (false, _) => Validity { + from, + from_precision, + from_grade, + to, + to_precision, + attested_at, + end_after_period: false, + }, + (true, Some("start")) => Validity { + from, + from_precision, + from_grade, + to: None, + to_precision: None, + attested_at, + end_after_period: false, + }, + (true, Some("end")) => Validity { + from: None, + from_precision: None, + from_grade: None, + to: from, + to_precision: from_precision, + attested_at, + end_after_period: false, + }, + (true, _) => { + if !attribute { + return None; + } + let start = from?; + let end = crate::graph::bucket_end(start, from_precision); + if end <= start { + return None; + } + Validity { + from, + from_precision, + from_grade, + to: Some(end), + to_precision: from_precision, + attested_at, + end_after_period: true, + } + } + }) +} + +/// 值那一格原文的字，给丢弃行当样例：`{"value": "超过2000人"}` → `超过2000人` +fn raw_text(raw: &Value) -> String { + match raw.get("value").unwrap_or(raw) { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +/// 读不出的值记成 `attr_datatype` 丢弃行（0001：丢弃是一行，不是沉默）。按库整个重写： +/// 物化每轮重算，上一轮读不出、这一轮属性改成文本读出来了的，行跟着退掉。文档从陈述的 +/// 证据取；没有证据的陈述（人写的）没有文档可挂，不记 +async fn record_unreadable( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + kb_id: Uuid, + unreadable: &[(Uuid, Uuid, String)], +) -> AppResult<()> { + let statements: Vec = unreadable.iter().map(|u| u.0).collect(); + let properties: Vec = unreadable.iter().map(|u| u.1).collect(); + let raws: Vec = unreadable.iter().map(|u| u.2.clone()).collect(); + let rows: Vec = + sqlx::query_as::<_, (Uuid, String, String)>( + "SELECT fe.document_id, rt.key, se.canonical_name || ' → ' || x.raw + FROM unnest($2::uuid[], $3::uuid[], $4::text[]) AS x(statement, property, raw) + JOIN relation_types rt ON rt.id = x.property + JOIN facts s ON s.id = x.statement + JOIN entities se ON se.id = s.subject_id + JOIN LATERAL (SELECT fe.document_id FROM fact_evidence fe + WHERE fe.fact_id = x.statement AND fe.document_id IS NOT NULL + ORDER BY fe.quote_start NULLS LAST LIMIT 1) fe ON true + WHERE rt.kb_id = $1 + ORDER BY x.statement", + ) + .persistent(false) + .bind(kb_id) + .bind(&statements) + .bind(&properties) + .bind(&raws) + .fetch_all(&mut **tx) + .await? + .into_iter() + .map( + |(document_id, detail, example)| crate::extraction_drops::Unreadable { + document_id, + detail, + example, + }, + ) + .collect(); + crate::extraction_drops::replace_reason_tx( + tx, + kb_id, + crate::extraction_drops::reason::ATTR_DATATYPE, + &rows, + ) + .await +} + /// 开放陈述的值（`{"value": "2.3亿元"}`）落到属性上的样子：`value` 那一格按 datatype 与单位读成 /// 数、日期、真假（[`utopia_extract::attribute_value`]），其余键照抄；绑到关系上的值原样留着。 /// @@ -721,3 +853,11 @@ mod delivery_tests; #[cfg(test)] #[path = "materialize_plan_tests.rs"] mod plan_tests; + +#[cfg(test)] +#[path = "materialize_unreadable_tests.rs"] +mod unreadable_tests; + +#[cfg(test)] +#[path = "materialize_period_tests.rs"] +mod period_tests; diff --git a/crates/utopia-store/src/materialize_period_tests.rs b/crates/utopia-store/src/materialize_period_tests.rs new file mode 100644 index 000000000..0b4fbf8f5 --- /dev/null +++ b/crates/utopia-store/src/materialize_period_tests.rs @@ -0,0 +1,156 @@ +//! 一刻的陈述落到状态属性上按绑定说的读（0053 修订 2026-10-22）：`none` 或没说按那个日期 +//! 命名的时段写成一段（起点是它，终点是桶的尽头，标 `end_after_period`），`start` 照旧开着。 +//! 没有 `UTOPIA_DATABASE_URL` 时跳过。 + +use super::materialize_in_tx; +use crate::phrase_bindings; +use chrono::{DateTime, Utc}; +use serde_json::json; +use sqlx::PgPool; +use uuid::Uuid; + +struct Fx { + org: Uuid, + kb: Uuid, + property: Uuid, + sig: phrase_bindings::PhraseSignature, +} + +async fn seed(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb, company, property, statement) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'period')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'period')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'period')") + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + sqlx::query("INSERT INTO entities(id,kb_id,canonical_name) VALUES($1,$2,'启明半导体')") + .bind(company) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO relation_types(id,kb_id,key,label,temporal,kind,datatype,unit) VALUES($1,$2,'revenue','营业收入','state','attribute','number','亿元')", + ) + .bind(property) + .bind(kb) + .execute(pool) + .await?; + // 「2023年营业收入58亿元」：时间解析把一年写成起止同值、精度 year 的一刻 + let year: DateTime = "2023-01-01T00:00:00Z".parse()?; + sqlx::query( + "INSERT INTO facts(id,kb_id,subject_id,object_value,layer,phrase,valid_from,valid_from_precision,valid_to,valid_to_precision) + VALUES($1,$2,$3,$4,'open','营业收入',$5,'year',$5,'year')", + ) + .bind(statement) + .bind(kb) + .bind(company) + .bind(json!({ "value": "58亿元" })) + .bind(year) + .execute(pool) + .await?; + let sig = phrase_bindings::signatures(pool, kb).await?.remove(0); + Ok(Fx { + org, + kb, + property, + sig, + }) +} + +async fn bind(pool: &PgPool, f: &Fx, marks: Option<&str>) -> anyhow::Result<()> { + let mut tx = pool.begin().await?; + phrase_bindings::decide_on( + &mut tx, + f.kb, + &f.sig, + phrase_bindings::Decision { + relation_type_id: Some(f.property), + direction: Some("forward"), + status: "bound", + votes: &serde_json::Value::Null, + decided_by: "person", + basis: None, + marks, + marks_asked: true, + }, + ) + .await?; + tx.commit().await?; + let mut tx = pool.begin().await?; + materialize_in_tx(&mut tx, f.kb).await?; + tx.commit().await?; + Ok(()) +} + +type Row = ( + serde_json::Value, + Option>, + Option, + Option>, + Option, + bool, +); + +async fn typed(pool: &PgPool, kb: Uuid) -> anyhow::Result> { + Ok(sqlx::query_as( + "SELECT object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, end_after_period + FROM facts WHERE kb_id=$1 AND layer='typed' AND invalidated_at IS NULL", + ) + .bind(kb) + .fetch_all(pool) + .await?) +} + +#[tokio::test] +async fn a_single_date_that_marks_neither_end_names_its_period() -> anyhow::Result<()> { + let Some(url) = crate::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + crate::db::migrate(&pool).await?; + let f = seed(&pool).await?; + let run = async { + let start: DateTime = "2023-01-01T00:00:00Z".parse()?; + let end: DateTime = "2024-01-01T00:00:00Z".parse()?; + // 绑定没说一刻标哪一端：按 2023 这一年读 + bind(&pool, &f, None).await?; + let rows = typed(&pool, f.kb).await?; + assert_eq!(rows.len(), 1, "{rows:?}"); + let (value, from, from_p, to, to_p, after) = &rows[0]; + assert_eq!(value["value"], json!(58.0)); + assert_eq!((*from, from_p.as_deref()), (Some(start), Some("year"))); + assert_eq!((*to, to_p.as_deref()), (Some(end), Some("year"))); + assert!( + after, + "the end is the moment after the period the document wrote" + ); + // 「都不是」是同一种读法；重算是集合运算，还是那一行 + bind(&pool, &f, Some("none")).await?; + let again = typed(&pool, f.kb).await?; + assert_eq!(again.len(), 1); + assert_eq!(again[0].3, Some(end)); + anyhow::Ok(()) + } + .await; + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(f.org) + .execute(&pool) + .await?; + run +} diff --git a/crates/utopia-store/src/materialize_unreadable_tests.rs b/crates/utopia-store/src/materialize_unreadable_tests.rs new file mode 100644 index 000000000..f64177cb9 --- /dev/null +++ b/crates/utopia-store/src/materialize_unreadable_tests.rs @@ -0,0 +1,168 @@ +//! 绑上了却按 datatype 读不出的值记成 `attr_datatype` 丢弃行，而不是无声地不写； +//! 属性改成读得出之后那一行退掉、类型化行写下。没有 `UTOPIA_DATABASE_URL` 时跳过。 + +use super::materialize_in_tx; +use crate::{extraction_drops, phrase_bindings}; +use serde_json::json; +use sqlx::PgPool; +use uuid::Uuid; + +struct Fx { + org: Uuid, + kb: Uuid, + document: Uuid, + property: Uuid, +} + +async fn seed(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb, document, chunk, company, property, statement) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'unreadable')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'unreadable')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'unreadable')") + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO documents(id,kb_id,filename,sha256,status,graph_status) VALUES($1,$2,'启明.txt','x','ready','done')", + ) + .bind(document) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES($1,$2,$3,0,'员工人数超过2000人')", + ) + .bind(chunk) + .bind(kb) + .bind(document) + .execute(pool) + .await?; + sqlx::query("INSERT INTO entities(id,kb_id,canonical_name) VALUES($1,$2,'启明半导体')") + .bind(company) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO relation_types(id,kb_id,key,label,temporal,kind,datatype) VALUES($1,$2,'employee_count','员工人数','state','attribute','number')", + ) + .bind(property) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO facts(id,kb_id,subject_id,object_value,layer,phrase) VALUES($1,$2,$3,$4,'open','员工人数')", + ) + .bind(statement) + .bind(kb) + .bind(company) + .bind(json!({ "value": "超过2000人" })) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO fact_evidence(fact_id,chunk_id,document_id,quote) VALUES($1,$2,$3,'员工人数超过2000人')", + ) + .bind(statement) + .bind(chunk) + .bind(document) + .execute(pool) + .await?; + let sig = phrase_bindings::signatures(pool, kb).await?.remove(0); + let mut tx = pool.begin().await?; + phrase_bindings::decide_on( + &mut tx, + kb, + &sig, + phrase_bindings::Decision { + relation_type_id: Some(property), + direction: Some("forward"), + status: "bound", + votes: &serde_json::Value::Null, + decided_by: "person", + basis: None, + marks: None, + marks_asked: false, + }, + ) + .await?; + tx.commit().await?; + Ok(Fx { + org, + kb, + document, + property, + }) +} + +async fn recompute(pool: &PgPool, kb: Uuid) -> anyhow::Result<()> { + let mut tx = pool.begin().await?; + materialize_in_tx(&mut tx, kb).await?; + tx.commit().await?; + Ok(()) +} + +async fn typed_rows(pool: &PgPool, kb: Uuid) -> anyhow::Result { + Ok(sqlx::query_scalar( + "SELECT count(*) FROM facts WHERE kb_id=$1 AND layer='typed' AND invalidated_at IS NULL", + ) + .bind(kb) + .fetch_one(pool) + .await?) +} + +#[tokio::test] +async fn a_value_the_datatype_cannot_read_is_a_drop_row_until_it_can() -> anyhow::Result<()> { + let Some(url) = crate::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + crate::db::migrate(&pool).await?; + let f = seed(&pool).await?; + let run = async { + recompute(&pool, f.kb).await?; + assert_eq!(typed_rows(&pool, f.kb).await?, 0); + let drops = extraction_drops::for_kb(&pool, f.kb).await?; + assert_eq!(drops.len(), 1, "{drops:?}"); + assert_eq!(drops[0].reason, extraction_drops::reason::ATTR_DATATYPE); + assert_eq!(drops[0].document_id, f.document); + assert_eq!(drops[0].detail, "employee_count"); + assert_eq!(drops[0].count, 1); + assert_eq!(drops[0].example.as_deref(), Some("启明半导体 → 超过2000人")); + // 再算一遍不翻倍：按集合重写 + recompute(&pool, f.kb).await?; + let drops = extraction_drops::for_kb(&pool, f.kb).await?; + assert_eq!(drops.len(), 1); + assert_eq!(drops[0].count, 1); + // 属性改成文本：值读得出了，行写下，丢弃退掉 + sqlx::query("UPDATE relation_types SET datatype='text' WHERE id=$1") + .bind(f.property) + .execute(&pool) + .await?; + recompute(&pool, f.kb).await?; + assert_eq!(typed_rows(&pool, f.kb).await?, 1); + assert!(extraction_drops::for_kb(&pool, f.kb).await?.is_empty()); + anyhow::Ok(()) + } + .await; + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(f.org) + .execute(&pool) + .await?; + run +} diff --git a/crates/utopia-store/src/type_bindings.rs b/crates/utopia-store/src/type_bindings.rs index 0c5e8b794..0616e5a49 100644 --- a/crates/utopia-store/src/type_bindings.rs +++ b/crates/utopia-store/src/type_bindings.rs @@ -597,6 +597,35 @@ pub async fn unapply<'e>( Ok(res.rows_affected()) } +/// 把库里每条绑上的类补到它的类别词下还不在那个类上的实体上。返回改动数。 +/// +/// [`apply`] 只在判定落下那一刻投影一次，之后建的实体它看不见。抽取开抽时读一次 +/// 绑定表（`bound_map`），一篇文档抽到一半时「公司」才绑上企业类，这篇后半建的公司 +/// 就落成没有类的实体；几篇并发上传时第一篇的判定对其余几篇都是「抽到一半」。没有类 +/// 的主语进不了任何声明了定义域的属性的候选，它的属性陈述全判成「无」——四篇测试 +/// 文档里表格那篇的六条注册资本、成立时间、员工人数就是这么丢的（2026-10-22）。 +/// +/// 每篇抽完、每轮类别词对齐开跑前各补一次。幂等：已在那个类上的不算改动。与 [`apply`] +/// 同一条规矩：人定过类的不动（`type_source = 'human'`），`proposed_type` 一并清掉 +pub async fn apply_bound<'e>( + pool: impl Executor<'e, Database = Postgres>, + kb_id: Uuid, +) -> AppResult { + let sql = format!( + "UPDATE entities e + SET type_id = b.type_id, type_source = 'aligned', proposed_type = NULL, updated_at = now() + FROM type_bindings b + WHERE e.kb_id = $1 AND b.kb_id = e.kb_id + AND b.status = 'bound' AND b.type_id IS NOT NULL + AND e.merged_into IS NULL AND e.type_source <> 'human' + AND e.specific_type IS NOT NULL AND {kind} = b.kind_word + AND e.type_id IS DISTINCT FROM b.type_id", + kind = kind_word_sql("e.specific_type") + ); + let res = sqlx::query(&sql).bind(kb_id).execute(pool).await?; + Ok(res.rows_affected()) +} + /// 抽取时用的绑定表：归一过的类别词 → 类 id，只有绑上的。 pub async fn bound_map(pool: &PgPool, kb_id: Uuid) -> AppResult> { let rows: Vec<(String, Uuid)> = sqlx::query_as( @@ -642,6 +671,10 @@ pub async fn propose( Ok(res.rows_affected()) } +#[cfg(test)] +#[path = "type_bindings_catch_up_tests.rs"] +mod catch_up_tests; + #[cfg(test)] mod tests { use super::{normalize, ClassSnapshot}; diff --git a/crates/utopia-store/src/type_bindings_catch_up_tests.rs b/crates/utopia-store/src/type_bindings_catch_up_tests.rs new file mode 100644 index 000000000..1db8b7515 --- /dev/null +++ b/crates/utopia-store/src/type_bindings_catch_up_tests.rs @@ -0,0 +1,135 @@ +//! 判定落下之后建的实体也拿到它类别词绑上的类（`apply_bound`）。 +//! +//! 判定那一刻的投影（`apply`）只看得见那一刻的实体；抽到一半的文档后半建的公司落成没有类的， +//! 它的属性陈述就进不了任何声明了定义域的属性的候选。没有 `UTOPIA_DATABASE_URL` 时跳过。 + +use super::apply_bound; +use sqlx::PgPool; +use uuid::Uuid; + +struct Fx { + org: Uuid, + kb: Uuid, + class: Uuid, + late: Uuid, + human: Uuid, + other: Uuid, +} + +async fn seed(pool: &PgPool) -> anyhow::Result { + let (org, ws, kb, class, late, human, other) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'catch-up')") + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'catch-up')") + .bind(ws) + .bind(org) + .execute(pool) + .await?; + sqlx::query("INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'catch-up')") + .bind(kb) + .bind(ws) + .execute(pool) + .await?; + sqlx::query("INSERT INTO entity_types(id,kb_id,key,label) VALUES($1,$2,'company','企业')") + .bind(class) + .bind(kb) + .execute(pool) + .await?; + // 「公司」已经绑到企业类；判定时还没有下面这些实体 + sqlx::query( + "INSERT INTO type_bindings(id,kb_id,kind_word,type_id,status) VALUES($1,$2,'公司',$3,'bound')", + ) + .bind(Uuid::now_v7()) + .bind(kb) + .bind(class) + .execute(pool) + .await?; + // 判定之后抽出来的公司：类别词写法带空白，归一后是同一个词 + sqlx::query( + "INSERT INTO entities(id,kb_id,canonical_name,specific_type) VALUES($1,$2,'蓝海物流有限公司',' 公司 ')", + ) + .bind(late) + .bind(kb) + .execute(pool) + .await?; + // 人说了它没有类：绑定不盖人的决定 + sqlx::query( + "INSERT INTO entities(id,kb_id,canonical_name,specific_type,type_source) VALUES($1,$2,'某公司','公司','human')", + ) + .bind(human) + .bind(kb) + .execute(pool) + .await?; + // 别的类别词：没有绑定，不动 + sqlx::query( + "INSERT INTO entities(id,kb_id,canonical_name,specific_type,proposed_type) VALUES($1,$2,'王海','董事长','chairman')", + ) + .bind(other) + .bind(kb) + .execute(pool) + .await?; + Ok(Fx { + org, + kb, + class, + late, + human, + other, + }) +} + +async fn class_of( + pool: &PgPool, + id: Uuid, +) -> anyhow::Result<(Option, String, Option)> { + Ok( + sqlx::query_as("SELECT type_id, type_source, proposed_type FROM entities WHERE id=$1") + .bind(id) + .fetch_one(pool) + .await?, + ) +} + +#[tokio::test] +async fn an_entity_built_after_the_decision_still_gets_its_class() -> anyhow::Result<()> { + let Some(url) = crate::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + crate::db::migrate(&pool).await?; + let f = seed(&pool).await?; + let run = async { + assert_eq!(apply_bound(&pool, f.kb).await?, 1); + assert_eq!( + class_of(&pool, f.late).await?, + (Some(f.class), "aligned".into(), None) + ); + assert_eq!( + class_of(&pool, f.human).await?, + (None, "human".into(), None) + ); + assert_eq!( + class_of(&pool, f.other).await?, + (None, "extracted".into(), Some("chairman".into())) + ); + // 幂等：已经在那个类上的不再算改动 + assert_eq!(apply_bound(&pool, f.kb).await?, 0); + anyhow::Ok(()) + } + .await; + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(f.org) + .execute(&pool) + .await?; + run +} diff --git a/crates/utopia-store/tests/store/a_moment_marks_its_state.rs b/crates/utopia-store/tests/store/a_moment_marks_its_state.rs index d454a9979..0af485079 100644 --- a/crates/utopia-store/tests/store/a_moment_marks_its_state.rs +++ b/crates/utopia-store/tests/store/a_moment_marks_its_state.rs @@ -7,7 +7,8 @@ //! （#966 的 A、B、C），都是一段从 2023-06-01 起、至今开着的状态； //! - end：到那天为止，关上开着的那段。终点说的是一个时段（「2024 年离开」）而开着的那段从 //! 时段里开始时，关在时段的尽头； -//! - none 或还没说：不算类型化行，陈述带着日期留在开放图谱。 +//! - none 或还没说：关系不算类型化行，陈述带着日期留在开放图谱；属性按那个日期命名的 +//! 时段读（0053 修订 2026-10-22，`materialize::period_tests`）。 //! //! 之前照抄写下的空段由物化作废重算，人写的、人改成一刻的不碰；声明成状态的属性下，两端 //! 相等的行写入、改区间、关在起点都被拒（同时开始的冲突也不能关在那一刻），没有属性的行 diff --git a/docs/decisions/0044-the-ontology-is-a-view-over-what-documents-say.md b/docs/decisions/0044-the-ontology-is-a-view-over-what-documents-say.md index 39e946640..29cb53776 100644 --- a/docs/decisions/0044-the-ontology-is-a-view-over-what-documents-say.md +++ b/docs/decisions/0044-the-ontology-is-a-view-over-what-documents-say.md @@ -1,6 +1,6 @@ # 0044 · The ontology is a view over what documents say -- **Status**: In progress · accepted 2026-09-17 · cut 1 built (#731), cut 2 mostly built (#751, #876, #878, #882), cut 5 is 0061, cut 6 built (#885) · cut 2 and cut 6 measured twice on 100 Re-DocRED documents, two groups each (bench README, the fourth run on gemini-3.5-flash 2026-09-25 and the ninth on deepseek-flash 2026-09-30): judged precision passes on both (80–89%), same-sentence recall passes on gemini (17.8% / 17.1%) and fails on deepseek-flash (12.3% / 13.2%, the extractor finds a third fewer entity pairs), tokens per document fail on both (31k–48k against the 6k threshold on a fresh base, 25k and 42k on a warm one: a second 100 documents bring as many new signatures as the first, and extraction with time interpretation alone is at the threshold; 19k–22k with thinking off in every call, #1066); the errata agent adds without retracting (0–2 retractions, none wrong) · open: the tokens-per-document threshold (about three times over with thinking off; prompting a general-purpose model does not close it, and a model trained for these steps is planned), cut 4 +- **Status**: In progress · accepted 2026-09-17 · cut 1 built (#731), cut 2 mostly built (#751, #876, #878, #882), cut 5 is 0061, cut 6 built (#885) · cut 2 and cut 6 measured twice on 100 Re-DocRED documents, two groups each (bench README, the fourth run on gemini-3.5-flash 2026-09-25 and the ninth on deepseek-flash 2026-09-30): judged precision passes on both (80–89%), same-sentence recall passes on gemini (17.8% / 17.1%) and fails on deepseek-flash (12.3% / 13.2%, the extractor finds a third fewer entity pairs), tokens per document fail on both (31k–48k against the 6k threshold on a fresh base, 25k and 42k on a warm one: a second 100 documents bring as many new signatures as the first, and extraction with time interpretation alone is at the threshold; 19k–22k with thinking off in every call, #1066); the errata agent adds without retracting (0–2 retractions, none wrong) · revised 2026-10-22 (a text attribute binds an entity object) · open: the tokens-per-document threshold (about three times over with thinking off; prompting a general-purpose model does not close it, and a model trained for these steps is planned), cut 4 - **Written**: 2026-09-16 (conventions in the [README](README.md)) - **Related**: [0022](0022-an-unknown-date-is-not-an-open-one.md) put a document's date in `attested_at` beside the world and record axes; [0025](0025-governance-reads-the-ledger-before-it-decides.md) and [0027](0027-an-automatic-merge-is-gated-by-what-it-can-undo.md) put agent decisions through a gate that weighs what they can undo; [0041](0041-a-name-is-a-claim-about-an-entity.md) made names facts and identity a matter of evidence; [0043](0043-every-review-queue-is-governed.md) sent every review queue through the governor; #714 found upload time used as the document date in extraction. @@ -68,6 +68,8 @@ This is how the facts a reader draws without the text stating them (a place's co When the ontology changes, only facts under changed signatures and rules are recomputed. A signature with no property stays in the open graph, loses nothing, and counts toward the workbench's suggestions. +**Revision 2026-10-22 (a text attribute binds a statement whose object is a named thing):** cut 2 admitted an attribute only for a signature whose object is a value, so an attribute whose value is a name (法定代表人, 注册地址, a product's manufacturer written as text) never had a candidate: extraction lists a person or a place as a named thing, the statement's object is an entity, and the signature is decided `no_candidates`. The person who declared the attribute reads that as the attribute not being extracted. An attribute of datatype `text` that declares a domain is now a candidate for a signature with an entity object as well, forward only, when that domain admits the subject's class (an attribute with no declared domain, `known_as` among them, would otherwise be offered on every relation signature); the aligner's prompt says that such an attribute takes the object's name as its value when the object is named only to carry that value. Materialisation writes the object's canonical name as the value, so the typed row is a value fact and does not link the two entities: a base that wants the link declares a relation, and a relation and a text attribute that both fit go to the model like any two candidates. Number, date and boolean attributes are not candidates for an entity object: a name is not a figure. + **Revision 2026-10-16 ([#1267](https://github.com/deeplethe/utopia/issues/1267)):** a statement's role-word qualifiers are read back: entity detail, graph edges and `entity_facts` carry them as `statement_qualifiers`, beside the typed `qualifiers`, on the open statement and on the typed rows it materialised. **Revision 2026-10-15 (the ontology decides a phrase it declares, [#1242](https://github.com/deeplethe/utopia/issues/1242)):** a signature from a document a `statements` source pushed ([0054](0054-a-source-may-push-statements-in-the-open-contract.md)) is bound without the model when its phrase, normalised as kind words are, equals the key or label of exactly one declared relation that is not a state, and the relation's declared domain admits the subject's class and its declared range the object's class, through the class hierarchy and forward only. The binding is recorded `decided_by = 'ontology'` (migration 0113), with `votes = {"reason": "declared_relation"}` and no basis. It is decided when the document's extraction finishes, and that extraction then queues `materialize` unless one is queued, so a base with no chat model gets typed facts from what it is pushed; on a base of 1,220 pushed entities every class was bound up front and every relation stayed open, because the aligner is the only writer of phrase bindings and needs a model. diff --git a/docs/decisions/0053-a-phrase-decision-records-the-inputs-it-considered.md b/docs/decisions/0053-a-phrase-decision-records-the-inputs-it-considered.md index 272890aa0..efffc60e4 100644 --- a/docs/decisions/0053-a-phrase-decision-records-the-inputs-it-considered.md +++ b/docs/decisions/0053-a-phrase-decision-records-the-inputs-it-considered.md @@ -1,6 +1,6 @@ # 0053 · A phrase decision records the inputs it considered -- **Status**: Implemented 2026-09-23 (#878, migration 0072) · revised 2026-09-26 (migration 0092) · revised 2026-09-27 (#975, migration 0098) · revised 2026-10-17 (#1295, migration 0114) · open: the backfill and printing the named period (#994 cut 1), time resolution's ends (cut 2) +- **Status**: Implemented 2026-09-23 (#878, migration 0072) · revised 2026-09-26 (migration 0092) · revised 2026-09-27 (#975, migration 0098) · revised 2026-10-17 (#1295, migration 0114) · revised 2026-10-22 (a single date on an attribute with `none` or no value names a period) · open: the backfill and printing the named period (#994 cut 1), time resolution's ends (cut 2), recomputing a period row when a person later sets `start` or `end` - **Written**: 2026-09-23 - **Related**: [0044](0044-the-ontology-is-a-view-over-what-documents-say.md) decision 3; [0051](0051-a-human-phrase-decision-carries-its-materialization-work.md); #807, #795, #801 (withdrawn), #773, #754, #966, #994 @@ -241,7 +241,8 @@ Regression coverage, with a real PostgreSQL: - The three orders from #966 (`joined` before `works for`, after it, and alone) give one row from 2023-06-01 when the binding says `start`. - A phrase that marks the end closes the open row at its date. With `none` or no value, a single - date computes no typed row. + date computes no typed row (overturned 2026-10-22: it computes a row over the period the date + names, see the revision below). - "Left in 2024" closes a row from 2024-03-01 at 2025-01-01, in both orders. A row from 2020 closes at 2024-01-01, a row from 2025-06-01 stays open, and a state that starts and ends on one day holds through it, each in both orders. @@ -382,6 +383,44 @@ Regression coverage, with a real PostgreSQL: explicit `true` without a start, a dated end or a state property; and refuses an interval inverted only after truncation. +## Revision 2026-10-22 (a single date that marks neither end names a period) + +The 2026-09-27 revision said that with `none` or no value a single date computes no typed row. +Measured on four Chinese company profiles with a hand-written ontology (one class, six +attributes), that rule was the largest single reason an attribute never reached the typed graph: +every annual figure ("2023年营业收入58亿元", "2022年营业收入15.6亿元") is a statement dated at a +single moment, the aligner answered `none` or gave no value for every attribute binding, and +`MOMENT_UNREAD` excluded all of them. A person answering `都不是` in the alignment queue changed +nothing, because `none` was excluded too. The same run showed the value is not stable: the same +signature was answered `start` in one run and `none` in the next, and the row computed under +`start` was retired. + +**With `none` or no value, a single date on an attribute is read as the period it names.** A +statement whose two ends are equal at precision P, bound to a state *attribute* whose binding marks +neither end, is written as holding from the date through the end of its bucket: `valid_from = t`, `valid_to = +bucket_end(t, P)` with `end_after_period` set, the encoding #994 already uses for "left in 2024", +so the reader prints the period the document wrote. "2023年营业收入58亿元" becomes a row that +holds during 2023 and no longer; "成立于2015年3月，注册资本为5000万元", whose date the model +attached to both statements, becomes a registered-capital row that held during March 2015, which +the sentence does say. The row claims nothing beyond the sentence: that the state held at that +period. `start` and `end` are unchanged. An implied row from a rule, which has no `marks`, takes +the same reading when the rule concludes an attribute. A single date on a *relation* with `none` +or no value still computes no row: a dated value is a measurement for its period, while "joined on +2023-06-01" read as a one-day employment would close the open row that starts the same day at the +next day (the write gate closes a same-start open row when an observation brings an end), which is +an ending the document never stated. The invariant that a state holds at no moment is kept: the row's end is after its +start by construction, and step 0 of the recompute still retires rows written with equal ends. + +- The binding still records that it was asked and is still listed in the alignment queue until a + person sets a value: the period reading is the default, not the answer. +- A row computed under `none` is not recomputed when a person later sets `start` or `end`: the + recompute keys a source on its binding's property and direction, not on `marks`, as it did + before this revision for `start` ↔ `end`. Open; the queue item shows the current reading. +- Regression coverage: a value statement dated at a single year bound to an attribute with `none` + computes a row from that year with `end_after_period`; the same statement bound with no value + computes the same row. The relation cases of the 2026-09-27 revision (`none` or no value computes + no row, a rule reads a start date but not a single date) stand unchanged. + ## Status history The status line as it stood on 2026-09-27, before status lines were cut to one line: diff --git a/docs/design/ontology.md b/docs/design/ontology.md index cf1c3ae03..f8df51ca0 100644 --- a/docs/design/ontology.md +++ b/docs/design/ontology.md @@ -47,7 +47,13 @@ retype, including "no type", out of every automatic path [0009, 0001]. Same-name across types and untyped [0009]. Type resolution (candidates from class descriptions and from the classes of context neighbours, interleaved and never scored together; a class inside the current subtree applied on its own, a cross-axis class sent to a person, each pair acknowledged once) runs -from the ontology page; extraction no longer enqueues it [0001, 0016, #736]. +from the ontology page; extraction no longer enqueues it [0001, 0016, #736]. A kind-word binding +is projected onto its entities when it is decided, and again at the end of every extraction and +the start of every kind-word alignment run (`type_bindings::apply_bound`): extraction reads the +binding table once per document, so a word bound while a document was still extracting left that +document's later entities untyped, and an untyped subject admits no property with a declared +domain. On a four-document test the table document's six attribute statements were lost this way +(2026-10-22). **A class's member list can name what its members lack** [#1269]. The instance list (`GET /kbs/{id}/ontology/entity-types/{type_id}/entities`) takes `missing_predicate`, a property key @@ -86,7 +92,12 @@ shortlist is part of the decision's basis, so a changed list re-asks. The candid properties are described once per batch and each item names only its keys. A signature the votes disagree on is `undecided` for the alignment queue of #725; one with no fitting property is `none`, its statements stay in the open graph and it counts toward the workbench's suggestions. Bindings live in `phrase_bindings`. A binding to a state property also -says what a statement dated at a single moment marks (`marks`: start, end or none). The aligner asks +says what a statement dated at a single moment marks (`marks`: start, end or none); with `none` or +no value a statement bound to an attribute is materialised over the period its date names, +`valid_to` at the bucket's end with `end_after_period` set, so "2023年营业收入58亿元" is a row that +holds during 2023; on a relation it still computes no row [0053 revision 2026-10-22]. An attribute of datatype `text` with a declared domain is also a candidate for a signature whose +object is a named thing (法定代表人 → 陈国华), forward only; its typed row carries the object's +name as the value and no link [0044 revision 2026-10-22]. The aligner asks it once, with the binding: the value is written when both votes give the same one, and a binding without one records the ask (`marks_asked_at`) and waits in the alignment queue, as a person's binding without one does. A binding made before the question is asked once, and that question only @@ -187,7 +198,11 @@ invalidated, rows that are due and missing are added, rows whose binding is unch id, evidence and recorded time. Several statements that produce the same triple give one row: the row is written through the typed graph's own gate (the dedup, time refinement and closing rules of `insert_fact`), and `typed_fact_sources` lists every statement behind it, so a row is retired only when none of its -sources holds [#753]. On the 25-document batch the 36 bound signatures give 184 typed rows. +sources holds [#753]. On the 25-document batch the 36 bound signatures give 184 typed rows. A +bound value the attribute's datatype cannot read (「超过2000人」as a number, a currency that does +not convert) stays an open statement and is counted in `extraction_drops` as `attr_datatype`, +keyed by the property and rewritten as a set on every recompute, so the row retires once the value +reads; before 2026-10-22 this skip was silent. **Argument order is enforced, participation is guided.** A declared domain or range shapes candidates and never discards a fact; argument order is the key's encoding convention, so a fact diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 8c5310367..b436fd1c8 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -595,6 +595,7 @@ export const en = { phrase_is_value: "Kept, but the phrase is the value itself", phrase_is_subject: "Kept, but the phrase is the subject's own name", object_is_value: "Kept, with the object read as a value", + attr_datatype: "Kept as an open statement, but the value cannot be read as the attribute's datatype", } as Record, // 来源级重抽：不危险，只是费时费钱——轻确认，文案直说成本与保留项 reExtractSource: "Re-extract", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index e63b50fc8..f2fb52b19 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -536,6 +536,7 @@ export const zh: Strings = { phrase_is_value: "已保留，但短语就是那个值", phrase_is_subject: "已保留，但短语就是主语的名字", object_is_value: "已保留，宾语按值记下", + attr_datatype: "已保留为开放陈述，但值按属性声明的类型读不出来", }, reExtractSource: "重新抽取", reExtractTitle: "重新抽取这个来源？", From b84d9b4eb5519266f133f5559f42cd9f65a54d68 Mon Sep 17 00:00:00 2001 From: WaylandYang <145302500+WaylandYang@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:53:13 +0800 Subject: [PATCH 04/10] A base can start from a scenario pack: the classes, attributes and competency questions one kind of corpus is read for (#1369) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The standard packs give types, not the predicates a corpus needs, and the empty start grows a vocabulary only after documents arrive. A scenario pack (0073) is the top-down start: one JSON file embedded in the binary, bilingual, with definitions written for the aligner, datatypes and units already right, and the questions the base is meant to answer. It installs at creation through the editor's own calls, so its elements are the base's own; a key that already exists is skipped and reported; questions are created accepted with the creating person as author; `embed_ontology` is queued. Three packs ship: 企业档案 (enterprise-profile), 合同链 (contract-chain), 人事档案 (personnel-records). `GET /scenario-packs` lists them and the creation dialog has a second picker beside the standard packs. A unit test checks every shipped file: keys unique, hosts, domains and parents resolve to classes listed earlier, datatypes valid, every definition in both languages. Decision 4 of 0073 (the competency bench per pack) is open. Signed-off-by: Wayland Yang Co-authored-by: Wayland Yang Co-authored-by: Claude Fable 5.1 --- crates/utopia-server/packs/README.md | 7 + .../packs/scenarios/contract-chain.json | 167 +++++++ .../packs/scenarios/enterprise-profile.json | 203 ++++++++ .../packs/scenarios/personnel-records.json | 144 ++++++ crates/utopia-server/src/api/kbs.rs | 36 ++ crates/utopia-server/src/api/mod.rs | 1 + crates/utopia-server/src/main.rs | 1 + crates/utopia-server/src/scenario_packs.rs | 448 ++++++++++++++++++ ...pack-is-a-vocabulary-with-its-questions.md | 49 ++ docs/decisions/README.md | 1 + docs/design/ontology.md | 9 +- web/src/api.ts | 14 + web/src/i18n/en.ts | 8 + web/src/i18n/zh.ts | 6 + web/src/pages/MyKbs.tsx | 27 +- 15 files changed, 1119 insertions(+), 2 deletions(-) create mode 100644 crates/utopia-server/packs/scenarios/contract-chain.json create mode 100644 crates/utopia-server/packs/scenarios/enterprise-profile.json create mode 100644 crates/utopia-server/packs/scenarios/personnel-records.json create mode 100644 crates/utopia-server/src/scenario_packs.rs create mode 100644 docs/decisions/0073-a-scenario-pack-is-a-vocabulary-with-its-questions.md diff --git a/crates/utopia-server/packs/README.md b/crates/utopia-server/packs/README.md index 69dbb3020..0ee145ea2 100644 --- a/crates/utopia-server/packs/README.md +++ b/crates/utopia-server/packs/README.md @@ -20,3 +20,10 @@ 重新抓取、`gzip -9c` 覆盖、更新 `ontology_packs.rs` 里的计数与本文件的抓取日。 **不要改动原文内容**——投影只覆盖当下能消费的部分，原文保真是 [0001](../../../docs/decisions/0001-ontology-import-and-governance.md) 判据 1。 + +## 场景包（`scenarios/`，0073） + +不是抓来的本体，是手写的：一个场景一个 JSON，几个类、十来条属性和关系、要回答的能力问题， +标签与定义两种语言都写。由 `src/scenario_packs.rs` 内嵌并在建库时走编辑器的路建进库。 +改一个包：改 JSON，跑 `cargo test -p utopia-server scenario_packs`——它检查键不重复、 +定义域和父类指向本包排在前面的类、datatype 合法、每条定义两种语言都有。 diff --git a/crates/utopia-server/packs/scenarios/contract-chain.json b/crates/utopia-server/packs/scenarios/contract-chain.json new file mode 100644 index 000000000..5abe189ed --- /dev/null +++ b/crates/utopia-server/packs/scenarios/contract-chain.json @@ -0,0 +1,167 @@ +{ + "id": "contract-chain", + "name": { "zh": "合同链", "en": "Contract chain" }, + "summary": { + "zh": "主合同与补充协议：当事方、签订日期、合同金额、有效期、付款条款，以及哪份补充协议改了哪一条", + "en": "Master agreements and their amendments: parties, signing dates, amounts, terms, payment conditions, and which amendment changed what" + }, + "classes": [ + { + "key": "contract", + "label": { "zh": "合同", "en": "Contract" }, + "description": { + "zh": "一份合同、协议、补充协议或修订本，有自己的名称或编号。", + "en": "A contract, agreement, supplementary agreement or amendment, with a title or number of its own." + } + }, + { + "key": "party", + "label": { "zh": "当事方", "en": "Party" }, + "description": { + "zh": "合同的一方：甲方、乙方、委托方、承包方，可以是企业、机构或个人。", + "en": "A party to a contract: the buyer, seller, principal or contractor; a company, institution or person." + } + }, + { + "key": "clause", + "label": { "zh": "条款", "en": "Clause" }, + "description": { + "zh": "合同里的一条约定：付款条款、交付条款、违约责任、保密条款等，以条号或标题指称。", + "en": "A clause of a contract: payment, delivery, liability, confidentiality, referred to by number or heading." + } + } + ], + "attributes": [ + { + "key": "signed_on", + "host": "contract", + "label": { "zh": "签订日期", "en": "Signed on" }, + "description": { + "zh": "合同签订或签署的日期。原文写「于 2023 年 5 月 8 日签订」「签订日期：2024-01-15」。", + "en": "The date the contract was signed: 于 2023 年 5 月 8 日签订, signed on 15 January 2024." + }, + "datatype": "date", + "functional": true + }, + { + "key": "contract_amount", + "host": "contract", + "label": { "zh": "合同金额", "en": "Contract amount" }, + "description": { + "zh": "合同约定的总价款或总金额。原文写「合同总金额 380 万元」「合同价款为人民币 1.2 亿元」。", + "en": "The total price or amount the contract fixes: 合同总金额 380 万元, a contract price of RMB 120 million." + }, + "datatype": "number", + "unit": "万元", + "functional": true + }, + { + "key": "effective_from", + "host": "contract", + "label": { "zh": "生效日期", "en": "Effective from" }, + "description": { + "zh": "合同开始生效的日期。原文写「自 2024 年 1 月 1 日起生效」。", + "en": "The date the contract takes effect: 自 2024 年 1 月 1 日起生效." + }, + "datatype": "date", + "functional": true + }, + { + "key": "expires_on", + "host": "contract", + "label": { "zh": "到期日期", "en": "Expires on" }, + "description": { + "zh": "合同有效期截止的日期。原文写「有效期至 2026 年 12 月 31 日」「到期日为 2025-06-30」。", + "en": "The date the contract's term ends: 有效期至 2026 年 12 月 31 日." + }, + "datatype": "date", + "functional": true + }, + { + "key": "payment_terms", + "host": "contract", + "label": { "zh": "付款条件", "en": "Payment terms" }, + "description": { + "zh": "合同约定的付款方式和期限，照原文记：「签订后 30 日内支付 30%」「货到验收后 60 天付清」。", + "en": "How and when payment is due, as the contract words it: 30% within 30 days of signing, balance 60 days after acceptance." + }, + "datatype": "text", + "functional": true + }, + { + "key": "contract_number", + "host": "contract", + "label": { "zh": "合同编号", "en": "Contract number" }, + "description": { + "zh": "合同的编号。原文写「合同编号：HT-2024-0017」。", + "en": "The contract's reference number: 合同编号 HT-2024-0017." + }, + "datatype": "text", + "functional": true + } + ], + "relations": [ + { + "key": "party_a", + "label": { "zh": "甲方", "en": "Party A" }, + "description": { + "zh": "宾语是主语合同的甲方（委托方、买方、发包方）。方向：合同 → 当事方。", + "en": "The object is Party A of the subject contract: the principal, buyer or employer. Direction: contract → party." + }, + "domains": ["contract"], + "ranges": ["party"], + "temporal": "state" + }, + { + "key": "party_b", + "label": { "zh": "乙方", "en": "Party B" }, + "description": { + "zh": "宾语是主语合同的乙方（受托方、卖方、承包方）。方向：合同 → 当事方。", + "en": "The object is Party B of the subject contract: the contractor, seller or supplier. Direction: contract → party." + }, + "domains": ["contract"], + "ranges": ["party"], + "temporal": "state" + }, + { + "key": "amends", + "label": { "zh": "修订", "en": "Amends" }, + "description": { + "zh": "主语补充协议或修订本修改、补充或替换宾语合同的约定：「本补充协议对主合同第五条作如下修改」。方向：补充协议 → 主合同。", + "en": "The subject amendment modifies, supplements or replaces terms of the object contract: 本补充协议对主合同第五条作如下修改. Direction: amendment → master agreement." + }, + "domains": ["contract"], + "ranges": ["contract"], + "temporal": "event" + }, + { + "key": "stipulates", + "label": { "zh": "约定", "en": "Stipulates" }, + "description": { + "zh": "主语合同包含宾语条款：「合同第五条约定付款方式」。方向：合同 → 条款。", + "en": "The subject contract contains the object clause: 合同第五条约定付款方式. Direction: contract → clause." + }, + "domains": ["contract"], + "ranges": ["clause"], + "temporal": "state" + } + ], + "questions": [ + { + "question": { "zh": "某份合同的甲方和乙方是谁？", "en": "Who are the parties to a contract?" }, + "needs": { "classes": ["contract", "party"], "properties": ["party_a", "party_b"] } + }, + { + "question": { "zh": "某个日期当天，某份合同适用的付款条件是什么？", "en": "Which payment terms applied under a contract on a given date?" }, + "needs": { "classes": ["contract"], "properties": ["payment_terms", "amends", "effective_from"] } + }, + { + "question": { "zh": "哪份补充协议修改了某份合同，什么时候签订的？", "en": "Which amendment changed a contract, and when was it signed?" }, + "needs": { "classes": ["contract"], "properties": ["amends", "signed_on"] } + }, + { + "question": { "zh": "某份合同的金额是多少，有效期到什么时候？", "en": "What is a contract's amount, and when does it expire?" }, + "needs": { "classes": ["contract"], "properties": ["contract_amount", "expires_on"] } + } + ] +} diff --git a/crates/utopia-server/packs/scenarios/enterprise-profile.json b/crates/utopia-server/packs/scenarios/enterprise-profile.json new file mode 100644 index 000000000..9b24ddef4 --- /dev/null +++ b/crates/utopia-server/packs/scenarios/enterprise-profile.json @@ -0,0 +1,203 @@ +{ + "id": "enterprise-profile", + "name": { "zh": "企业档案", "en": "Enterprise profile" }, + "summary": { + "zh": "公司简介、工商信息、年报：企业的注册资本、成立时间、人员规模、营收、法定代表人与控股关系", + "en": "Company profiles, registry records and annual reports: capital, founding, headcount, revenue, legal representative and holding chains" + }, + "classes": [ + { + "key": "company", + "label": { "zh": "企业", "en": "Company" }, + "description": { + "zh": "企业、公司、集团、工厂、药房等以营利为目的的法人组织；一家分公司或子公司也是一个企业。", + "en": "A business organisation: a company, corporation, group, plant or shop; a subsidiary or branch is a company too." + } + }, + { + "key": "person", + "label": { "zh": "人员", "en": "Person" }, + "description": { + "zh": "一个具体的人：董事长、法定代表人、高管、员工、创始人。", + "en": "A particular human being: a chairman, legal representative, executive, employee or founder." + } + }, + { + "key": "product", + "label": { "zh": "产品", "en": "Product" }, + "description": { + "zh": "企业生产、销售或提供的具体产品、品牌或服务，有自己的名字。", + "en": "A named product, brand or service a company makes, sells or provides." + } + }, + { + "key": "place", + "label": { "zh": "地点", "en": "Place" }, + "description": { + "zh": "城市、区县、园区、地址等地理位置。", + "en": "A geographic place: a city, district, park or address." + } + } + ], + "attributes": [ + { + "key": "registered_capital", + "host": "company", + "label": { "zh": "注册资本", "en": "Registered capital" }, + "description": { + "zh": "企业在登记机关登记的注册资本金额。原文写「注册资本 5000 万元」「注册资本为 1.2 亿元」。", + "en": "The registered capital a company declares to the registry, as written: 注册资本 5000 万元, registered capital of RMB 120 million." + }, + "datatype": "number", + "unit": "万元", + "functional": true + }, + { + "key": "founded_at", + "host": "company", + "label": { "zh": "成立时间", "en": "Founded" }, + "description": { + "zh": "企业成立、注册登记的日期。原文写「成立于 2015 年 3 月」「2008 年在上海成立」。", + "en": "The date a company was founded or registered: 成立于 2015 年 3 月, founded in 2008." + }, + "datatype": "date", + "functional": true + }, + { + "key": "employee_count", + "host": "company", + "label": { "zh": "员工人数", "en": "Employees" }, + "description": { + "zh": "企业现有员工的人数。原文写「员工 1200 人」「现有员工约 350 名」。", + "en": "How many people a company employs: 员工 1200 人, about 350 employees." + }, + "datatype": "number", + "functional": true + }, + { + "key": "revenue", + "host": "company", + "label": { "zh": "营业收入", "en": "Revenue" }, + "description": { + "zh": "企业某一年度的营业收入金额，不是增长率或变化。原文写「2023 年营业收入 2.3 亿元」。", + "en": "A company's revenue for a period, an amount and not a change or growth rate: 2023 年营业收入 2.3 亿元." + }, + "datatype": "number", + "unit": "万元", + "functional": true + }, + { + "key": "net_profit", + "host": "company", + "label": { "zh": "净利润", "en": "Net profit" }, + "description": { + "zh": "企业某一年度的净利润金额，不是增长率或变化。原文写「净利润 1800 万元」。", + "en": "A company's net profit for a period, an amount and not a change: 净利润 1800 万元." + }, + "datatype": "number", + "unit": "万元", + "functional": true + }, + { + "key": "legal_representative", + "host": "company", + "label": { "zh": "法定代表人", "en": "Legal representative" }, + "description": { + "zh": "企业的法定代表人的姓名。原文写「法定代表人为李明」「法定代表人：陈国华」。", + "en": "The name of the company's legal representative: 法定代表人为李明." + }, + "datatype": "text", + "functional": true + }, + { + "key": "registered_address", + "host": "company", + "label": { "zh": "注册地址", "en": "Registered address" }, + "description": { + "zh": "企业的注册地或注册地址，一个地名或地址。原文写「注册地为浙江绍兴」「注册地址：杭州市滨江区」。", + "en": "Where the company is registered, a place name or address: 注册地为浙江绍兴." + }, + "datatype": "text", + "functional": true + }, + { + "key": "credit_code", + "host": "company", + "label": { "zh": "统一社会信用代码", "en": "Unified social credit code" }, + "description": { + "zh": "企业的 18 位统一社会信用代码。", + "en": "The company's 18-character unified social credit code." + }, + "datatype": "text", + "functional": true + } + ], + "relations": [ + { + "key": "controlled_by", + "label": { "zh": "控股方", "en": "Controlled by" }, + "description": { + "zh": "主语企业由宾语企业控股或全资持有：「由仁和医药集团控股」「集团旗下的仁和制药厂」。方向：子公司 → 母公司。", + "en": "The subject company is controlled or wholly owned by the object company: 由仁和医药集团控股, 集团旗下的仁和制药厂. Direction: subsidiary → parent." + }, + "domains": ["company"], + "ranges": ["company"], + "temporal": "state" + }, + { + "key": "officer_of", + "label": { "zh": "任职于", "en": "Officer of" }, + "description": { + "zh": "主语人员在宾语企业担任董事长、总经理、高管等职务：「董事长为王海」「周敏任心内科主任」。方向：人员 → 企业。", + "en": "The subject person holds an office at the object company (chairman, CEO, executive): 董事长为王海. Direction: person → company." + }, + "domains": ["person"], + "ranges": ["company"], + "temporal": "state" + }, + { + "key": "produces", + "label": { "zh": "主要产品", "en": "Produces" }, + "description": { + "zh": "主语企业生产、销售或提供宾语产品：「主要产品为启明 X7 推理加速卡」「主要生产降压药」。方向：企业 → 产品。", + "en": "The subject company makes, sells or provides the object product: 主要产品为启明 X7 推理加速卡. Direction: company → product." + }, + "domains": ["company"], + "ranges": ["product"], + "temporal": "state" + }, + { + "key": "headquartered_in", + "label": { "zh": "总部位于", "en": "Headquartered in" }, + "description": { + "zh": "主语企业的总部、注册地或主要经营地在宾语地点：「总部位于杭州市滨江区」「位于绍兴」。方向：企业 → 地点。", + "en": "The subject company's headquarters, registered seat or main place of business is the object place: 总部位于杭州市滨江区. Direction: company → place." + }, + "domains": ["company"], + "ranges": ["place"], + "temporal": "state" + } + ], + "questions": [ + { + "question": { "zh": "某家企业的注册资本是多少？", "en": "What is a company's registered capital?" }, + "needs": { "classes": ["company"], "properties": ["registered_capital"] } + }, + { + "question": { "zh": "某家企业某一年的营业收入和净利润是多少？", "en": "What were a company's revenue and net profit in a given year?" }, + "needs": { "classes": ["company"], "properties": ["revenue", "net_profit"] } + }, + { + "question": { "zh": "某家企业的法定代表人是谁，成立于什么时候？", "en": "Who is a company's legal representative, and when was it founded?" }, + "needs": { "classes": ["company"], "properties": ["legal_representative", "founded_at"] } + }, + { + "question": { "zh": "某家企业由谁控股，旗下有哪些企业？", "en": "Who controls a company, and which companies does it control?" }, + "needs": { "classes": ["company"], "properties": ["controlled_by"] } + }, + { + "question": { "zh": "某个人在哪些企业任职？", "en": "At which companies does a person hold office?" }, + "needs": { "classes": ["person", "company"], "properties": ["officer_of"] } + } + ] +} diff --git a/crates/utopia-server/packs/scenarios/personnel-records.json b/crates/utopia-server/packs/scenarios/personnel-records.json new file mode 100644 index 000000000..89038e914 --- /dev/null +++ b/crates/utopia-server/packs/scenarios/personnel-records.json @@ -0,0 +1,144 @@ +{ + "id": "personnel-records", + "name": { "zh": "人事档案", "en": "Personnel records" }, + "summary": { + "zh": "任免通知、组织架构、花名册：谁在哪个部门担任什么岗位、从何时到何时、向谁汇报", + "en": "Appointment notices, org charts and rosters: who held which post in which unit, from when to when, reporting to whom" + }, + "classes": [ + { + "key": "person", + "label": { "zh": "人员", "en": "Person" }, + "description": { + "zh": "一个具体的人：员工、干部、负责人。", + "en": "A particular human being: an employee, officer or manager." + } + }, + { + "key": "department", + "label": { "zh": "部门", "en": "Department" }, + "description": { + "zh": "组织里的一个单位：部门、处室、事业部、分公司、团队。", + "en": "A unit of an organisation: a department, division, business unit, branch or team." + } + }, + { + "key": "post", + "label": { "zh": "岗位", "en": "Post" }, + "description": { + "zh": "一个职位或岗位：总经理、部门主任、项目负责人、财务经理。岗位是职位本身，不是担任它的人。", + "en": "A post or position: general manager, department head, project lead. The post itself, not the person holding it." + } + } + ], + "attributes": [ + { + "key": "hired_on", + "host": "person", + "label": { "zh": "入职日期", "en": "Hired on" }, + "description": { + "zh": "人员入职、报到的日期。原文写「2019 年 7 月入职」「入职时间：2021-03-01」。", + "en": "The date a person joined the organisation: 2019 年 7 月入职." + }, + "datatype": "date", + "functional": true + }, + { + "key": "left_on", + "host": "person", + "label": { "zh": "离职日期", "en": "Left on" }, + "description": { + "zh": "人员离职、退休或调离的日期。原文写「2024 年 3 月离职」。", + "en": "The date a person left, retired or transferred out: 2024 年 3 月离职." + }, + "datatype": "date", + "functional": true + }, + { + "key": "grade", + "host": "person", + "label": { "zh": "职级", "en": "Grade" }, + "description": { + "zh": "人员的职级或级别，照原文记：「P7」「正处级」「高级工程师」。", + "en": "A person's grade or rank as written: P7, senior engineer." + }, + "datatype": "text", + "functional": true + }, + { + "key": "employee_id", + "host": "person", + "label": { "zh": "工号", "en": "Employee ID" }, + "description": { + "zh": "人员的工号或员工编号。", + "en": "A person's employee number." + }, + "datatype": "text", + "functional": true + } + ], + "relations": [ + { + "key": "member_of", + "label": { "zh": "所属部门", "en": "Member of" }, + "description": { + "zh": "主语人员属于宾语部门：「调入财务部」「研发中心员工张三」。方向：人员 → 部门。", + "en": "The subject person belongs to the object department: 调入财务部. Direction: person → department." + }, + "domains": ["person"], + "ranges": ["department"], + "temporal": "state" + }, + { + "key": "holds_post", + "label": { "zh": "担任", "en": "Holds post" }, + "description": { + "zh": "主语人员担任宾语岗位：「任命李四为市场部经理」「周敏担任心内科主任」。方向：人员 → 岗位。", + "en": "The subject person holds the object post: 任命李四为市场部经理. Direction: person → post." + }, + "domains": ["person"], + "ranges": ["post"], + "temporal": "state" + }, + { + "key": "reports_to", + "label": { "zh": "汇报给", "en": "Reports to" }, + "description": { + "zh": "主语人员向宾语人员汇报、受其领导：「向总经理汇报」「分管副总为王五」。方向：下级 → 上级。", + "en": "The subject person reports to the object person: 向总经理汇报. Direction: subordinate → superior." + }, + "domains": ["person"], + "ranges": ["person"], + "temporal": "state" + }, + { + "key": "part_of", + "label": { "zh": "隶属", "en": "Part of" }, + "description": { + "zh": "主语部门隶属于宾语部门：「财务部下设会计科」「研发中心归属技术事业部」。方向：下级部门 → 上级部门。", + "en": "The subject department is part of the object department: 财务部下设会计科. Direction: sub-unit → parent unit." + }, + "domains": ["department"], + "ranges": ["department"], + "temporal": "state" + } + ], + "questions": [ + { + "question": { "zh": "某个时间段内，谁担任某个岗位？", "en": "Who held a given post during a period?" }, + "needs": { "classes": ["person", "post"], "properties": ["holds_post"] } + }, + { + "question": { "zh": "某个人属于哪个部门，向谁汇报？", "en": "Which department is a person in, and whom do they report to?" }, + "needs": { "classes": ["person", "department"], "properties": ["member_of", "reports_to"] } + }, + { + "question": { "zh": "某个人什么时候入职、什么时候离职？", "en": "When did a person join, and when did they leave?" }, + "needs": { "classes": ["person"], "properties": ["hired_on", "left_on"] } + }, + { + "question": { "zh": "某个部门下设哪些部门，有哪些人？", "en": "Which units sit under a department, and who is in them?" }, + "needs": { "classes": ["department", "person"], "properties": ["part_of", "member_of"] } + } + ] +} diff --git a/crates/utopia-server/src/api/kbs.rs b/crates/utopia-server/src/api/kbs.rs index 9937d5070..ab1dccdd4 100644 --- a/crates/utopia-server/src/api/kbs.rs +++ b/crates/utopia-server/src/api/kbs.rs @@ -25,6 +25,10 @@ pub struct CreateKbReq { /// 后面的包撞名时查对齐表。schema.org 放第一个，别的包才对得上。 #[serde(default)] pub ontology_packs: Vec, + /// 场景包的 id（0073）：按业务场景预置的类、属性和能力问题，走编辑器的路建出来。 + /// 排在标准包之后装，撞键的跳过 + #[serde(default)] + pub scenario_packs: Vec, } #[derive(Deserialize)] @@ -132,6 +136,19 @@ pub async fn create( .await?; install_packs(&state, kb.id, user.id, token_id, &req.ontology_packs).await?; let kb = utopia_store::kbs::get(&state.pool, kb.id).await?; + // 场景包排在标准包之后：它的类键撞上标准包的就让给标准包，声明还能指向它们（0073） + let scenarios = crate::scenario_packs::install( + &state, + kb.id, + user.id, + &kb.ontology_lang, + &req.scenario_packs, + ) + .await?; + if scenarios.classes + scenarios.properties + scenarios.questions > 0 { + tracing::info!(kb_id = %kb.id, classes = scenarios.classes, properties = scenarios.properties, + questions = scenarios.questions, "场景包装好"); + } Ok(Json(kb)) } @@ -444,6 +461,25 @@ pub(super) async fn install_packs( Ok(()) } +/// 场景包清单（0073），给建库界面：两种语言的名字都给，界面按自己的语言挑； +/// 装进库时按库的 `ontology_lang` 挑 +pub async fn list_scenario_packs(AuthUser(_): AuthUser) -> ApiResult> { + let packs: Vec<_> = crate::scenario_packs::PACKS + .iter() + .map(|p| { + json!({ + "id": p.id, + "name": { "zh": p.name.zh, "en": p.name.en }, + "summary": { "zh": p.summary.zh, "en": p.summary.en }, + "classes": p.classes.len(), + "properties": p.attributes.len() + p.relations.len(), + "questions": p.questions.len(), + }) + }) + .collect(); + Ok(Json(json!({ "packs": packs }))) +} + /// 可选的本体包清单，给建库界面。不需要登录之外的权限——它是静态数据。 pub async fn list_packs(AuthUser(_): AuthUser) -> ApiResult> { let packs: Vec<_> = crate::ontology_packs::PACKS diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rs index db71ba3b0..7d488e3f1 100644 --- a/crates/utopia-server/src/api/mod.rs +++ b/crates/utopia-server/src/api/mod.rs @@ -140,6 +140,7 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router { .route("/workspaces/{id}/kbs", get(kbs::list).post(kbs::create)) // 建库界面用的静态清单，与具体工作区无关 .route("/ontology-packs", get(kbs::list_packs)) + .route("/scenario-packs", get(kbs::list_scenario_packs)) .route("/workspaces/{id}/my-kbs", get(kbs::my_kbs)) .route( "/workspaces/{id}/settings", diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs index 918a5027a..83d113057 100644 --- a/crates/utopia-server/src/main.rs +++ b/crates/utopia-server/src/main.rs @@ -37,6 +37,7 @@ mod rdf; mod readers; mod retrieval; mod rss_full_content; +mod scenario_packs; mod search_recovery; mod state; mod ticket; diff --git a/crates/utopia-server/src/scenario_packs.rs b/crates/utopia-server/src/scenario_packs.rs new file mode 100644 index 000000000..9a573358f --- /dev/null +++ b/crates/utopia-server/src/scenario_packs.rs @@ -0,0 +1,448 @@ +//! 场景包（0073）：按业务场景预置的一小套词汇，连同它要回答的问题。 +//! +//! 标准本体包（`ontology_packs`，0008）给的是通用类型，不给语料要的谓词；场景包反过来： +//! 几个类、十来条属性和关系，定义是写给对齐器读的，单位和 datatype 已经选对，再加上这类 +//! 语料要回答的能力问题（0061）。文件内嵌进二进制，离线部署也有。 +//! +//! **走编辑器的路，不走 OWL 导入。** 类和属性用本体页同样的调用建出来，没有 IRI，是这个库 +//! 自己的元素（0012）：能改、能删、导出时是库自己的词汇。问题按人提的记（`origin = person`， +//! 建库的人是作者）：勾上场景包就是这个人对库的要求。只在建库时装：新库没有事实，不必对账。 +//! +//! 键撞上已有元素（先装的标准包、另一个场景包）的跳过并报告，不合并不覆盖；定义域、值域 +//! 只认包自己的类键或库里已有的键，认不出的丢掉那条声明、照样装。 + +use std::collections::HashMap; +use std::sync::LazyLock; + +use serde::Deserialize; +use utopia_core::models::RelationAxioms; +use utopia_core::{AppError, AppResult}; +use uuid::Uuid; + +use crate::state::AppState; + +/// 一段双语文字：标签是数据，按库的 `ontology_lang` 取（0004） +#[derive(Debug, Clone, Deserialize)] +pub struct Text { + pub zh: String, + pub en: String, +} + +impl Text { + pub fn get(&self, lang: &str) -> &str { + if lang == "zh" { + &self.zh + } else { + &self.en + } + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Class { + pub key: String, + pub label: Text, + pub description: Text, + /// 父类的键，必须在本包里排在前面或已在库里 + #[serde(default)] + pub parents: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Attribute { + pub key: String, + /// 挂在哪个类上（定义域） + pub host: String, + pub label: Text, + pub description: Text, + /// text / number / date / bool + pub datatype: String, + /// 数的单位，按这个语料写的那种；读不出的值记成 `attr_datatype` 丢弃 + #[serde(default)] + pub unit: Option, + #[serde(default = "yes")] + pub functional: bool, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Relation { + pub key: String, + pub label: Text, + pub description: Text, + #[serde(default)] + pub domains: Vec, + #[serde(default)] + pub ranges: Vec, + /// state / event / eternal（0031） + #[serde(default = "state")] + pub temporal: String, + #[serde(default)] + pub functional: bool, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Question { + pub question: Text, + #[serde(default)] + pub expected_answer: Option, + /// 答案要经过的类和属性（0061 决定 1 的「形状」），照原样存 + #[serde(default)] + pub needs: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ScenarioPack { + pub id: String, + pub name: Text, + pub summary: Text, + #[serde(default)] + pub classes: Vec, + #[serde(default)] + pub attributes: Vec, + #[serde(default)] + pub relations: Vec, + #[serde(default)] + pub questions: Vec, +} + +fn yes() -> bool { + true +} +fn state() -> String { + "state".into() +} + +const FILES: &[(&str, &str)] = &[ + ( + "enterprise-profile", + include_str!("../packs/scenarios/enterprise-profile.json"), + ), + ( + "contract-chain", + include_str!("../packs/scenarios/contract-chain.json"), + ), + ( + "personnel-records", + include_str!("../packs/scenarios/personnel-records.json"), + ), +]; + +/// 内嵌的全部场景包。文件写错是构建的问题，不是运行时的：这里直接 panic，测试 +/// [`tests::every_shipped_pack_parses_and_resolves`] 在 CI 里先于任何部署撞上它 +pub static PACKS: LazyLock> = LazyLock::new(|| { + FILES + .iter() + .map(|(id, text)| { + let pack: ScenarioPack = serde_json::from_str(text) + .unwrap_or_else(|e| panic!("scenario pack {id} does not parse: {e}")); + assert_eq!(&pack.id, id, "scenario pack file {id} carries another id"); + pack + }) + .collect() +}); + +pub fn get(id: &str) -> Option<&'static ScenarioPack> { + PACKS.iter().find(|p| p.id == id) +} + +/// 装了什么、跳过了什么。`skipped` 是人读的句子：键撞上了谁、哪条声明认不出 +#[derive(Debug, Default, serde::Serialize)] +pub struct Report { + pub classes: usize, + pub properties: usize, + pub questions: usize, + pub skipped: Vec, +} + +/// 把几个场景包装进一个新库。`lang` 是库的 `ontology_lang`：标签与定义按它取。 +/// 建了东西就排一份 `embed_ontology`，第一次对齐之前元素得有向量（同编辑器与导入） +pub async fn install( + state: &AppState, + kb_id: Uuid, + actor: Uuid, + lang: &str, + ids: &[String], +) -> AppResult { + let mut report = Report::default(); + if ids.is_empty() { + return Ok(report); + } + let packs: Vec<&ScenarioPack> = ids + .iter() + .map(|id| { + get(id).ok_or_else(|| AppError::invalid("unknown_pack", format!("未知的场景包：{id}"))) + }) + .collect::>()?; + let pool = &state.pool; + // 库里已有的键（先装的标准包、前一个场景包）：撞上的跳过，声明可以指向它们 + let mut classes: HashMap = + sqlx::query_as::<_, (String, Uuid)>("SELECT key, id FROM entity_types WHERE kb_id = $1") + .bind(kb_id) + .fetch_all(pool) + .await? + .into_iter() + .collect(); + let mut properties: HashMap = + sqlx::query_as::<_, (String, Uuid)>("SELECT key, id FROM relation_types WHERE kb_id = $1") + .bind(kb_id) + .fetch_all(pool) + .await? + .into_iter() + .collect(); + for pack in &packs { + for c in &pack.classes { + if classes.contains_key(&c.key) { + report + .skipped + .push(format!("{}: class {} already exists", pack.id, c.key)); + continue; + } + let mut parents = Vec::new(); + for p in &c.parents { + match classes.get(p) { + Some(id) => parents.push(*id), + None => report + .skipped + .push(format!("{}: parent {p} of {} not found", pack.id, c.key)), + } + } + let id = utopia_store::ontology::create_entity_type( + pool, + kb_id, + &c.key, + c.label.get(lang), + "#64748b", + "circle", + &parents, + c.description.get(lang), + ) + .await?; + classes.insert(c.key.clone(), id); + report.classes += 1; + } + for a in &pack.attributes { + if properties.contains_key(&a.key) { + report + .skipped + .push(format!("{}: property {} already exists", pack.id, a.key)); + continue; + } + let Some(host) = classes.get(&a.host) else { + report.skipped.push(format!( + "{}: host {} of {} not found", + pack.id, a.host, a.key + )); + continue; + }; + let id = utopia_store::ontology::create_relation_type( + pool, + kb_id, + &a.key, + a.label.get(lang), + "state", + RelationAxioms { + functional: a.functional, + ..RelationAxioms::default() + }, + a.description.get(lang), + "attribute", + &[*host], + &[], + Some(a.datatype.as_str()), + a.unit.as_deref(), + ) + .await?; + properties.insert(a.key.clone(), id); + report.properties += 1; + } + for r in &pack.relations { + if properties.contains_key(&r.key) { + report + .skipped + .push(format!("{}: property {} already exists", pack.id, r.key)); + continue; + } + let mut resolve = |keys: &[String], side: &str| -> Vec { + keys.iter() + .filter_map(|k| { + let id = classes.get(k).copied(); + if id.is_none() { + report + .skipped + .push(format!("{}: {side} {k} of {} not found", pack.id, r.key)); + } + id + }) + .collect() + }; + let domains = resolve(&r.domains, "domain"); + let ranges = resolve(&r.ranges, "range"); + let id = utopia_store::ontology::create_relation_type( + pool, + kb_id, + &r.key, + r.label.get(lang), + &r.temporal, + RelationAxioms { + functional: r.functional, + ..RelationAxioms::default() + }, + r.description.get(lang), + "relation", + &domains, + &ranges, + None, + None, + ) + .await?; + properties.insert(r.key.clone(), id); + report.properties += 1; + } + for q in &pack.questions { + let text = q.question.get(lang); + if utopia_store::competency_questions::exists_text(pool, kb_id, text).await? { + report + .skipped + .push(format!("{}: question already exists: {text}", pack.id)); + continue; + } + utopia_store::competency_questions::create( + pool, + kb_id, + utopia_store::competency_questions::NewQuestion { + question: text, + expected_answer: q.expected_answer.as_ref().map(|t| t.get(lang)), + needs: q.needs.as_ref(), + origin: "person", + status: "accepted", + created_by: Some(actor), + }, + ) + .await?; + report.questions += 1; + } + let _ = utopia_store::audit::record( + pool, + Some(kb_id), + actor, + "scenario_pack.installed", + "knowledge_base", + Some(kb_id), + serde_json::json!({ "pack": pack.id, "lang": lang }), + ) + .await; + } + if report.classes + report.properties > 0 { + // 新元素要有向量，对齐的短名单才看得见它（同 `reindex_then_align`） + utopia_store::jobs::enqueue_unless_queued_after( + pool, + "embed_ontology", + serde_json::json!({ "kb_id": kb_id }), + std::time::Duration::from_secs(2), + ) + .await?; + } + if !report.skipped.is_empty() { + tracing::info!(%kb_id, skipped = ?report.skipped, "场景包有跳过的元素"); + } + Ok(report) +} + +#[cfg(test)] +mod tests { + use super::{Text, PACKS}; + use std::collections::HashSet; + + /// 文件是手写的 JSON：解析得了、键不重复、每条声明都指向本包排在前面的类、datatype 合法 + #[test] + fn every_shipped_pack_parses_and_resolves() { + assert_eq!(PACKS.len(), 3); + for pack in PACKS.iter() { + let mut classes: HashSet<&str> = HashSet::new(); + for c in &pack.classes { + for p in &c.parents { + assert!( + classes.contains(p.as_str()), + "{}: parent {p} of {} comes later", + pack.id, + c.key + ); + } + assert!(classes.insert(&c.key), "{}: class {} twice", pack.id, c.key); + } + let mut properties: HashSet<&str> = HashSet::new(); + for a in &pack.attributes { + assert!( + classes.contains(a.host.as_str()), + "{}: host {} of {}", + pack.id, + a.host, + a.key + ); + assert!( + matches!(a.datatype.as_str(), "text" | "number" | "date" | "bool"), + "{}: {}", + pack.id, + a.key + ); + assert!( + properties.insert(&a.key), + "{}: property {} twice", + pack.id, + a.key + ); + } + for r in &pack.relations { + for k in r.domains.iter().chain(r.ranges.iter()) { + assert!( + classes.contains(k.as_str()), + "{}: {k} of {}", + pack.id, + r.key + ); + } + assert!( + matches!(r.temporal.as_str(), "state" | "event" | "eternal"), + "{}: {}", + pack.id, + r.key + ); + assert!( + properties.insert(&r.key), + "{}: property {} twice", + pack.id, + r.key + ); + } + assert!( + !pack.questions.is_empty(), + "{}: a pack ships with its questions", + pack.id + ); + for t in pack + .classes + .iter() + .map(|c| &c.description) + .chain(pack.attributes.iter().map(|a| &a.description)) + .chain(pack.relations.iter().map(|r| &r.description)) + { + assert!( + !t.zh.trim().is_empty() && !t.en.trim().is_empty(), + "{}: a definition in both languages", + pack.id + ); + } + } + } + + #[test] + fn a_label_follows_the_base_language() { + let t = Text { + zh: "企业".into(), + en: "Company".into(), + }; + assert_eq!(t.get("zh"), "企业"); + assert_eq!(t.get("en"), "Company"); + assert_eq!(t.get("fr"), "Company"); + } +} diff --git a/docs/decisions/0073-a-scenario-pack-is-a-vocabulary-with-its-questions.md b/docs/decisions/0073-a-scenario-pack-is-a-vocabulary-with-its-questions.md new file mode 100644 index 000000000..2fb06af60 --- /dev/null +++ b/docs/decisions/0073-a-scenario-pack-is-a-vocabulary-with-its-questions.md @@ -0,0 +1,49 @@ +# 0073 · A scenario pack is a vocabulary with its questions + +- **Status**: In progress · accepted 2026-10-22 · decisions 1 to 3 and the first three packs built 2026-10-22 (`scenario_packs.rs`, `packs/scenarios/`, `CreateKbReq.scenario_packs`, `GET /scenario-packs`) · open: decision 4, the competency bench per pack +- **Written**: 2026-10-22 (conventions in the [README](README.md)) +- **Related**: [0008](0008-ontology-packs-as-cold-start.md) ships standard vocabularies as the cold start and decided against bundles; [0061](0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md) makes competency questions the standard an ontology is judged by; [0044](0044-the-ontology-is-a-view-over-what-documents-say.md) makes the ontology a view bound after extraction; [0012](0012-the-ontology-is-a-contract-not-a-suggestion.md) makes every element a contract a person signed; [0004](0004-language-and-localization.md) says a label is data and a description follows `ontology_lang`. + +## Context + +A base starts with one of three vocabularies: nothing, a standard pack (schema.org, W3C Org, PROV-O, FOAF, IOF Core), or an OWL file the person already has [0008]. None of these is a scenario. A pack gives types, not the predicates a corpus needs (81% of facts had no predicate after a pack, 0008); an OWL file presumes the modelling has been done elsewhere; and the empty start grows a vocabulary bottom-up from what the documents say, judged by questions the person writes afterwards [0061]. + +What a person who uploads company profiles, a contract chain or personnel records wants is the top-down start: the classes and properties that scenario is about, with definitions the aligner can read, units and datatypes already right, and the questions the base is meant to answer already written down. Measured on 2026-10-22 with a hand-written scenario (one class, six attributes, four documents), the modelling itself took minutes of someone who knew that an attribute needs a domain, that `注册资本` wants `number` in `万元` and that a definition is what the aligner reads; a person who does not know that builds an ontology that binds nothing, and reads the result as "extraction does not work". + +0008 decided against bundles because the standard packs align pairwise and a bundle of them is guesswork. A scenario pack is not a bundle of standard packs: it is a small vocabulary written for one kind of corpus, in the corpus's own words, with the questions that corpus is read for. + +## Decisions + +**1. A scenario pack is a small vocabulary with its questions, shipped in the binary.** A pack is one JSON file under `crates/utopia-server/packs/scenarios/`, embedded like the standard packs so an air-gapped deployment has it. It carries: classes (key, label, description, parents), attributes (key, label, description, datatype, unit, functional, the class it hangs on), relations (key, label, description, domains, ranges, temporal, functional), and competency questions (the question, optionally the expected answer and the shape it needs). Labels and descriptions are written in both `zh` and `en`; installation picks the base's `ontology_lang` for the description and the label, since the label is data the documents will be matched against and a Chinese corpus writes `企业`, not `Company` [0004]. Keys are the same in both languages and are the API's tokens. + +**2. A scenario pack installs through the editor's path, not the OWL importer.** Classes and properties are created with `ontology::create_entity_type` and `create_relation_type`, the same calls the ontology page makes, so every element is a contract in the base's own tables with no IRI [0012]: a pack element is editable and deletable like a typed one, and the export writes it as the base's own vocabulary. Questions are created `accepted` with `origin = 'person'` and the creating person as author: ticking the pack is the person's statement of what the base is for [0061 decision 1]. Installation queues `embed_ontology` so the elements have vectors before the first alignment, as the editor and the importer do. A pack installs only at creation (`CreateKbReq.scenario_packs`, beside `ontology_packs`), for the reason `install_packs` gives: a new base has no facts, so nothing needs reconciling. + +**3. Scenario packs are additive and compose with standard packs by key.** A pack's keys are chosen not to collide with each other; a key that already exists in the base (from another scenario pack or a standard pack installed first) is kept as it is and the pack's element is skipped and reported, never merged or overwritten. Domains and ranges inside a pack name the pack's own class keys; a pack may name a class another pack defines only by the same key, and an unresolved key is reported and the declaration dropped rather than the install refused. No undo, as for 0008: knowledge vetoes deleting an element with facts, and the bulk view of empty elements is the answer to a wrong pack. + +**4. A pack ships with the questions it is measured by.** The competency bench of 0061 runs each pack's questions over a corpus of its kind, and the first three packs are measured before the record is marked implemented. The two numbers of 0061 decision 5 are the pack's numbers. + +**5. Not in this record.** No regression cases in a pack: a case names a statement, and a new base has none [0061 cut 2]. No implication rules: a rule is keyed by a signature the documents have not written yet. No per-scenario view over a base's ontology: a base has one ontology, and two scenario packs installed together are one vocabulary. No scenario packs in the ontology page's import menu: that is the open question below. + +## The first three packs + +- `enterprise-profile` 企业档案: 企业, 人员, 产品, 地点; 注册资本 (number, 万元), 成立时间 (date), 员工人数 (number), 营业收入 (number, 万元), 净利润 (number, 万元), 法定代表人 (text), 注册地址 (text), 统一社会信用代码 (text); 控股 (企业 → 企业), 任职于 (人员 → 企业, state), 主要产品 (企业 → 产品), 总部位于 (企业 → 地点). Questions: a company's registered capital, revenue in a year, legal representative, holding chain. +- `contract-chain` 合同链: 合同, 当事方, 条款, 补充协议; 签订日期, 合同金额 (number, 万元), 有效期起止, 付款期限; 甲方, 乙方, 修订 (补充协议 → 合同), 约定 (合同 → 条款). Questions: which terms applied on a date, who the parties were, which amendment changed a term. +- `personnel-records` 人事档案: 人员, 部门, 岗位; 入职日期, 离职日期, 职级, 工号; 任职于 (人员 → 部门, state), 担任 (人员 → 岗位, state), 汇报给 (人员 → 人员, state), 隶属 (部门 → 部门). Questions: who held a post in a period, who reported to whom, what the base believed about a holder at a time. + +## What it costs + +- One JSON file per pack and a loader of a few hundred lines; no schema change: packs write the tables the editor writes. +- A pack's elements are the base's own, so a pack update does not reach bases that installed the old one. That is the price of decision 2 and the same as a typed element. +- Descriptions are load-bearing (0061): a pack's definitions are what the aligner reads, so a pack is written like an ontology, not like a form. + +## Dead ends + +- **A scenario as a selection of standard-pack elements.** schema.org has `Organization` but not `注册资本`; the value of a scenario is the properties, and the standard packs do not have them (0008 measured 81% unbound). +- **A scenario as an extraction prompt.** Extraction does not read the ontology [0044]; a scenario that changed the prompt would bind at write time, which 0044 measured at 4 to 9% not stated against 0 of 333. +- **Installing a scenario pack into an existing base.** `install_packs` is written for an empty base; a base with facts needs the import's baseline and reconciliation, which is a second cut if it is wanted. + +## Open questions + +- Whether a pack should be installable later from the ontology page, with the import path's reconciliation. +- Whether a pack's questions should carry expected answers for a sample corpus shipped with the pack, so the competency bench runs without a corpus of the person's own. +- Where the line between a scenario pack and a standard pack falls for industry vocabularies (FIBO, a pharma pack): size, or whether it is written in the corpus's words. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index bda0b9c92..8fc45e2d8 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -43,6 +43,7 @@ By domain; the domains are the files of [../design/](../design/README.md). **Sta | 0053 | [A phrase decision records the inputs it considered](0053-a-phrase-decision-records-the-inputs-it-considered.md) | Implemented | | | 0061 | [The ontology is proposed from the open graph and judged by its questions](0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md) | In progress | | | 0062 | [The export says what is contested](0062-the-export-says-what-is-contested.md) | Implemented | | +| 0073 | [A scenario pack is a vocabulary with its questions](0073-a-scenario-pack-is-a-vocabulary-with-its-questions.md) | In progress | | ### [extraction](../design/extraction.md) diff --git a/docs/design/ontology.md b/docs/design/ontology.md index f8df51ca0..e56ce3714 100644 --- a/docs/design/ontology.md +++ b/docs/design/ontology.md @@ -36,7 +36,14 @@ the person ticks one in the creation dialog [0008, 0009]. Five packs ship inside of about twenty rows, additive and without undo [0008]. An OWL file (Turtle or RDF/XML) is stored verbatim in the blob store and projected into the tables by a re-runnable derivation that reports what it did not project; a dry run precedes every import; individuals are never imported as facts -[0001]. The rest is typed on the ontology page or adopted from a proposal. The self-check of [0002] +[0001]. A **scenario pack** [0073] is the top-down start the standard packs are not: a small +vocabulary written for one kind of corpus (企业档案, 合同链, 人事档案 ship first), with definitions +written for the aligner, datatypes and units already right, and the competency questions that +corpus is read for. It is one JSON file embedded in the binary, bilingual, installed at creation +beside the standard packs through the editor's own calls (`scenario_packs::install`): its elements +are the base's own, without IRIs; a key that already exists is skipped and reported; its questions +are created accepted with the creating person as author; `embed_ontology` is queued. The rest is +typed on the ontology page or adopted from a proposal. The self-check of [0002] runs after an import: eight defect kinds (a relation both symmetric and asymmetric, a subclass cycle, an inverse that does not point back) are shown before any fact-level check, and a base without axioms reports zero. diff --git a/web/src/api.ts b/web/src/api.ts index 6a997ba74..661b585c1 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -109,6 +109,16 @@ export type OntologyPack = { properties: number; }; +/** 场景包（0073）：名字与说明两种语言都给，界面按自己的语言挑 */ +export type ScenarioPack = { + id: string; + name: { zh: string; en: string }; + summary: { zh: string; en: string }; + classes: number; + properties: number; + questions: number; +}; + export interface Kb { id: string; workspace_id: string; @@ -1761,6 +1771,8 @@ export const api = { visibility?: string; /** 预置本体包 id，顺序有意义：第一个会认领同名的种子类 */ ontology_packs?: string[]; + /** 场景包 id（0073）：按场景预置的类、属性和能力问题 */ + scenario_packs?: string[]; }, ) => request(`/api/v1/workspaces/${workspaceId}/kbs`, { @@ -1770,6 +1782,8 @@ export const api = { ontologyPacks: () => request<{ packs: OntologyPack[] }>("/api/v1/ontology-packs"), + scenarioPacks: () => + request<{ packs: ScenarioPack[] }>("/api/v1/scenario-packs"), kbDetail: (kbId: string) => request(`/api/v1/kbs/${kbId}`), updateKb: (kbId: string, body: Record) => diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index b436fd1c8..f7dafe2f3 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -1320,6 +1320,14 @@ export const en = { packsPick: "Search packs…", packsNone: "None — the ontology grows out of the documents", packsCount: (c: number, p: number) => `${c} classes · ${p} properties`, + scenariosLabel: "Scenario packs", + scenariosHint: + "Classes, properties and the questions to answer for one kind of corpus; " + + "installed as this base's own ontology. Optional.", + scenariosPick: "Search scenarios…", + scenariosNone: "None — build the ontology by hand later", + scenariosCount: (c: number, p: number, q: number) => + `${c} classes · ${p} properties · ${q} questions`, name: "Name", description: "Description", visibility: "Visibility", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index f2fb52b19..8156dbbb6 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -1167,6 +1167,12 @@ export const zh: Strings = { packsPick: "搜索本体包…", packsNone: "不选 —— 本体从文档里长出来", packsCount: (c: number, p: number) => `${c} 个类 · ${p} 个属性`, + scenariosLabel: "场景包", + scenariosHint: "按业务场景预置的类、属性和要回答的问题，建出来就是这个库自己的本体。可不选。", + scenariosPick: "搜索场景…", + scenariosNone: "不选 —— 之后在本体页手工建", + scenariosCount: (c: number, p: number, q: number) => + `${c} 个类 · ${p} 个属性 · ${q} 个问题`, name: "名称", description: "描述", visibility: "可见性", diff --git a/web/src/pages/MyKbs.tsx b/web/src/pages/MyKbs.tsx index 0eb321967..b30044f06 100644 --- a/web/src/pages/MyKbs.tsx +++ b/web/src/pages/MyKbs.tsx @@ -7,7 +7,7 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useNavigate, useSearch } from "@tanstack/react-router"; import { Lock, Plus, Search } from "lucide-react"; import { api, type MyKb } from "../api"; -import { S } from "../i18n"; +import { S, lang } from "../i18n"; import { useKb } from "../kb"; import { kbGroups } from "./kbGroups"; import { @@ -261,11 +261,17 @@ function NewKbModal({ // 组织、产品的起点"；量过之后本体从文档里长出来的是库自己的十几个类，而 915 个 // 类的包让每块抽取提示词从 2k 涨到 18k tokens。要包的人在这里勾 const [packs, setPacks] = useState([]); + // 场景包（0073）：按业务场景预置的类、属性和能力问题，排在标准包之后装 + const [scenarios, setScenarios] = useState([]); const available = useQuery({ queryKey: ["ontologyPacks"], queryFn: api.ontologyPacks, }); + const scenarioPacks = useQuery({ + queryKey: ["scenarioPacks"], + queryFn: api.scenarioPacks, + }); // 勾选顺序即安装顺序：第一个包的类会认领同名的种子类， // 后面的撞名才查得到对齐表。所以取消再勾会排到末尾——这是对的 @@ -281,6 +287,7 @@ function NewKbModal({ description: desc.trim() || null, visibility: restricted ? "restricted" : "open", ontology_packs: packs, + scenario_packs: scenarios, }), onSuccess: (kb) => onDone(kb.id), }); @@ -343,6 +350,24 @@ function NewKbModal({ onToggle={toggle} /> + + ({ + value: p.id, + label: p.name[lang], + hint: `${p.summary[lang]} · ${S.settings.kbs.scenariosCount(p.classes, p.properties, p.questions)}`, + }))} + placeholder={S.settings.kbs.scenariosPick} + emptyHint={S.settings.kbs.scenariosNone} + onToggle={(id) => + setScenarios((prev) => + prev.includes(id) ? prev.filter((p) => p !== id) : [...prev, id], + ) + } + /> + {create.isError && (

{(create.error as Error).message} From eadbed210f120433bb7f44a0018c5197e990dba0 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:49:14 -0400 Subject: [PATCH 05/10] An interval written from one quarter to another ends in the last month, so Q1 to Q3 runs through September instead of stopping in July (#1404) Signed-off-by: Rudy Celekli Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-server/src/time_resolution.rs | 41 ++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/crates/utopia-server/src/time_resolution.rs b/crates/utopia-server/src/time_resolution.rs index c7f386b89..69c00e1cb 100644 --- a/crates/utopia-server/src/time_resolution.rs +++ b/crates/utopia-server/src/time_resolution.rs @@ -80,6 +80,17 @@ fn parts_to_time(p: &DateParts) -> Option<(DateTime, &'static str)> { Some((Utc.from_utc_datetime(&date.and_time(time)), precision)) } +/// 区间终点。一个季度单独写时止于它的最后一个月（7 月到 9 月）；写在「到第三季度」 +/// 的那一端也是最后一个月，不能停在这一季的头一个月。 +fn parts_to_end(p: &DateParts) -> Option<(DateTime, &'static str)> { + let (t, precision) = parts_to_time(p)?; + if p.quarter.is_some() { + Some((t.checked_add_months(Months::new(2))?, "month")) + } else { + Some((t, precision)) + } +} + fn granularity_str(g: Granularity) -> &'static str { match g { Granularity::Year => "year", @@ -252,7 +263,7 @@ fn resolve_one( let Some((t, p)) = parts_to_time(from) else { return UNRESOLVED; }; - let until = to.as_ref().and_then(parts_to_time); + let until = to.as_ref().and_then(parts_to_end); // 一个季度是三个月的区间：到它最后一个月为止 if let (Some(_), None) = (from.quarter, &until) { let last = t.checked_add_months(Months::new(2)).map(|m| (m, "month")); @@ -1348,6 +1359,34 @@ mod tests { assert_eq!(r.to_p, Some("month")); } + #[test] + fn an_interval_from_one_quarter_to_another_ends_in_the_last_month() { + let quarter = |q| DateParts { + quarter: Some(q), + ..parts(2025, None, None) + }; + let i = Interpretation { + id: 0, + shape: Shape::Interval, + reference: Reference::Absolute { + from: quarter(1), + to: Some(quarter(3)), + }, + granularity: Granularity::Month, + }; + let r = resolve_one(&i, &DocumentDating::default(), &HashMap::new()); + assert_eq!( + (r.from, r.to, r.from_p, r.to_p, r.grade), + ( + Some(at("2025-01-01T00:00:00Z")), + Some(at("2025-09-01T00:00:00Z")), + Some("month"), + Some("month"), + "A" + ) + ); + } + #[test] fn two_years_later_follows_the_mention_it_names() { let mut earlier = HashMap::new(); From 1ba7d6b42385249121a38ac07440e45dfd8dfea0 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:53:39 -0400 Subject: [PATCH 06/10] A magnitude word written with a capital scales its number, so "$2.5 Billion" reads as 2.5 billion instead of 2.5 (#1400) Signed-off-by: Rudy Celekli Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-extract/src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs index e20959395..5f397d60f 100644 --- a/crates/utopia-extract/src/lib.rs +++ b/crates/utopia-extract/src/lib.rs @@ -512,7 +512,8 @@ fn scan_quantity(s: &str, strict: bool) -> Option { // 取下一个记号：中文按字（量级/货币词最长两三个字），其它按空白分词 let (tok, next) = next_token(tail); if !ate_magnitude { - if let Some(m) = magnitude(tok) { + // 标题里写成「$5 Billion」：量级词不分大小写，不然 5 billion 读成 5 + if let Some(m) = magnitude(&tok.to_ascii_lowercase()) { // Parse the written decimal with its scale in one conversion. Multiplying // an already rounded f64 needs an epsilon that can erase real fractions. // The suffix is at most three bytes (e12), so allocation stays O(num.len()). @@ -1172,6 +1173,10 @@ mod tests { normalize_attr_value("number", &json!("$5 billion")), Some(json!(5e9)) ); + assert_eq!( + normalize_attr_value("number", &json!("$2.5 Billion")), + Some(json!(2.5e9)) + ); assert_eq!( normalize_attr_value("number", &json!("52%")), Some(json!(52.0)) From 2dc58cf08e3de239a09592923d8eb04782c3119f Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:00:17 -0400 Subject: [PATCH 07/10] An approximate figure that starts with a capital is still read, so "About 350 people" becomes 350 instead of no value (#1402) Signed-off-by: Rudy Celekli Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-extract/src/lib.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs index 5f397d60f..898a64653 100644 --- a/crates/utopia-extract/src/lib.rs +++ b/crates/utopia-extract/src/lib.rs @@ -582,8 +582,11 @@ fn strip_approx(s: &str) -> &str { "~", "≈", ] { - if let Some(rest) = s.strip_prefix(p) { - s = rest.trim_start(); + // 句首的「About 350」「Approximately 1,200」：英文的约数词不分大小写 + if s.get(..p.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(p)) + { + s = s[p.len()..].trim_start(); break; } } @@ -1457,6 +1460,14 @@ mod an_attribute_value_is_read_in_its_declared_unit { // 光秃秃的数就是声明单位下的数 assert_eq!(value("number", Some("万元"), "5000"), Some(json!(5000.0))); assert_eq!(value("number", Some("人"), "约350名"), Some(json!(350.0))); + assert_eq!( + value("number", Some("人"), "Approximately 1,200"), + Some(json!(1200.0)) + ); + assert_eq!( + value("number", Some("people"), "About 350 people"), + Some(json!(350.0)) + ); // 只写到年的日期按年 assert_eq!(value("date", None, "1982年"), Some(json!("1982"))); } From 14bbb76d53e3a58419bab9ae16cebee38b66b72d Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:05:45 -0400 Subject: [PATCH 08/10] A written date with an ordinal day is read, so "March 17th, 2020" and "2nd March 2020" become those days instead of no value (#1406) Signed-off-by: Rudy Celekli Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-extract/src/lib.rs | 46 ++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs index 898a64653..b1c1f63e9 100644 --- a/crates/utopia-extract/src/lib.rs +++ b/crates/utopia-extract/src/lib.rs @@ -836,6 +836,33 @@ pub fn read_time(s: &str) -> Option<(DateTime, &'static str)> { }) } +/// 「17th」「2nd」「23rd」里的日。后缀得和数字相符：11、12、13 是 th，不是 11st。 +fn ordinal_day(w: &str) -> Option<&str> { + let (digits, suffix) = w.split_at_checked(w.len().checked_sub(2)?)?; + if digits.is_empty() || digits.len() > 2 || !digits.chars().all(|c| c.is_ascii_digit()) { + return None; + } + let n: u32 = digits.parse().ok()?; + let expected = match n % 100 { + 11..=13 => "th", + _ => match n % 10 { + 1 => "st", + 2 => "nd", + 3 => "rd", + _ => "th", + }, + }; + suffix.eq_ignore_ascii_case(expected).then_some(digits) +} + +fn day_digits(w: &str) -> Option<&str> { + if w.chars().all(|c| c.is_ascii_digit()) { + Some(w) + } else { + ordinal_day(w) + } +} + /// 没照规则 3 写、但说得清是哪一天（或哪个月）的日期（#688）。 /// /// 合同、公告里的日期多半这么写，模型常常照抄；从前这些值全被当成「不是日期」丢掉。 @@ -857,9 +884,14 @@ pub fn written_date(s: &str) -> Option<(NaiveDate, &'static str)> { .filter(|w| !w.is_empty()) .collect(); let is_number = |w: &str| w.chars().all(|c| c.is_ascii_digit()); + // 日可以带序数后缀（March 17th、2nd March）；月名不带 let (day, month, year) = match words.as_slice() { - [m, d, y] if !is_number(m) && is_number(d) && is_number(y) => (Some(*d), *m, *y), - [d, m, y] if is_number(d) && !is_number(m) && is_number(y) => (Some(*d), *m, *y), + [m, d, y] if !is_number(m) && day_digits(d).is_some() && is_number(y) => { + (day_digits(d), *m, *y) + } + [d, m, y] if day_digits(d).is_some() && !is_number(m) && is_number(y) => { + (day_digits(d), *m, *y) + } [m, y] if !is_number(m) && is_number(y) => (None, *m, *y), _ => return None, }; @@ -1237,6 +1269,7 @@ mod tests { use serde_json::json; let day = |s: &str| normalize_attr_value("date", &json!(s)); for written in [ + "March 17th, 2020", "March 17, 2020", "March 17 2020", "march 17, 2020", @@ -1253,6 +1286,15 @@ mod tests { ] { assert_eq!(day(written), Some(json!("2020-03-17")), "{written}"); } + for (written, iso) in [ + ("2nd March 2020", "2020-03-02"), + ("March 11th, 2020", "2020-03-11"), + ("March 21st, 2020", "2020-03-21"), + ("March 22nd, 2020", "2020-03-22"), + ("March 23rd, 2020", "2020-03-23"), + ] { + assert_eq!(day(written), Some(json!(iso)), "{written}"); + } // 只写到月的是月，不补日 for written in ["March 2020", "Mar. 2020", "2020年3月"] { assert_eq!(day(written), Some(json!("2020-03")), "{written}"); From d552c30c3b40e245ea4744761b5bee5e99e89e7a Mon Sep 17 00:00:00 2001 From: Nash Aschenbath Date: Sat, 24 Oct 2026 18:50:57 +0800 Subject: [PATCH 09/10] An extraction reply that arrives after its job is superseded is discarded (#1395) * fix: discard extraction replies after their job is superseded Signed-off-by: Aschenbath * fix: discard errors after an extraction job is superseded Signed-off-by: Aschenbath --------- Signed-off-by: Aschenbath Co-authored-by: Aschenbath Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-server/src/extraction.rs | 15 +- crates/utopia-server/src/extraction_open.rs | 7 +- .../src/extraction_open_known_flow_tests.rs | 216 +++++++++++++++++- 3 files changed, 230 insertions(+), 8 deletions(-) diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs index 993655e63..021405a83 100644 --- a/crates/utopia-server/src/extraction.rs +++ b/crates/utopia-server/src/extraction.rs @@ -383,7 +383,7 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow: utopia_store::documents::set_graph_status(&state.pool, document_id, "extracting").await?; state.emit_document(doc.kb_id, document_id); let await_nod = utopia_store::memory::is_memory_document(&state.pool, document_id).await?; - crate::extraction_open::run_open( + let result = crate::extraction_open::run_open( state, &doc, &kb, @@ -395,7 +395,18 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow: pushed, identity, ) - .await + .await; + // epoch 检查之后的多条写入之间仍可能发生重建；被接管的旧任务不应重试或报失败。 + // 只有明确读到新 epoch 才丢弃错误，查询失败则保留原始错误。 + if result.is_err() + && utopia_store::documents::extract_epoch(&state.pool, document_id) + .await + .is_ok_and(|epoch| epoch != my_epoch) + { + tracing::info!(document = %document_id, "discarding an extraction error after its job was superseded"); + return Ok(()); + } + result } #[cfg(test)] diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs index 3a8b378a2..b328583b6 100644 --- a/crates/utopia-server/src/extraction_open.rs +++ b/crates/utopia-server/src/extraction_open.rs @@ -98,7 +98,7 @@ enum Cut { /// `Err` has two causes: no chat model, or an empty balance (#1095). An empty balance /// stops the document, and the error keeps its chain for `alerting::hopeless`. /// -/// 被接管的检查（epoch）留在主循环顶上，不在这里：它要在调模型之前 +/// 被接管的检查（epoch）留在主循环，不在这里：调模型之前、回复落库之前都要核对。 #[allow(clippy::too_many_arguments)] async fn call_and_parse_one_chunk( state: &AppState, @@ -1100,6 +1100,11 @@ pub(crate) async fn run_open( Some(p) => p, None => continue, }; + // 模型回答期间，重建可能已删掉缓存里的实体（#1361）；旧回复不能写事实或标记分块完成。 + if utopia_store::documents::extract_epoch(pool, document_id).await? != my_epoch { + tracing::info!(%document_id, "抽取任务已被新一轮接管，落库前退出"); + return Ok(()); + } let ChunkParsed { extraction, name_vecs, diff --git a/crates/utopia-server/src/extraction_open_known_flow_tests.rs b/crates/utopia-server/src/extraction_open_known_flow_tests.rs index 737194081..2677547e4 100644 --- a/crates/utopia-server/src/extraction_open_known_flow_tests.rs +++ b/crates/utopia-server/src/extraction_open_known_flow_tests.rs @@ -15,19 +15,36 @@ use super::*; use axum::{extract::State, response::IntoResponse, routing::post, Json, Router}; use serde_json::{json, Value}; use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::sync::Notify; use uuid::Uuid; +#[derive(Default)] +struct Pause { + arrived: Notify, + resume: Notify, +} + #[derive(Clone)] struct Model { replies: Arc>>, requests: Arc>>, + pause_second: Option>, } /// 开放抽取的回复：脚本里的字符串就是 chat 流里的 `delta.content`。 /// 与 `errata_tests.rs::reply` 同一档（text/event-stream + `usage` 收尾）， /// 但 `chat_retrying_rate_limits_at` 解析流时只读 `content` 这一段。 async fn reply(State(m): State, Json(body): Json) -> impl IntoResponse { - m.requests.lock().unwrap().push(body); + let request = { + let mut requests = m.requests.lock().unwrap(); + requests.push(body); + requests.len() + }; + if let Some(pause) = m.pause_second.filter(|_| request == 2) { + pause.arrived.notify_one(); + pause.resume.notified().await; + } let text = { let mut replies = m.replies.lock().unwrap(); if replies.is_empty() { @@ -56,7 +73,11 @@ struct Fx { } impl Fx { - async fn new(replies: Vec, known_in_prompt: bool) -> anyhow::Result> { + async fn new( + replies: Vec, + known_in_prompt: bool, + pause_second: Option>, + ) -> anyhow::Result> { let Some(url) = utopia_store::test_db::url() else { return Ok(None); }; @@ -69,7 +90,7 @@ impl Fx { "INSERT INTO organizations(id,name) VALUES ('{org}','open-known-flow'); INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','open-known-flow'); INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','open-known-flow'); - INSERT INTO documents(id,kb_id,filename,sha256) VALUES ('{doc}','{kb}','acme.txt','x'); + INSERT INTO documents(id,kb_id,filename,sha256,status) VALUES ('{doc}','{kb}','acme.txt','x','ready'); INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES ('{chunk0}','{kb}','{doc}',0,'Acme is based in London.'), ('{chunk1}','{kb}','{doc}',1,'Acme runs the harbour facility.');" @@ -79,6 +100,7 @@ impl Fx { let model = Model { replies: Arc::new(Mutex::new(replies)), requests: Arc::new(Mutex::new(Vec::new())), + pause_second, }; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let endpoint = format!("http://{}", listener.local_addr()?); @@ -121,6 +143,12 @@ impl Fx { } async fn cleanup(&self) -> anyhow::Result<()> { + self._server.abort(); + sqlx::query("DELETE FROM jobs WHERE payload->>'document_id'=$1 OR payload->>'kb_id'=$2") + .bind(self.doc.to_string()) + .bind(self.kb.to_string()) + .execute(&self.pool) + .await?; sqlx::query("DELETE FROM organizations WHERE id=$1") .bind(self.org) .execute(&self.pool) @@ -185,7 +213,7 @@ async fn extract_and_read_the_second_prompt(f: &Fx) -> anyhow::Result { #[tokio::test] async fn a_thing_listed_by_an_earlier_chunk_reaches_a_later_one_via_known() -> anyhow::Result<()> { - let Some(f) = Fx::new(replies(), true).await? else { + let Some(f) = Fx::new(replies(), true, None).await? else { return Ok(()); }; let run = extract_and_read_the_second_prompt(&f).await; @@ -202,7 +230,7 @@ async fn a_thing_listed_by_an_earlier_chunk_reaches_a_later_one_via_known() -> a #[tokio::test] async fn with_known_left_out_of_the_prompt_a_later_chunk_still_lands_on_the_same_entity( ) -> anyhow::Result<()> { - let Some(f) = Fx::new(replies(), false).await? else { + let Some(f) = Fx::new(replies(), false, None).await? else { return Ok(()); }; let run = extract_and_read_the_second_prompt(&f).await; @@ -214,3 +242,181 @@ async fn with_known_left_out_of_the_prompt_a_later_chunk_still_lands_on_the_same ); Ok(()) } + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Interruption { + RebuildInFlight, + ReextractInFlight, + RebuildDuringApply, + PurgeDuringApply, +} + +// facts 上的触发器 DDL 会锁整张表，两条屏障用例必须串行到清理结束。 +static FACTS_TRIGGER_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +/// 第一块留下缓存的 Acme id；重建会删掉它，单篇重抽则保留实体、重置分块。 +/// 写入阶段的屏障放在真正的 INSERT 上，确保重建发生在 epoch 检查之后。 +async fn interrupt_the_second_chunk(interruption: Interruption) -> anyhow::Result<()> { + let during_apply = matches!( + interruption, + Interruption::RebuildDuringApply | Interruption::PurgeDuringApply + ); + let _trigger_guard = if during_apply { + Some(FACTS_TRIGGER_LOCK.lock().await) + } else { + None + }; + let pause = Arc::new(Pause::default()); + let Some(f) = Fx::new(replies(), true, Some(pause.clone())).await? else { + return Ok(()); + }; + let rebuild = interruption != Interruption::ReextractInFlight; + let barrier = format!("pause_extraction_{}", f.doc.simple()); + let mut gate = None; + let mut gate_pid = 0i32; + if during_apply { + let mut tx = f.pool.begin().await?; + gate_pid = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *tx) + .await?; + sqlx::query("SELECT pg_advisory_xact_lock(1361, $1)") + .bind(gate_pid) + .execute(&mut *tx) + .await?; + gate = Some(tx); + } + let run = tokio::time::timeout(Duration::from_secs(30), async { + let extraction = crate::extraction::extract_document(&f.state, f.doc, Proposer::default()); + tokio::pin!(extraction); + tokio::select! { + result = &mut extraction => { + result?; + anyhow::bail!("extraction finished before the second reply was released"); + } + _ = pause.arrived.notified() => {} + } + if during_apply { + // 标识符和条件都来自本夹具的 UUID；别的知识库不会等这把锁。 + sqlx::raw_sql(&format!( + "CREATE FUNCTION {barrier}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.kb_id = '{}'::uuid THEN + PERFORM pg_advisory_xact_lock(1361, {gate_pid}); + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER {barrier} BEFORE INSERT ON facts + FOR EACH ROW EXECUTE FUNCTION {barrier}();", + f.kb + )) + .execute(&f.pool) + .await?; + pause.resume.notify_one(); + loop { + let blocked: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE $1 = ANY(pg_blocking_pids(pid)))", + ) + .bind(gate_pid) + .fetch_one(&f.pool) + .await?; + if blocked { + break; + } + tokio::select! { + result = &mut extraction => { + result?; + anyhow::bail!("extraction finished before the INSERT barrier"); + } + _ = tokio::time::sleep(Duration::from_millis(10)) => {} + } + } + } + if rebuild { + utopia_store::graph::purge_graph(&f.pool, f.kb).await?; + if interruption != Interruption::PurgeDuringApply { + let queued = utopia_store::documents::queue_extraction(&f.pool, f.kb, None).await?; + anyhow::ensure!(queued == vec![f.doc], "rebuild must queue the document"); + } + } else { + utopia_store::documents::queue_extraction_one(&f.pool, f.doc).await?; + } + if let Some(tx) = gate.take() { + tx.rollback().await?; + } + pause.resume.notify_one(); + let result = extraction.await; + if interruption == Interruption::PurgeDuringApply { + // 同样的外键失败，没有新 epoch 就仍是当前任务的错误，不能一律吞掉。 + let error = result.err().ok_or_else(|| anyhow::anyhow!("current job lost its error"))?; + anyhow::ensure!(format!("{error:#}").contains("facts_subject_same_kb"), "{error:#}"); + return Ok(()); + } + result?; + + let doc = utopia_store::documents::get(&f.pool, f.doc).await?; + anyhow::ensure!(doc.graph_status == "queued", "the new job keeps its status"); + anyhow::ensure!(doc.graph_error.is_none(), "superseded work is not a failure"); + let extracted: i64 = sqlx::query_scalar( + "SELECT count(*) FROM chunks WHERE document_id=$1 AND extracted_at IS NOT NULL", + ) + .bind(f.doc) + .fetch_one(&f.pool) + .await?; + anyhow::ensure!(extracted == 0, "the old reply must not mark a chunk extracted"); + let statements: i64 = + sqlx::query_scalar("SELECT count(*) FROM facts WHERE kb_id=$1 AND layer='open'") + .bind(f.kb) + .fetch_one(&f.pool) + .await?; + anyhow::ensure!( + statements == if rebuild { 0 } else { 1 }, + "only statements from before re-extract may remain, got {statements}" + ); + let jobs: Vec<(String, String)> = sqlx::query_as( + "SELECT kind, status FROM jobs WHERE payload->>'document_id'=$1 OR payload->>'kb_id'=$2", + ) + .bind(f.doc.to_string()) + .bind(f.kb.to_string()) + .fetch_all(&f.pool) + .await?; + anyhow::ensure!( + jobs == vec![("extract_document".into(), "queued".into())], + "only the replacement job remains, with no follow-up jobs: {jobs:?}" + ); + Ok::<_, anyhow::Error>(()) + }) + .await; + pause.resume.notify_one(); + if let Some(tx) = gate.take() { + tx.rollback().await?; + } + if during_apply { + sqlx::raw_sql(&format!( + "DROP TRIGGER IF EXISTS {barrier} ON facts; DROP FUNCTION IF EXISTS {barrier}();" + )) + .execute(&f.pool) + .await?; + } + f.cleanup().await?; + run? +} + +#[tokio::test] +async fn a_rebuild_discards_an_extraction_reply_that_is_already_in_flight() -> anyhow::Result<()> { + interrupt_the_second_chunk(Interruption::RebuildInFlight).await +} + +#[tokio::test] +async fn reextract_discards_an_extraction_reply_that_is_already_in_flight() -> anyhow::Result<()> { + interrupt_the_second_chunk(Interruption::ReextractInFlight).await +} + +#[tokio::test] +async fn a_rebuild_after_the_epoch_check_does_not_fail_the_superseded_job() -> anyhow::Result<()> { + interrupt_the_second_chunk(Interruption::RebuildDuringApply).await +} + +#[tokio::test] +async fn an_apply_error_without_a_replacement_epoch_is_still_returned() -> anyhow::Result<()> { + interrupt_the_second_chunk(Interruption::PurgeDuringApply).await +} From 03562d0f66a864d1881b1a9fe03f53285624a298 Mon Sep 17 00:00:00 2001 From: Nash Aschenbath Date: Sat, 24 Oct 2026 18:56:32 +0800 Subject: [PATCH 10/10] A model that refuses max_tokens or a non-default temperature is retried with max_completion_tokens and no temperature, learned once per client (#1394) * fix: let model jobs recover from unsupported request parameters Signed-off-by: Aschenbath * Keep the model compatibility rationale in Chinese Signed-off-by: Nash Aschenbath Co-authored-by: Schro-code <78734653+Schro-code@users.noreply.github.com> --- crates/utopia-llm/src/lib.rs | 261 ++++++++++++++++++++++++++++++++--- 1 file changed, 243 insertions(+), 18 deletions(-) diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rs index aa2bfe67a..bd95945d0 100644 --- a/crates/utopia-llm/src/lib.rs +++ b/crates/utopia-llm/src/lib.rs @@ -446,6 +446,9 @@ pub struct LlmClient { /// 上限后记在这里，同一个客户端（含它的克隆）此后都按它发，一个进程只吃一次拒绝 /// （#891，#892 评审）。只降不升 completion_ceiling: std::sync::Arc, + /// 端点拒绝过的参数记在客户端上，克隆也共用；下一块不再为同一种请求吃一次 400（#1376）。 + use_max_completion_tokens: std::sync::Arc, + suppress_temperature: std::sync::Arc, /// 端点报过的上下文窗口（token），起初是 `u32::MAX`。同一个客户端（含它的克隆）此后 /// 都按它算历史预算，只降不升；对话客户端按工作区留着，所以下一轮也记得（#964） context_window: std::sync::Arc, @@ -587,6 +590,10 @@ impl LlmClient { completion_ceiling: std::sync::Arc::new(std::sync::atomic::AtomicU32::new( MAX_COMPLETION_TOKENS, )), + use_max_completion_tokens: std::sync::Arc::new(std::sync::atomic::AtomicBool::new( + false, + )), + suppress_temperature: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } @@ -629,22 +636,39 @@ impl LlmClient { messages: &[ChatMessage], temperature: Option, ) -> anyhow::Result { + use std::sync::atomic::Ordering; let mut body = self.with_effort(json!({ "model": self.model, "messages": messages, "stream": false })); - if let Some(t) = temperature { + let mut dropped_temperature = self.suppress_temperature.load(Ordering::Relaxed); + if let Some(t) = temperature.filter(|_| !dropped_temperature) { body["temperature"] = json!(t); } - let resp = self - .request("/chat/completions") - .json(&body) - .send() - .await - .map_err(Unreachable)?; - let status = resp.status(); - let retry_after = retry_after_of(resp.headers()); - if !status.is_success() { - return Err(response_failure("LLM", status, retry_after, resp).await?); - } + let resp = loop { + let resp = self + .request("/chat/completions") + .json(&body) + .send() + .await + .map_err(Unreachable)?; + let status = resp.status(); + let retry_after = retry_after_of(resp.headers()); + if status.is_success() { + break resp; + } + let raw = resp.text().await.map_err(Unreachable)?; + let parsed = serde_json::from_str(&raw).unwrap_or_default(); + if status == reqwest::StatusCode::BAD_REQUEST + && !dropped_temperature + && temperature.is_some() + && says_temperature_unsupported(&parsed, &raw) + { + self.suppress_temperature.store(true, Ordering::Relaxed); + body.as_object_mut().unwrap().remove("temperature"); + dropped_temperature = true; + continue; + } + return Err(failure("LLM", status, retry_after, &parsed, &raw)); + }; let body: serde_json::Value = resp.json().await.map_err(Unreachable)?; log_usage(&self.model, &call_label(messages), &body); body["choices"][0]["message"]["content"] @@ -670,6 +694,7 @@ impl LlmClient { messages: &[ChatMessage], temperature: Option, ) -> anyhow::Result { + use std::sync::atomic::Ordering; let body = json!({ "model": self.model, "messages": messages, @@ -677,11 +702,10 @@ impl LlmClient { // 用量随最后一帧回来：比较两次运行的第一件事是看 completion token， // 换成流式不能把这个数弄丢 "stream_options": { "include_usage": true }, - // 上限归我们，不归端点的默认值（[`MAX_COMPLETION_TOKENS`]） - "max_tokens": MAX_COMPLETION_TOKENS, }); let mut body = self.with_effort(body); - if let Some(t) = temperature { + let mut dropped_temperature = self.suppress_temperature.load(Ordering::Relaxed); + if let Some(t) = temperature.filter(|_| !dropped_temperature) { body["temperature"] = json!(t); } // One retry per call, and only downwards: an endpoint that refuses our ceiling has @@ -689,11 +713,18 @@ impl LlmClient { // to the endpoint default this field exists to displace (#760). The lowered // ceiling is kept on the client, so the next call starts from it instead of // paying the 400 again (extraction calls once per chunk). - use std::sync::atomic::Ordering; let mut ceiling = self.completion_ceiling.load(Ordering::Relaxed); let mut lowered = false; + // 每种调整每次调用只试一次：网关可能把我们已删掉的参数重新加回来（#1232）。 + let mut renamed = self.use_max_completion_tokens.load(Ordering::Relaxed); let resp = loop { - body["max_tokens"] = json!(ceiling); + let limit_key = if renamed { + "max_completion_tokens" + } else { + "max_tokens" + }; + // 上限归我们，不归端点的默认值（[`MAX_COMPLETION_TOKENS`]）；改字段名也不能丢掉它。 + body[limit_key] = json!(ceiling); let resp = self .request("/chat/completions") .json(&body) @@ -706,7 +737,28 @@ impl LlmClient { break resp; } let raw = resp.text().await.map_err(Unreachable)?; - let parsed = serde_json::from_str(&raw).unwrap_or_default(); + let parsed: serde_json::Value = serde_json::from_str(&raw).unwrap_or_default(); + if status == reqwest::StatusCode::BAD_REQUEST + && !renamed + && parsed["error"]["param"] == "max_tokens" + && parsed["error"]["code"] == "unsupported_parameter" + { + self.use_max_completion_tokens + .store(true, Ordering::Relaxed); + body.as_object_mut().unwrap().remove("max_tokens"); + renamed = true; + continue; + } + if status == reqwest::StatusCode::BAD_REQUEST + && !dropped_temperature + && temperature.is_some() + && says_temperature_unsupported(&parsed, &raw) + { + self.suppress_temperature.store(true, Ordering::Relaxed); + body.as_object_mut().unwrap().remove("temperature"); + dropped_temperature = true; + continue; + } if status == reqwest::StatusCode::BAD_REQUEST && !lowered && !says_context_too_long(&parsed, &err_detail(&parsed, &raw)) @@ -1298,6 +1350,24 @@ fn err_detail(body: &serde_json::Value, raw: &str) -> String { .unwrap_or_else(|| "unknown error".to_string()) } +/// 先认参数名和错误码；Azure 可能只给一段消息。笼统的温度无效不足以丢掉调用方的设置。 +fn says_temperature_unsupported(body: &serde_json::Value, raw: &str) -> bool { + if let Some(parameter) = body["error"]["param"].as_str() { + if parameter != "temperature" { + return false; + } + if matches!( + body["error"]["code"].as_str(), + Some("unsupported_parameter" | "unsupported_value") + ) { + return true; + } + } + let detail = err_detail(body, raw).to_lowercase(); + detail.contains("temperature") + && (detail.contains("not support") || detail.contains("unsupported")) +} + /// 端点说不说「带工具的请求不收推理强度」。只有措辞可认：Azure 给的 code 是笼统的 /// `invalid_request_error`。两样都得提到——只说工具或只说推理强度的 400 是别的毛病 fn says_tools_refuse_effort(detail: &str) -> bool { @@ -2597,6 +2667,161 @@ data: {\"choices\":[{\"delta\":{\"content\":\"tail\"},\"finish_reason\":\"stop\" assert_eq!(sent_max_tokens(&sent[3]), json!(16_384), "so does a clone"); } + fn complete_chat_response(streaming: bool) -> wiremock::ResponseTemplate { + let response = wiremock::ResponseTemplate::new(200); + if streaming { + response.set_body_raw( + "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", + "text/event-stream", + ) + } else { + response.set_body_json(json!({"choices": [{"message": {"content": "ok"}}]})) + } + } + + async fn ask_at_zero(client: &LlmClient, streaming: bool) -> anyhow::Result { + let messages = [ChatMessage { + role: "user".into(), + content: "hi".into(), + }]; + if streaming { + Ok(client.chat_at_streaming(&messages, Some(0.0)).await?.text) + } else { + client.chat_at(&messages, Some(0.0)).await + } + } + + #[tokio::test] + async fn refused_parameters_and_a_lower_ceiling_are_learned_and_shared_with_clones() { + use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(|request: &Request| { + let body: serde_json::Value = request.body_json().unwrap(); + if body.get("max_tokens").is_some() { + ResponseTemplate::new(400).set_body_json(json!({"error": { + "param": "max_tokens", "code": "unsupported_parameter", + "message": "Use max_completion_tokens instead." + }})) + } else if body["max_completion_tokens"] != 16_384 { + ResponseTemplate::new(400).set_body_json(json!({"error": { + "message": "max_completion_tokens must be at most 16384 tokens" + }})) + } else if body.get("temperature").is_some() { + ResponseTemplate::new(400).set_body_json(json!({"error": { + "param": "temperature", "code": "unsupported_value", + "message": "Use the default sampling settings." + }})) + } else { + complete_chat_response(true) + } + }) + .expect(6) + .mount(&server) + .await; + let client = LlmClient::new(&server.uri(), None, "m"); + let cloned = client.clone(); + for caller in [&client, &client, &cloned] { + assert_eq!(ask_at_zero(caller, true).await.unwrap(), "ok"); + } + let sent: Vec = server + .received_requests() + .await + .unwrap() + .iter() + .map(|r| r.body_json().unwrap()) + .collect(); + assert_eq!(sent[0]["max_tokens"], MAX_COMPLETION_TOKENS); + assert_eq!(sent[1]["max_completion_tokens"], MAX_COMPLETION_TOKENS); + for body in &sent[1..] { + assert!(body.get("max_tokens").is_none()); + } + for body in &sent[2..] { + assert_eq!(body["max_completion_tokens"], 16_384); + } + assert_eq!( + sent.iter() + .map(|b| b.get("temperature").is_some()) + .collect::>(), + [true, true, true, false, false, false] + ); + } + + #[tokio::test] + async fn a_refused_temperature_is_learned_across_streaming_calls_and_clones() { + use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate}; + for refusal in [ + json!({"error": {"param": "temperature", "code": "unsupported_value", + "message": "Only the default sampling settings are supported."}}), + json!({"error": {"param": "temperature", "code": "unsupported_parameter", + "message": "Use the default sampling settings."}}), + json!({"message": "Unsupported value: 'temperature' does not support 0 with this model. Only the default (1) value is supported."}), + ] { + for streaming in [false, true] { + let server = MockServer::start().await; + let refusal = refusal.clone(); + Mock::given(method("POST")) + .respond_with(move |request: &Request| { + let body: serde_json::Value = request.body_json().unwrap(); + if body.get("temperature").is_some() { + ResponseTemplate::new(400).set_body_json(&refusal) + } else { + complete_chat_response(body["stream"] == true) + } + }) + .expect(4) + .mount(&server) + .await; + let client = LlmClient::new(&server.uri(), None, "m"); + let cloned = client.clone(); + assert_eq!(ask_at_zero(&client, streaming).await.unwrap(), "ok"); + assert_eq!(ask_at_zero(&cloned, !streaming).await.unwrap(), "ok"); + assert_eq!(ask_at_zero(&client, streaming).await.unwrap(), "ok"); + let temperatures: Vec = server + .received_requests() + .await + .unwrap() + .iter() + .map(|r| { + r.body_json::() + .unwrap() + .get("temperature") + .is_some() + }) + .collect(); + assert_eq!(temperatures, [true, false, false, false]); + } + } + } + + #[tokio::test] + async fn each_compatibility_adjustment_is_tried_only_once_per_call() { + use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate}; + for (streaming, parameter) in [ + (true, "max_tokens"), + (true, "temperature"), + (false, "temperature"), + ] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400).set_body_json(json!({"error": { + "param": parameter, "code": "unsupported_parameter", + "message": "Repeated refusal" + }}))) + // 第三次会拿到 404，让没有次数上限的重试立即暴露。 + .up_to_n_times(2) + .expect(2) + .mount(&server) + .await; + let client = LlmClient::new(&server.uri(), None, "m"); + let err = ask_at_zero(&client, streaming).await.unwrap_err(); + let refusal = rejected(&err).unwrap(); + assert_eq!(refusal.status, 400); + assert_eq!(refusal.detail, "Repeated refusal"); + assert_eq!(server.received_requests().await.unwrap().len(), 2); + } + } + /// Only a number the message calls tokens is a ceiling; the rest of the digits in a /// refusal are a model name, a status code, or our own request echoed back. #[test]