Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 63 additions & 5 deletions crates/utopia-extract/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,8 @@ fn scan_quantity(s: &str, strict: bool) -> Option<Quantity> {
// 取下一个记号:中文按字(量级/货币词最长两三个字),其它按空白分词
let (tok, next) = next_token(tail);
if !ate_magnitude {
if let Some(m) = magnitude(tok) {
// 标题里写成「$5 Billion」:量级词不分大小写,不然 5 billion 读成 5
if let Some(m) = magnitude(&tok.to_ascii_lowercase()) {
// Parse the written decimal with its scale in one conversion. Multiplying
// an already rounded f64 needs an epsilon that can erase real fractions.
// The suffix is at most three bytes (e12), so allocation stays O(num.len()).
Expand Down Expand Up @@ -581,8 +582,11 @@ fn strip_approx(s: &str) -> &str {
"~",
"≈",
] {
if let Some(rest) = s.strip_prefix(p) {
s = rest.trim_start();
// 句首的「About 350」「Approximately 1,200」:英文的约数词不分大小写
if s.get(..p.len())
.is_some_and(|head| head.eq_ignore_ascii_case(p))
{
s = s[p.len()..].trim_start();
break;
}
}
Expand Down Expand Up @@ -832,6 +836,33 @@ pub fn read_time(s: &str) -> Option<(DateTime<Utc>, &'static str)> {
})
}

/// 「17th」「2nd」「23rd」里的日。后缀得和数字相符:11、12、13 是 th,不是 11st。
fn ordinal_day(w: &str) -> Option<&str> {
let (digits, suffix) = w.split_at_checked(w.len().checked_sub(2)?)?;
if digits.is_empty() || digits.len() > 2 || !digits.chars().all(|c| c.is_ascii_digit()) {
return None;
}
let n: u32 = digits.parse().ok()?;
let expected = match n % 100 {
11..=13 => "th",
_ => match n % 10 {
1 => "st",
2 => "nd",
3 => "rd",
_ => "th",
},
};
suffix.eq_ignore_ascii_case(expected).then_some(digits)
}

fn day_digits(w: &str) -> Option<&str> {
if w.chars().all(|c| c.is_ascii_digit()) {
Some(w)
} else {
ordinal_day(w)
}
}

/// 没照规则 3 写、但说得清是哪一天(或哪个月)的日期(#688)。
///
/// 合同、公告里的日期多半这么写,模型常常照抄;从前这些值全被当成「不是日期」丢掉。
Expand All @@ -853,9 +884,14 @@ pub fn written_date(s: &str) -> Option<(NaiveDate, &'static str)> {
.filter(|w| !w.is_empty())
.collect();
let is_number = |w: &str| w.chars().all(|c| c.is_ascii_digit());
// 日可以带序数后缀(March 17th、2nd March);月名不带
let (day, month, year) = match words.as_slice() {
[m, d, y] if !is_number(m) && is_number(d) && is_number(y) => (Some(*d), *m, *y),
[d, m, y] if is_number(d) && !is_number(m) && is_number(y) => (Some(*d), *m, *y),
[m, d, y] if !is_number(m) && day_digits(d).is_some() && is_number(y) => {
(day_digits(d), *m, *y)
}
[d, m, y] if day_digits(d).is_some() && !is_number(m) && is_number(y) => {
(day_digits(d), *m, *y)
}
[m, y] if !is_number(m) && is_number(y) => (None, *m, *y),
_ => return None,
};
Expand Down Expand Up @@ -1172,6 +1208,10 @@ mod tests {
normalize_attr_value("number", &json!("$5 billion")),
Some(json!(5e9))
);
assert_eq!(
normalize_attr_value("number", &json!("$2.5 Billion")),
Some(json!(2.5e9))
);
assert_eq!(
normalize_attr_value("number", &json!("52%")),
Some(json!(52.0))
Expand Down Expand Up @@ -1229,6 +1269,7 @@ mod tests {
use serde_json::json;
let day = |s: &str| normalize_attr_value("date", &json!(s));
for written in [
"March 17th, 2020",
"March 17, 2020",
"March 17 2020",
"march 17, 2020",
Expand All @@ -1245,6 +1286,15 @@ mod tests {
] {
assert_eq!(day(written), Some(json!("2020-03-17")), "{written}");
}
for (written, iso) in [
("2nd March 2020", "2020-03-02"),
("March 11th, 2020", "2020-03-11"),
("March 21st, 2020", "2020-03-21"),
("March 22nd, 2020", "2020-03-22"),
("March 23rd, 2020", "2020-03-23"),
] {
assert_eq!(day(written), Some(json!(iso)), "{written}");
}
// 只写到月的是月,不补日
for written in ["March 2020", "Mar. 2020", "2020年3月"] {
assert_eq!(day(written), Some(json!("2020-03")), "{written}");
Expand Down Expand Up @@ -1452,6 +1502,14 @@ mod an_attribute_value_is_read_in_its_declared_unit {
// 光秃秃的数就是声明单位下的数
assert_eq!(value("number", Some("万元"), "5000"), Some(json!(5000.0)));
assert_eq!(value("number", Some("人"), "约350名"), Some(json!(350.0)));
assert_eq!(
value("number", Some("人"), "Approximately 1,200"),
Some(json!(1200.0))
);
assert_eq!(
value("number", Some("people"), "About 350 people"),
Some(json!(350.0))
);
// 只写到年的日期按年
assert_eq!(value("date", None, "1982年"), Some(json!("1982")));
}
Expand Down
5 changes: 4 additions & 1 deletion crates/utopia-extract/src/phrase_align.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,10 @@ phrases may be in any language.\n\
3. Answer null when no candidate fits; when the sentences show the phrase meaning different \
things under this signature; when the phrase only reports, introduces or evaluates (\"said\", \
\"announced\", \"is expected to\") unless a candidate is about that; or when a value is not \
what the attribute measures (a share count is not revenue, a date is not an amount).\n\
what the attribute measures (a share count is not revenue, a date is not an amount). A text \
attribute may be a candidate for an item whose object is a named thing: choose it when the \
object is named only to carry that attribute's value for the subject (a legal representative's \
name, a registered address), and the object's name becomes the value.\n\
4. The direction follows the definition: for \"X —is a subsidiary of→ Y\" subsidiary_of is \
forward; for \"X —owns→ Y\", if subsidiary_of is the only fitting candidate, it is reverse.\n\
5. Never invent a key, never answer with a label, never choose for an item a key that is not \
Expand Down
Loading
Loading