From 5c70f11bfeea0e795ba952348d2589831570159b Mon Sep 17 00:00:00 2001 From: Sunshine Date: Fri, 16 Oct 2026 07:48:59 -0400 Subject: [PATCH] perform second look at pages which language couldn't be detected --- ARCHITECTURE.md | 2 +- src/lang.rs | 63 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 380ca17..6c56f78 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -264,7 +264,7 @@ Sources that aren't web pages are turned into pages at import time. A Hacker New | `links` | string, multi-valued | yes | Outgoing links; `link:` with a full URL | | `link_hosts` | string, multi-valued | no | Every level of every linked host, reversed (like `host_rev`); `link:` with a site | | `link_ext` | string, multi-valued | no | The file extensions of the links (`pdf`, `zip`, …); `contains:` | -| `lang` | string | no | The page's language, an ISO 639-3 code (`deu`), detected from its title and the first 1,000 characters of its text when it's indexed (`whatlang`, 70 languages); none when the text is too short or too mixed to tell. Not stored with the page: it's worked out again whenever the index is built. `lang:` | +| `lang` | string | no | The page's language, an ISO 639-3 code (`deu`), detected from its title and the first 1,000 characters of its text when it's indexed (`whatlang`, 70 languages) — or, if that can't tell and the page is longer, from 20,000 characters out of its middle (past the menus and banners at the top); none when the text is too short or too mixed to tell. Not stored with the page: it's worked out again whenever the index is built. `lang:` | | `crawled` | date | yes | When the page was captured | | `import_id` | u64, indexed | yes | The import this version of the page came from | diff --git a/src/lang.rs b/src/lang.rs index 7576be6..945b41d 100644 --- a/src/lang.rs +++ b/src/lang.rs @@ -11,9 +11,13 @@ /// How much of a page is read to tell its language: enough to be sure, /// little enough not to slow imports down. const SAMPLE_CHARS: usize = 1_000; +/// When that isn't enough to tell (menus, banners, an interface in another +/// language at the top), this much from the middle of a longer page. +const SECOND_SAMPLE_CHARS: usize = 20_000; /// The language of a page, as an ISO 639-3 code, or `None` when its text is -/// too short or too mixed to tell. +/// too short or too mixed to tell. The title and the start of the page +/// first; failing that, if there's more to the page, its middle. pub fn detect(title: Option<&str>, body: &str) -> Option<&'static str> { let mut sample = String::with_capacity(SAMPLE_CHARS * 2); if let Some(title) = title { @@ -21,10 +25,35 @@ pub fn detect(title: Option<&str>, body: &str) -> Option<&'static str> { sample.push('\n'); } sample.extend(body.chars().take(SAMPLE_CHARS)); - let info = whatlang::detect(&sample)?; + if let Some(code) = reliable(&sample) { + return Some(code); + } + let length = body.chars().count(); + if length <= SAMPLE_CHARS { + // The whole page was read already. + return None; + } + reliable(middle(body, length, SECOND_SAMPLE_CHARS)) +} + +fn reliable(text: &str) -> Option<&'static str> { + let info = whatlang::detect(text)?; info.is_reliable().then(|| info.lang().code()) } +/// `chars` characters from the middle of `text` (`length` characters long), +/// or all of it if it isn't longer. +fn middle(text: &str, length: usize, chars: usize) -> &str { + if length <= chars { + return text; + } + let skip = (length - chars) / 2; + let mut ends = text.char_indices().map(|(i, _)| i).skip(skip); + let start = ends.next().unwrap_or(text.len()); + let end = ends.nth(chars - 1).unwrap_or(text.len()); + &text[start..end] +} + /// Other ways to write the languages `whatlang` knows: ISO 639-1 codes, /// ISO 639-2/B codes, and common names that aren't its own. Its codes /// (ISO 639-3) and its English and native names work too, without being @@ -194,4 +223,34 @@ mod tests { assert_eq!(detect(Some("Welcome"), ""), None); assert_eq!(detect(None, ""), None); } + + #[test] + fn a_second_look() { + let german = "Das ist ein ganz normaler deutscher Satz über das Wetter in Berlin \ + und die vielen schönen Häuser der Stadt. "; + // A start with nothing to tell by (a menu of numbers and bars), then + // the page: the first sample can't say, the middle can. + let menu = "| 1 | 2 | 3 | 4 | 5 | ".repeat(60); + assert!(menu.chars().count() > SAMPLE_CHARS); + assert_eq!(detect(None, &menu), None); + let page = format!("{menu}{}", german.repeat(40)); + assert_eq!(detect(None, &page), Some("deu")); + // A long page: its middle, not its start or end. + let page = format!( + "{}{}{}", + menu.repeat(30), + german.repeat(200), + menu.repeat(30) + ); + assert!(page.chars().count() > SECOND_SAMPLE_CHARS * 2); + assert_eq!(detect(None, &page), Some("deu")); + } + + #[test] + fn middles() { + assert_eq!(middle("abcdefghij", 10, 4), "defg"); + assert_eq!(middle("abc", 3, 4), "abc"); + assert_eq!(middle("ünïcødé", 7, 3), "ïcø"); + assert_eq!(middle("ünïcødé", 7, 7), "ünïcødé"); + } }