Perform second look at pages which language couldn't be detected

ARCHITECTURE.md

@@ -264,7 +264,7 @@ Sources that aren't web pages are turned into pages at import time. A Hacker New
 | `links` | string, multi-valued | yes | Outgoing links; `link:` with a full URL |
 | `link_hosts` | string, multi-valued | no | Every level of every linked host, reversed (like `host_rev`); `link:` with a site |
 | `link_ext` | string, multi-valued | no | The file extensions of the links (`pdf`, `zip`, …); `contains:` |
-| `lang` | string | no | The page's language, an ISO 639-3 code (`deu`), detected from its title and the first 1,000 characters of its text when it's indexed (`whatlang`, 70 languages); none when the text is too short or too mixed to tell. Not stored with the page: it's worked out again whenever the index is built. `lang:` |
+| `lang` | string | no | The page's language, an ISO 639-3 code (`deu`), detected from its title and the first 1,000 characters of its text when it's indexed (`whatlang`, 70 languages) — or, if that can't tell and the page is longer, from 20,000 characters out of its middle (past the menus and banners at the top); none when the text is too short or too mixed to tell. Not stored with the page: it's worked out again whenever the index is built. `lang:` |
 | `crawled` | date | yes | When the page was captured |
 | `import_id` | u64, indexed | yes | The import this version of the page came from |
 

src/lang.rs

@@ -11,20 +11,49 @@
 /// How much of a page is read to tell its language: enough to be sure,
 /// little enough not to slow imports down.
 const SAMPLE_CHARS: usize = 1_000;
+/// When that isn't enough to tell (menus, banners, an interface in another
+/// language at the top), this much from the middle of a longer page.
+const SECOND_SAMPLE_CHARS: usize = 20_000;
 
 /// The language of a page, as an ISO 639-3 code, or `None` when its text is
-/// too short or too mixed to tell.
+/// too short or too mixed to tell. The title and the start of the page
+/// first; failing that, if there's more to the page, its middle.
 pub fn detect(title: Option<&str>, body: &str) -> Option<&'static str> {
     let mut sample = String::with_capacity(SAMPLE_CHARS * 2);
     if let Some(title) = title {
         sample.push_str(title);
         sample.push('\n');
     }
     sample.extend(body.chars().take(SAMPLE_CHARS));
-    let info = whatlang::detect(&sample)?;
+    if let Some(code) = reliable(&sample) {
+        return Some(code);
+    }
+    let length = body.chars().count();
+    if length <= SAMPLE_CHARS {
+        // The whole page was read already.
+        return None;
+    }
+    reliable(middle(body, length, SECOND_SAMPLE_CHARS))
+}
+
+fn reliable(text: &str) -> Option<&'static str> {
+    let info = whatlang::detect(text)?;
     info.is_reliable().then(|| info.lang().code())
 }
 
+/// `chars` characters from the middle of `text` (`length` characters long),
+/// or all of it if it isn't longer.
+fn middle(text: &str, length: usize, chars: usize) -> &str {
+    if length <= chars {
+        return text;
+    }
+    let skip = (length - chars) / 2;
+    let mut ends = text.char_indices().map(|(i, _)| i).skip(skip);
+    let start = ends.next().unwrap_or(text.len());
+    let end = ends.nth(chars - 1).unwrap_or(text.len());
+    &text[start..end]
+}
+
 /// Other ways to write the languages `whatlang` knows: ISO 639-1 codes,
 /// ISO 639-2/B codes, and common names that aren't its own. Its codes
 /// (ISO 639-3) and its English and native names work too, without being
@@ -194,4 +223,34 @@ mod tests {
         assert_eq!(detect(Some("Welcome"), ""), None);
         assert_eq!(detect(None, ""), None);
     }
+
+    #[test]
+    fn a_second_look() {
+        let german = "Das ist ein ganz normaler deutscher Satz über das Wetter in Berlin \
+                      und die vielen schönen Häuser der Stadt. ";
+        // A start with nothing to tell by (a menu of numbers and bars), then
+        // the page: the first sample can't say, the middle can.
+        let menu = "| 1 | 2 | 3 | 4 | 5 | ".repeat(60);
+        assert!(menu.chars().count() > SAMPLE_CHARS);
+        assert_eq!(detect(None, &menu), None);
+        let page = format!("{menu}{}", german.repeat(40));
+        assert_eq!(detect(None, &page), Some("deu"));
+        // A long page: its middle, not its start or end.
+        let page = format!(
+            "{}{}{}",
+            menu.repeat(30),
+            german.repeat(200),
+            menu.repeat(30)
+        );
+        assert!(page.chars().count() > SECOND_SAMPLE_CHARS * 2);
+        assert_eq!(detect(None, &page), Some("deu"));
+    }
+
+    #[test]
+    fn middles() {
+        assert_eq!(middle("abcdefghij", 10, 4), "defg");
+        assert_eq!(middle("abc", 3, 4), "abc");
+        assert_eq!(middle("ünïcødé", 7, 3), "ïcø");
+        assert_eq!(middle("ünïcødé", 7, 7), "ünïcødé");
+    }
 }