Implement automatic media type detection

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.2.1
+
+### Added
+
+ - `detect_media_type()`, which guesses the media type from the data's signature (using the
+   [infer](https://crates.io/crates/infer) crate), falling back to the file extension for text formats
+   like CSS and JavaScript, and recognizing SVG either way
+ - The CLI detects the media type of files and STDIN input when `-t` isn't given
+
+### Changed
+
+ - Minimum supported Rust version is now 1.74 (required by infer)
+
 ## 0.2.0
 
 This release is a rewrite of the parser to follow the WHATWG Fetch Standard (what browsers do),

Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "dataurl"
-version = "0.2.0"
+version = "0.2.1"
 authors = [
     "Sunshine <snshn@tutanota.com>",
 ]
 edition = "2021"
-rust-version = "1.73"
+rust-version = "1.74"
 description = "CLI tool and Rust crate for parsing and generating data URLs"
 homepage = "https://github.com/Y2Z/dataurl"
 repository = "https://github.com/Y2Z/dataurl"
@@ -25,6 +25,7 @@ license = "CC0-1.0"
 
 [dependencies]
 encoding_rs = "0.8.35" # Used for charsets (label validation, decoding and encoding text)
+infer = { version = "0.22.0", default-features = false } # Used for detecting media types by file signature
 
 [features]
 default = ["cli"]

README.md

@@ -7,7 +7,7 @@
 CLI tool / Rust crate for converting files and text into data URLs and back
 
 Parsing follows the [WHATWG Fetch Standard](https://fetch.spec.whatwg.org/#data-url-processor), which is what browsers implement, and is checked against the [Web Platform Tests](https://github.com/web-platform-tests/wpt) suites for data URLs, base64, and MIME types.
-The only dependency is [encoding_rs](https://crates.io/crates/encoding_rs), used for charsets.
+Dependencies are kept minimal: [encoding_rs](https://crates.io/crates/encoding_rs) for charsets, and [infer](https://crates.io/crates/infer) for detecting media types (both have no dependencies of their own here).
 
 
 ---------------------------------------------------
@@ -24,15 +24,15 @@ dataurl -d 'data:text/html,text<a id%3D"b">ok</a>?a=v#f' > index.html
 ```
 
 ```console
-dataurl -b -t image/png -i picture.png
+dataurl -b -i picture.png
 ```
 
 ```console
 cat file.txt | dataurl -i - -o - | dataurl -d
 ```
 
 ```console
-cat file.png | dataurl -b -t image/png
+cat file.png | dataurl -b
 ```
 
 ### Flags and options
@@ -43,7 +43,7 @@ cat file.png | dataurl -b -t image/png
  - `-f`: Append `fragment`
  - `-i`: Specify `file` to read data from (use `-` for STDIN)
  - `-o`: Provide `file` to write output to (use `-` for STDOUT)
- - `-t`: Adjust `media type`
+ - `-t`: Adjust `media type` (detected automatically for files and STDIN)
 
 
 ---------------------------------------------------
@@ -82,6 +82,16 @@ assert_eq!(data_url.to_string(), "data:image/png;base64,iVBORw==");
 Data is stored as raw bytes and serialized as-is; use `set_text()` to store text encoded in the data URL's charset, and `text()` to decode it.
 Disabling default features skips the `cli` feature, which only matters for `cargo install`.
 
+Media types can be detected from the data's signature, falling back to the file extension
+(which is what text formats like CSS and JavaScript need):
+
+```rust
+use dataurl::detect_media_type;
+
+assert_eq!(detect_media_type(b"\x89PNG\r\n\x1a\n", None), Some("image/png"));
+assert_eq!(detect_media_type(b"body{}", Some("style.css")), Some("text/css"));
+```
+
 If all you need is parsing, [data-url](https://crates.io/crates/data-url) (from the Servo project) is another spec-compliant option;
 dataurl also generates data URLs, and comes with a CLI tool.
 

src/detect.rs

@@ -0,0 +1,169 @@
+//! Media type detection, based on the data's signature (via the infer crate) and the file extension.
+
+use infer::MatcherType;
+
+const SVG: &str = "image/svg+xml";
+
+/// Media types for file extensions (lowercase), using the names browsers expect.
+/// This is also used to normalize what infer returns, since some of its names are legacy ones
+/// (e.g. application/font-woff for both WOFF and WOFF2).
+fn media_type_for_extension(extension: &str) -> Option<&'static str> {
+    Some(match extension {
+        // Images
+        "apng" => "image/apng",
+        "avif" => "image/avif",
+        "bmp" => "image/bmp",
+        "gif" => "image/gif",
+        "ico" | "cur" => "image/x-icon",
+        "jpg" | "jpeg" | "jfif" | "pjpeg" | "pjp" => "image/jpeg",
+        "jxl" => "image/jxl",
+        "png" => "image/png",
+        "svg" | "svgz" => SVG,
+        "tif" | "tiff" => "image/tiff",
+        "webp" => "image/webp",
+        // Audio
+        "aac" => "audio/aac",
+        "flac" => "audio/flac",
+        "m4a" => "audio/mp4",
+        "mid" | "midi" => "audio/midi",
+        "mp3" => "audio/mpeg",
+        "oga" | "ogg" => "audio/ogg",
+        "opus" => "audio/opus",
+        "wav" => "audio/wav",
+        "weba" => "audio/webm",
+        // Video
+        "avi" => "video/avi",
+        "m4v" | "mp4" => "video/mp4",
+        "mkv" => "video/x-matroska",
+        "mov" => "video/quicktime",
+        "mpeg" | "mpg" => "video/mpeg",
+        "ogv" => "video/ogg",
+        "webm" => "video/webm",
+        // Fonts
+        "eot" => "application/vnd.ms-fontobject",
+        "otf" => "font/otf",
+        "ttf" => "font/ttf",
+        "woff" => "font/woff",
+        "woff2" => "font/woff2",
+        // Text and documents
+        "css" => "text/css",
+        "csv" => "text/csv",
+        "htm" | "html" => "text/html",
+        "ics" => "text/calendar",
+        "js" | "mjs" | "cjs" => "text/javascript",
+        "json" | "map" => "application/json",
+        "jsonld" => "application/ld+json",
+        "md" | "markdown" => "text/markdown",
+        "pdf" => "application/pdf",
+        "rss" => "application/rss+xml",
+        "txt" => "text/plain",
+        "vtt" => "text/vtt",
+        "webmanifest" => "application/manifest+json",
+        "xhtml" | "xht" => "application/xhtml+xml",
+        "xml" | "xsl" => "text/xml",
+        // Other
+        "swf" => "application/x-shockwave-flash",
+        "wasm" => "application/wasm",
+        "zip" => "application/zip",
+        _ => return None,
+    })
+}
+
+fn file_extension(file_name: &str) -> Option<String> {
+    let (stem, extension) = file_name.rsplit_once('.')?;
+    if stem.is_empty() || extension.is_empty() {
+        // Dotfiles like ".htaccess" have no extension
+        return None;
+    }
+    Some(extension.to_ascii_lowercase())
+}
+
+fn skip_leading_whitespace_and_bom(data: &[u8]) -> &[u8] {
+    let data = data.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(data);
+    let start = data
+        .iter()
+        .position(|b| !b.is_ascii_whitespace())
+        .unwrap_or(data.len());
+    &data[start..]
+}
+
+fn starts_with_svg_tag(data: &[u8]) -> bool {
+    let data = skip_leading_whitespace_and_bom(data);
+    data.len() > 4 && data[..4].eq_ignore_ascii_case(b"<svg") && !data[4].is_ascii_alphanumeric()
+}
+
+/// For XML documents, which often start with a declaration, a doctype or comments before the root element
+fn contains_svg_tag(data: &[u8]) -> bool {
+    data[..data.len().min(1024)]
+        .windows(5)
+        .any(|w| w[..4].eq_ignore_ascii_case(b"<svg") && !w[4].is_ascii_alphanumeric())
+}
+
+/// Guesses the media type of `data`, e.g. `"image/png"`.
+///
+/// The data's signature (its first bytes) takes precedence, which is how binary formats such as
+/// images, audio, video, fonts and archives get identified, regardless of what the file is named.
+/// Text formats such as CSS and JavaScript have no signature, so the extension of `file_name`
+/// is used for them; HTML, XML and SVG can be recognized either way.
+///
+/// Returns `None` if neither the data nor the file name give a clue.
+///
+/// ```
+/// use dataurl::detect_media_type;
+///
+/// assert_eq!(detect_media_type(b"\x89PNG\r\n\x1a\n", None), Some("image/png"));
+/// assert_eq!(detect_media_type(b"\x89PNG\r\n\x1a\n", Some("image.jpg")), Some("image/png"));
+/// assert_eq!(detect_media_type(b"body{}", Some("style.CSS")), Some("text/css"));
+/// assert_eq!(detect_media_type(b"<svg xmlns=\"http://www.w3.org/2000/svg\"/>", None), Some("image/svg+xml"));
+/// assert_eq!(detect_media_type(b"just some text", None), None);
+/// ```
+pub fn detect_media_type(data: &[u8], file_name: Option<&str>) -> Option<&'static str> {
+    let by_extension = || {
+        file_name
+            .and_then(file_extension)
+            .and_then(|extension| media_type_for_extension(&extension))
+    };
+
+    if starts_with_svg_tag(data) {
+        return Some(SVG);
+    }
+
+    match infer::get(data) {
+        // Binary formats have reliable signatures
+        Some(kind) if kind.matcher_type() != MatcherType::Text => {
+            Some(media_type_for_extension(kind.extension()).unwrap_or(kind.mime_type()))
+        }
+        // Text formats (HTML, XML, shell scripts) are only recognized by how they start,
+        // so the extension is more specific (e.g. an .svg or .xhtml file that starts with "<?xml")
+        Some(kind) => by_extension().or_else(|| {
+            if kind.mime_type() == "text/xml" && contains_svg_tag(data) {
+                Some(SVG)
+            } else {
+                Some(media_type_for_extension(kind.extension()).unwrap_or(kind.mime_type()))
+            }
+        }),
+        None => by_extension(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn file_extensions() {
+        assert_eq!(file_extension("a.PNG").as_deref(), Some("png"));
+        assert_eq!(file_extension("archive.tar.gz").as_deref(), Some("gz"));
+        assert_eq!(file_extension(".htaccess"), None);
+        assert_eq!(file_extension("README"), None);
+        assert_eq!(file_extension("trailing."), None);
+    }
+
+    #[test]
+    fn svg_tag_needs_a_delimiter() {
+        assert!(starts_with_svg_tag(b"<svg>"));
+        assert!(starts_with_svg_tag(b"\xEF\xBB\xBF\n  <SVG width=\"1\">"));
+        assert!(!starts_with_svg_tag(b"<svgfoo>"));
+        assert!(!starts_with_svg_tag(b"<sv"));
+    }
+}

src/lib.rs

@@ -26,9 +26,12 @@
 //! [data: URL processor]: https://fetch.spec.whatwg.org/#data-url-processor
 
 mod base64;
+mod detect;
 mod mime;
 mod percent;
 
+pub use detect::detect_media_type;
+
 use encoding_rs::{Encoding, REPLACEMENT, UTF_16BE, UTF_16LE, WINDOWS_1252};
 use std::borrow::Cow;
 use std::fmt;

src/main.rs

@@ -1,7 +1,8 @@
-use dataurl::DataUrl;
+use dataurl::{detect_media_type, DataUrl};
 use std::ffi::OsString;
 use std::fs;
 use std::io::{self, IsTerminal, Read, Write};
+use std::path::Path;
 use std::process::exit;
 
 const NAME: &str = env!("CARGO_PKG_NAME");
@@ -28,7 +29,7 @@ Options:
   -f, --fragment <FRAGMENT>   Append URL fragment
   -i, --input-file <FILE>     Read input from FILE (\"-\" for STDIN)
   -o, --output-file <FILE>    Write output to FILE (\"-\" for STDOUT)
-  -t, --media-type <TYPE>     Set media type
+  -t, --media-type <TYPE>     Set media type (detected for files and STDIN if omitted)
   -h, --help                  Print help
   -V, --version               Print version
 ",
@@ -206,6 +207,16 @@ fn main() {
             data_url
                 .set_media_type(Some(media_type))
                 .unwrap_or_else(|_| fail(&format!("Invalid media type '{media_type}'")));
+        } else if options.input.is_none() {
+            // Files and STDIN get their media type detected (text given as an argument has none)
+            let file_name = options
+                .input_file
+                .as_deref()
+                .and_then(|path| Path::new(path).file_name())
+                .and_then(|name| name.to_str());
+            if let Some(media_type) = detect_media_type(&input, file_name) {
+                data_url.set_media_type(Some(media_type)).unwrap();
+            }
         }
 
         match (&options.charset, &options.input) {

tests/cli.rs

@@ -207,6 +207,49 @@ mod encode {
         assert!(output.status.success());
         let data_url = dataurl::DataUrl::parse(&String::from_utf8(output.stdout).unwrap()).unwrap();
         assert_eq!(data_url.data(), contents);
+        assert_eq!(data_url.media_type_no_default(), Some("text/plain"));
+    }
+
+    #[test]
+    fn media_type_detected_for_input_file() {
+        let file = data_file("lizard.png");
+        let output = run(&["-b", "-i", file.to_str().unwrap()], b"");
+        assert!(output.status.success());
+        assert!(output.stdout.starts_with(b"data:image/png;base64,iVBOR"));
+    }
+
+    #[test]
+    fn media_type_detected_for_stdin() {
+        let png = std::fs::read(data_file("lizard.png")).unwrap();
+        let output = run(&["-b"], &png);
+        assert!(output.status.success());
+        assert!(output.stdout.starts_with(b"data:image/png;base64,iVBOR"));
+    }
+
+    #[test]
+    fn media_type_option_overrides_detection() {
+        let file = data_file("lizard.png");
+        let output = run(
+            &[
+                "-b",
+                "-t",
+                "application/octet-stream",
+                "-i",
+                file.to_str().unwrap(),
+            ],
+            b"",
+        );
+        assert!(output
+            .stdout
+            .starts_with(b"data:application/octet-stream;base64,iVBOR"));
+    }
+
+    #[test]
+    fn media_type_not_detected_for_argument() {
+        assert_success(
+            &run(&["<html></html>"], b""),
+            b"data:,%3Chtml%3E%3C/html%3E\n",
+        );
     }
 
     #[test]