From 3086087b1106e433faaa28626b48521f3d668c52 Mon Sep 17 00:00:00 2001 From: Sunshine Date: Fri, 16 Oct 2026 12:59:30 -0400 Subject: [PATCH] add media type detection via `infer` crate, use it in CLI --- CHANGELOG.md | 13 ++++ Cargo.lock | 9 ++- Cargo.toml | 5 +- README.md | 18 ++++-- src/detect.rs | 169 ++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 3 + src/main.rs | 15 ++++- tests/cli.rs | 43 ++++++++++++ tests/detect.rs | 156 ++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 422 insertions(+), 9 deletions(-) create mode 100644 src/detect.rs create mode 100644 tests/detect.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index de600e5..0a7f1a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 0.2.1 + +### Added + + - `detect_media_type()`, which guesses the media type from the data's signature (using the + [infer](https://crates.io/crates/infer) crate), falling back to the file extension for text formats + like CSS and JavaScript, and recognizing SVG either way + - The CLI detects the media type of files and STDIN input when `-t` isn't given + +### Changed + + - Minimum supported Rust version is now 1.74 (required by infer) + ## 0.2.0 This release is a rewrite of the parser to follow the WHATWG Fetch Standard (what browsers do), diff --git a/Cargo.lock b/Cargo.lock index b16a177..0fd20ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,9 +16,10 @@ checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" [[package]] name = "dataurl" -version = "0.2.0" +version = "0.2.1" dependencies = [ "encoding_rs", + "infer", ] [[package]] @@ -35,6 +36,12 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "infer" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4200d433cbd5178df7797c9c2e75b348b728e39631cf14520d1e2fc424201f4" + [[package]] name = "multiversion_no_op" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index d1857e2..ee5d240 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "dataurl" -version = "0.2.0" +version = "0.2.1" authors = [ "Sunshine ", ] edition = "2021" -rust-version = "1.73" +rust-version = "1.74" description = "CLI tool and Rust crate for parsing and generating data URLs" homepage = "https://github.com/Y2Z/dataurl" repository = "https://github.com/Y2Z/dataurl" @@ -25,6 +25,7 @@ license = "CC0-1.0" [dependencies] encoding_rs = "0.8.35" # Used for charsets (label validation, decoding and encoding text) +infer = { version = "0.22.0", default-features = false } # Used for detecting media types by file signature [features] default = ["cli"] diff --git a/README.md b/README.md index de2c904..4087f04 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ CLI tool / Rust crate for converting files and text into data URLs and back Parsing follows the [WHATWG Fetch Standard](https://fetch.spec.whatwg.org/#data-url-processor), which is what browsers implement, and is checked against the [Web Platform Tests](https://github.com/web-platform-tests/wpt) suites for data URLs, base64, and MIME types. -The only dependency is [encoding_rs](https://crates.io/crates/encoding_rs), used for charsets. +Dependencies are kept minimal: [encoding_rs](https://crates.io/crates/encoding_rs) for charsets, and [infer](https://crates.io/crates/infer) for detecting media types (both have no dependencies of their own here). --------------------------------------------------- @@ -24,7 +24,7 @@ dataurl -d 'data:text/html,textok?a=v#f' > index.html ``` ```console -dataurl -b -t image/png -i picture.png +dataurl -b -i picture.png ``` ```console @@ -32,7 +32,7 @@ cat file.txt | dataurl -i - -o - | dataurl -d ``` ```console -cat file.png | dataurl -b -t image/png +cat file.png | dataurl -b ``` ### Flags and options @@ -43,7 +43,7 @@ cat file.png | dataurl -b -t image/png - `-f`: Append `fragment` - `-i`: Specify `file` to read data from (use `-` for STDIN) - `-o`: Provide `file` to write output to (use `-` for STDOUT) - - `-t`: Adjust `media type` + - `-t`: Adjust `media type` (detected automatically for files and STDIN) --------------------------------------------------- @@ -82,6 +82,16 @@ assert_eq!(data_url.to_string(), "data:image/png;base64,iVBORw=="); Data is stored as raw bytes and serialized as-is; use `set_text()` to store text encoded in the data URL's charset, and `text()` to decode it. Disabling default features skips the `cli` feature, which only matters for `cargo install`. +Media types can be detected from the data's signature, falling back to the file extension +(which is what text formats like CSS and JavaScript need): + +```rust +use dataurl::detect_media_type; + +assert_eq!(detect_media_type(b"\x89PNG\r\n\x1a\n", None), Some("image/png")); +assert_eq!(detect_media_type(b"body{}", Some("style.css")), Some("text/css")); +``` + If all you need is parsing, [data-url](https://crates.io/crates/data-url) (from the Servo project) is another spec-compliant option; dataurl also generates data URLs, and comes with a CLI tool. diff --git a/src/detect.rs b/src/detect.rs new file mode 100644 index 0000000..c641ae6 --- /dev/null +++ b/src/detect.rs @@ -0,0 +1,169 @@ +//! Media type detection, based on the data's signature (via the infer crate) and the file extension. + +use infer::MatcherType; + +const SVG: &str = "image/svg+xml"; + +/// Media types for file extensions (lowercase), using the names browsers expect. +/// This is also used to normalize what infer returns, since some of its names are legacy ones +/// (e.g. application/font-woff for both WOFF and WOFF2). +fn media_type_for_extension(extension: &str) -> Option<&'static str> { + Some(match extension { + // Images + "apng" => "image/apng", + "avif" => "image/avif", + "bmp" => "image/bmp", + "gif" => "image/gif", + "ico" | "cur" => "image/x-icon", + "jpg" | "jpeg" | "jfif" | "pjpeg" | "pjp" => "image/jpeg", + "jxl" => "image/jxl", + "png" => "image/png", + "svg" | "svgz" => SVG, + "tif" | "tiff" => "image/tiff", + "webp" => "image/webp", + // Audio + "aac" => "audio/aac", + "flac" => "audio/flac", + "m4a" => "audio/mp4", + "mid" | "midi" => "audio/midi", + "mp3" => "audio/mpeg", + "oga" | "ogg" => "audio/ogg", + "opus" => "audio/opus", + "wav" => "audio/wav", + "weba" => "audio/webm", + // Video + "avi" => "video/avi", + "m4v" | "mp4" => "video/mp4", + "mkv" => "video/x-matroska", + "mov" => "video/quicktime", + "mpeg" | "mpg" => "video/mpeg", + "ogv" => "video/ogg", + "webm" => "video/webm", + // Fonts + "eot" => "application/vnd.ms-fontobject", + "otf" => "font/otf", + "ttf" => "font/ttf", + "woff" => "font/woff", + "woff2" => "font/woff2", + // Text and documents + "css" => "text/css", + "csv" => "text/csv", + "htm" | "html" => "text/html", + "ics" => "text/calendar", + "js" | "mjs" | "cjs" => "text/javascript", + "json" | "map" => "application/json", + "jsonld" => "application/ld+json", + "md" | "markdown" => "text/markdown", + "pdf" => "application/pdf", + "rss" => "application/rss+xml", + "txt" => "text/plain", + "vtt" => "text/vtt", + "webmanifest" => "application/manifest+json", + "xhtml" | "xht" => "application/xhtml+xml", + "xml" | "xsl" => "text/xml", + // Other + "swf" => "application/x-shockwave-flash", + "wasm" => "application/wasm", + "zip" => "application/zip", + _ => return None, + }) +} + +fn file_extension(file_name: &str) -> Option { + let (stem, extension) = file_name.rsplit_once('.')?; + if stem.is_empty() || extension.is_empty() { + // Dotfiles like ".htaccess" have no extension + return None; + } + Some(extension.to_ascii_lowercase()) +} + +fn skip_leading_whitespace_and_bom(data: &[u8]) -> &[u8] { + let data = data.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(data); + let start = data + .iter() + .position(|b| !b.is_ascii_whitespace()) + .unwrap_or(data.len()); + &data[start..] +} + +fn starts_with_svg_tag(data: &[u8]) -> bool { + let data = skip_leading_whitespace_and_bom(data); + data.len() > 4 && data[..4].eq_ignore_ascii_case(b" bool { + data[..data.len().min(1024)] + .windows(5) + .any(|w| w[..4].eq_ignore_ascii_case(b"", None), Some("image/svg+xml")); +/// assert_eq!(detect_media_type(b"just some text", None), None); +/// ``` +pub fn detect_media_type(data: &[u8], file_name: Option<&str>) -> Option<&'static str> { + let by_extension = || { + file_name + .and_then(file_extension) + .and_then(|extension| media_type_for_extension(&extension)) + }; + + if starts_with_svg_tag(data) { + return Some(SVG); + } + + match infer::get(data) { + // Binary formats have reliable signatures + Some(kind) if kind.matcher_type() != MatcherType::Text => { + Some(media_type_for_extension(kind.extension()).unwrap_or(kind.mime_type())) + } + // Text formats (HTML, XML, shell scripts) are only recognized by how they start, + // so the extension is more specific (e.g. an .svg or .xhtml file that starts with " by_extension().or_else(|| { + if kind.mime_type() == "text/xml" && contains_svg_tag(data) { + Some(SVG) + } else { + Some(media_type_for_extension(kind.extension()).unwrap_or(kind.mime_type())) + } + }), + None => by_extension(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_extensions() { + assert_eq!(file_extension("a.PNG").as_deref(), Some("png")); + assert_eq!(file_extension("archive.tar.gz").as_deref(), Some("gz")); + assert_eq!(file_extension(".htaccess"), None); + assert_eq!(file_extension("README"), None); + assert_eq!(file_extension("trailing."), None); + } + + #[test] + fn svg_tag_needs_a_delimiter() { + assert!(starts_with_svg_tag(b"")); + assert!(starts_with_svg_tag(b"\xEF\xBB\xBF\n ")); + assert!(!starts_with_svg_tag(b"")); + assert!(!starts_with_svg_tag(b" Append URL fragment -i, --input-file Read input from FILE (\"-\" for STDIN) -o, --output-file Write output to FILE (\"-\" for STDOUT) - -t, --media-type Set media type + -t, --media-type Set media type (detected for files and STDIN if omitted) -h, --help Print help -V, --version Print version ", @@ -206,6 +207,16 @@ fn main() { data_url .set_media_type(Some(media_type)) .unwrap_or_else(|_| fail(&format!("Invalid media type '{media_type}'"))); + } else if options.input.is_none() { + // Files and STDIN get their media type detected (text given as an argument has none) + let file_name = options + .input_file + .as_deref() + .and_then(|path| Path::new(path).file_name()) + .and_then(|name| name.to_str()); + if let Some(media_type) = detect_media_type(&input, file_name) { + data_url.set_media_type(Some(media_type)).unwrap(); + } } match (&options.charset, &options.input) { diff --git a/tests/cli.rs b/tests/cli.rs index 81db4ba..2a987ca 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -207,6 +207,49 @@ mod encode { assert!(output.status.success()); let data_url = dataurl::DataUrl::parse(&String::from_utf8(output.stdout).unwrap()).unwrap(); assert_eq!(data_url.data(), contents); + assert_eq!(data_url.media_type_no_default(), Some("text/plain")); + } + + #[test] + fn media_type_detected_for_input_file() { + let file = data_file("lizard.png"); + let output = run(&["-b", "-i", file.to_str().unwrap()], b""); + assert!(output.status.success()); + assert!(output.stdout.starts_with(b"data:image/png;base64,iVBOR")); + } + + #[test] + fn media_type_detected_for_stdin() { + let png = std::fs::read(data_file("lizard.png")).unwrap(); + let output = run(&["-b"], &png); + assert!(output.status.success()); + assert!(output.stdout.starts_with(b"data:image/png;base64,iVBOR")); + } + + #[test] + fn media_type_option_overrides_detection() { + let file = data_file("lizard.png"); + let output = run( + &[ + "-b", + "-t", + "application/octet-stream", + "-i", + file.to_str().unwrap(), + ], + b"", + ); + assert!(output + .stdout + .starts_with(b"data:application/octet-stream;base64,iVBOR")); + } + + #[test] + fn media_type_not_detected_for_argument() { + assert_success( + &run(&[""], b""), + b"data:,%3Chtml%3E%3C/html%3E\n", + ); } #[test] diff --git a/tests/detect.rs b/tests/detect.rs new file mode 100644 index 0000000..a522d6e --- /dev/null +++ b/tests/detect.rs @@ -0,0 +1,156 @@ +use dataurl::detect_media_type; + +/// Pads a file header with zeros, since some signatures need a minimum length +/// (e.g. WebM needs more than 256 bytes to be told apart from other Matroska files) +fn header(start: &[u8]) -> Vec { + let mut data = start.to_vec(); + data.resize(512.max(start.len()), 0); + data +} + +#[test] +fn binary_formats_by_signature() { + for (data, expected) in [ + (header(b"\x89PNG\r\n\x1a\n"), "image/png"), + (header(b"\xFF\xD8\xFF\xE0"), "image/jpeg"), + (header(b"GIF87a"), "image/gif"), + (header(b"GIF89a"), "image/gif"), + (header(b"RIFF\x24\x00\x00\x00WEBPVP8 "), "image/webp"), + (header(b"\x00\x00\x01\x00\x01\x00"), "image/x-icon"), + (header(b"BM\x36\x00\x0C\x00\x00\x00"), "image/bmp"), + (header(b"%PDF-1.7"), "application/pdf"), + (header(b"ID3\x04\x00"), "audio/mpeg"), + (header(b"fLaC\x00\x00\x00\x22"), "audio/flac"), + (header(b"OggS\x00\x02"), "audio/ogg"), + (header(b"RIFF\x24\x00\x00\x00WAVEfmt "), "audio/wav"), + (header(b"RIFF\x24\x00\x00\x00AVI LIST"), "video/avi"), + ( + header(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00"), + "video/mp4", + ), + ( + header(b"\x00\x00\x00\x14ftypqt \x00\x00\x00\x00"), + "video/quicktime", + ), + (header(b"\x00\x00\x01\xBA"), "video/mpeg"), + (header(b"wOFF\x00\x01\x00\x00"), "font/woff"), + (header(b"wOF2\x00\x01\x00\x00"), "font/woff2"), + (header(b"\x00\x01\x00\x00\x00"), "font/ttf"), + (header(b"OTTO\x00"), "font/otf"), + (header(b"\x00asm\x01\x00\x00\x00"), "application/wasm"), + (header(b"PK\x03\x04"), "application/zip"), + ] { + assert_eq!(detect_media_type(&data, None), Some(expected), "{expected}"); + } +} + +#[test] +fn webm() { + // EBML header with "webm" as the document type + let data = b"\x1A\x45\xDF\xA3\x9F\x42\x86\x81\x01\x42\xF7\x81\x01\x42\xF2\x81\x04\x42\xF3\x81\x08\x42\x82\x84webm\x42\x87\x81\x04"; + assert_eq!(detect_media_type(&header(data), None), Some("video/webm")); +} + +#[test] +fn signature_beats_extension() { + let png = header(b"\x89PNG\r\n\x1a\n"); + assert_eq!( + detect_media_type(&png, Some("photo.jpg")), + Some("image/png") + ); + assert_eq!( + detect_media_type(&png, Some("style.css")), + Some("image/png") + ); +} + +#[test] +fn text_formats_by_extension() { + for (file_name, expected) in [ + ("style.css", "text/css"), + ("STYLE.CSS", "text/css"), + ("app.js", "text/javascript"), + ("module.mjs", "text/javascript"), + ("data.json", "application/json"), + ("notes.txt", "text/plain"), + ("subtitles.vtt", "text/vtt"), + ("site.webmanifest", "application/manifest+json"), + ("font.woff2", "font/woff2"), + ("image.SVG", "image/svg+xml"), + ] { + assert_eq!( + detect_media_type(b"", Some(file_name)), + Some(expected), + "{file_name}" + ); + } +} + +#[test] +fn html() { + let html = b""; + assert_eq!(detect_media_type(html, None), Some("text/html")); + assert_eq!(detect_media_type(html, Some("page.htm")), Some("text/html")); + assert_eq!( + detect_media_type(html, Some("page.xhtml")), + Some("application/xhtml+xml") + ); +} + +#[test] +fn svg() { + assert_eq!(detect_media_type(b"", + None + ), + Some("image/svg+xml") + ); + + let with_declaration = + b"\n\n"; + assert_eq!( + detect_media_type(with_declaration, None), + Some("image/svg+xml") + ); + assert_eq!( + detect_media_type(b"", Some("drawing.svg")), + Some("image/svg+xml") + ); +} + +#[test] +fn xml() { + let xml = b""; + assert_eq!(detect_media_type(xml, None), Some("text/xml")); + assert_eq!( + detect_media_type(xml, Some("feed.rss")), + Some("application/rss+xml") + ); + // An element merely starting with "svg" isn't an SVG + assert_eq!( + detect_media_type(b"", None), + Some("text/xml") + ); +} + +#[test] +fn script_with_shebang_uses_extension() { + assert_eq!( + detect_media_type(b"#!/usr/bin/env node\nconsole.log(1);", Some("cli.js")), + Some("text/javascript") + ); +} + +#[test] +fn unknown() { + assert_eq!(detect_media_type(b"", None), None); + assert_eq!(detect_media_type(b"just some text", None), None); + assert_eq!( + detect_media_type(b"abcdef0123456789", Some("file.unknown")), + None + ); + assert_eq!(detect_media_type(b"x", Some("README")), None); + assert_eq!(detect_media_type(b"x", Some(".css")), None); +}