feat: --hex flag augments record-level output with raw record bytes

CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 * **Layered text output format** (`render::text::format_record`, `--format text`): one human-readable, indented block per MRT record — session context (`TIME`/`TYPE`/`FROM`/`TO`), `UPDATE:` sections with withdrawn/announced prefixes (including those carried in MP_REACH/MP_UNREACH) and every path attribute, `OPEN:` capabilities, session states, RIB entries, the peer table, and full legacy type-5 records. RFC 7606 validation findings render under `WARNINGS:` when present. The format is designed around this crate's own models and `Display` vocabulary — inspired by bgpdump's human-readable output, not byte-compatible with it. Rendering is a pure function of the record. In the CLI, `--format text` always uses record-level output (implies `--level records`); the other formats follow `--level` and default to elems.
+* **`--hex` flag**: include each record's original wire bytes as hex in record-level output — a `HEX:` line at the end of `--format text` blocks, a `hex` field in `--format json`/`json-pretty` records. Designed as the pipe from filtered CLI output into byte-level tools (e.g. wirescope's hex paste input): `bgpkit-parser -p 8.8.8.0/24 --format text --hex updates.mrt.gz` yields blocks whose `HEX:` line can be pasted straight into a dissector. Bytes come from a new library iterator, `BgpkitParser::into_filtered_raw_record_iter()` — raw MRT records with the same record-level filter semantics as `into_record_iter`, yielding the untouched original bytes (no re-encoding, so attribute-ordering quirks such as BGP-LS hash-map iteration can never alter the output). The flag implies `--level records`, requires text/json/json-pretty formats, is mutually exclusive with `--recover`, and is ignored by count-only runs (`-e`/`-r`), which keep the normal counting pipelines and their per-elem filter semantics.
 * Record-level filter semantics are now explicit: records that produce no elems (KEEPALIVE, OPEN, NOTIFICATION, state changes) never match elem-oriented filters and are dropped from record iteration while filters are active — a `debug!` line notes each drop.
 
 ## v0.21.0 - 2026-08-21

src/bin/main.rs

@@ -69,6 +69,13 @@ struct Opts {
     #[clap(long)]
     psv: bool,
 
+    /// Include each record's raw bytes as hex: a `HEX:` line in text
+    /// blocks, a `hex` field in JSON records. Record-level only — implies
+    /// `--level records`; requires `--format text`, `json`, or
+    /// `json-pretty`. Count-only runs (-e/-r) ignore this flag.
+    #[clap(long)]
+    hex: bool,
+
     /// Count BGP elems
     #[clap(short, long)]
     elems_count: bool,
@@ -257,6 +264,23 @@ fn main() {
         opts.format
     };
 
+    // Count-only runs emit no hex, so the flag is ignored there (and the
+    // format/recover constraints do not apply).
+    let counting = opts.elems_count || opts.records_count;
+    if opts.hex && !counting {
+        if !matches!(
+            output_format,
+            OutputFormat::Json | OutputFormat::JsonPretty | OutputFormat::Text
+        ) {
+            eprintln!("Error: --hex requires --format text, json, or json-pretty");
+            std::process::exit(1);
+        }
+        if opts.recover {
+            eprintln!("Error: --hex does not support --recover");
+            std::process::exit(1);
+        }
+    }
+
     let recovery_config = RecoveryConfig::default();
     // Element-level runs (element output or counting only elements) use the elem
     // iterators, which apply filters per element; everything else stays at the record
@@ -265,41 +289,50 @@ fn main() {
         || (!opts.elems_count && !opts.records_count && matches!(opts.level, OutputLevel::Elems)))
         && output_format != OutputFormat::Text;
 
-    let result = match (opts.recover, use_elem_stream) {
-        (true, true) => run_elems(
-            parser
-                .into_recovering_elem_iter(recovery_config)
-                .map(|event| event.map_err(|error| error.to_string())),
-            output_format,
-            opts.elems_count,
-            true,
-        ),
-        (false, true) => run_elems(
-            parser
-                .into_elem_iter()
-                .map(|elem| Ok(RecoveryEvent::Item(elem))),
-            output_format,
-            opts.elems_count,
-            false,
-        ),
-        (true, false) => run_records(
-            parser
-                .into_recovering_record_iter(recovery_config)
-                .map(|event| event.map_err(|error| error.to_string())),
-            output_format,
-            opts.elems_count,
-            opts.records_count,
-            true,
-        ),
-        (false, false) => run_records(
-            parser
-                .into_record_iter()
-                .map(|record| Ok(RecoveryEvent::Item(record))),
-            output_format,
-            opts.elems_count,
-            opts.records_count,
-            false,
-        ),
+    // The hex path always works on original wire bytes from the
+    // raw-record pipeline; it cannot share run_records, which holds
+    // parsed records. Count-only runs emit no hex, so they keep the
+    // normal elem/record counting pipelines and their semantics
+    // (per-elem filtering for -e).
+    let result = if opts.hex && !counting {
+        run_hex_records(parser.into_filtered_raw_record_iter(), output_format)
+    } else {
+        match (opts.recover, use_elem_stream) {
+            (true, true) => run_elems(
+                parser
+                    .into_recovering_elem_iter(recovery_config)
+                    .map(|event| event.map_err(|error| error.to_string())),
+                output_format,
+                opts.elems_count,
+                true,
+            ),
+            (false, true) => run_elems(
+                parser
+                    .into_elem_iter()
+                    .map(|elem| Ok(RecoveryEvent::Item(elem))),
+                output_format,
+                opts.elems_count,
+                false,
+            ),
+            (true, false) => run_records(
+                parser
+                    .into_recovering_record_iter(recovery_config)
+                    .map(|event| event.map_err(|error| error.to_string())),
+                output_format,
+                opts.elems_count,
+                opts.records_count,
+                true,
+            ),
+            (false, false) => run_records(
+                parser
+                    .into_record_iter()
+                    .map(|record| Ok(RecoveryEvent::Item(record))),
+                output_format,
+                opts.elems_count,
+                opts.records_count,
+                false,
+            ),
+        }
     };
     if let Err(error) = result {
         eprintln!("{error}");
@@ -425,7 +458,7 @@ where
                 if records_count_requested {
                     continue;
                 }
-                let output = format_record(&record, output_format);
+                let output = format_record(&record, output_format, None);
                 if !write_output(&mut stdout, &output)? {
                     return Ok(());
                 }
@@ -446,6 +479,25 @@ where
     terminal_error.map_or(Ok(()), Err)
 }
 
+/// Hex-augmented output over the raw-record pipeline: hex comes from the
+/// original wire bytes (never a re-encoding), the block/JSON body from the
+/// parsed record.
+fn run_hex_records(
+    records: impl Iterator<Item = (bgpkit_parser::RawMrtRecord, bgpkit_parser::MrtRecord)>,
+    output_format: OutputFormat,
+) -> Result<(), String> {
+    let mut stdout = std::io::stdout();
+
+    for (raw, record) in records {
+        let hex = bgpkit_parser::render::hex::encode(raw.raw_bytes().as_ref());
+        let output = format_record(&record, output_format, Some(&hex));
+        if !write_output(&mut stdout, &output)? {
+            return Ok(());
+        }
+    }
+    Ok(())
+}
+
 fn write_output(stdout: &mut std::io::Stdout, output: &str) -> Result<bool, String> {
     if let Err(error) = writeln!(stdout, "{output}") {
         if error.kind() == std::io::ErrorKind::BrokenPipe {
@@ -480,17 +532,30 @@ fn format_elem(elem: &BgpElem, format: OutputFormat, index: usize) -> String {
     }
 }
 
-fn format_record(record: &bgpkit_parser::MrtRecord, format: OutputFormat) -> String {
+fn format_record(
+    record: &bgpkit_parser::MrtRecord,
+    format: OutputFormat,
+    record_hex: Option<&str>,
+) -> String {
     match format {
         OutputFormat::Json => {
-            let val = json!(record);
+            let mut val = json!(record);
+            if let (Some(hex), Some(obj)) = (&record_hex, val.as_object_mut()) {
+                obj.insert("hex".to_string(), json!(hex));
+            }
             val.to_string()
         }
         OutputFormat::JsonPretty => {
-            let val = json!(record);
+            let mut val = json!(record);
+            if let (Some(hex), Some(obj)) = (&record_hex, val.as_object_mut()) {
+                obj.insert("hex".to_string(), json!(hex));
+            }
             serde_json::to_string_pretty(&val).unwrap()
         }
-        OutputFormat::Text => bgpkit_parser::render::text::format_record(record),
+        OutputFormat::Text => match record_hex {
+            Some(hex) => bgpkit_parser::render::text::format_record_with_hex(record, hex),
+            None => bgpkit_parser::render::text::format_record(record),
+        },
         OutputFormat::Psv | OutputFormat::Default => {
             // Use the Display implementation for MrtRecord
             format!("{}", record)

src/parser/iters/default.rs

@@ -1,12 +1,10 @@
 /*!
 Default iterator implementations that skip errors and return successfully parsed items.
 */
-use crate::error::ParserError;
 use crate::models::*;
-use crate::parser::iters::{record_matches_filters, write_mrt_core_dump};
+use crate::parser::iters::{handle_record_parse_error, record_matches_filters};
 use crate::parser::BgpkitParser;
 use crate::{Elementor, Filterable};
-use log::{error, warn};
 use std::io::Read;
 
 /*********
@@ -49,52 +47,10 @@ impl<R: Read> Iterator for RecordIterator<R> {
                     }
                 }
                 Err(e) => {
-                    match e.error {
-                        ParserError::TruncatedMsg(err_str) | ParserError::Unsupported(err_str) => {
-                            if self.parser.options.show_warnings {
-                                warn!("parser warn: {}", err_str);
-                            }
-                            write_mrt_core_dump(self.parser.core_dump, e.bytes);
-                            continue;
-                        }
-                        ParserError::ParseError(err_str) => {
-                            error!("parser error: {}", err_str);
-                            write_mrt_core_dump(self.parser.core_dump, e.bytes);
-                            if self.parser.core_dump {
-                                None
-                            } else {
-                                continue;
-                            }
-                        }
-                        ParserError::EofExpected => {
-                            // normal end of file
-                            None
-                        }
-                        ParserError::IoError(err) | ParserError::EofError(err) => {
-                            // when reaching IO error, stop iterating
-                            error!("{:?}", err);
-                            write_mrt_core_dump(self.parser.core_dump, e.bytes);
-                            None
-                        }
-                        #[cfg(feature = "oneio")]
-                        ParserError::OneIoError(_) => None,
-                        ParserError::FilterError(_) => {
-                            // this should not happen at this stage
-                            None
-                        }
-                        // Labeled NLRI parsing errors - treat as malformed and skip
-                        ParserError::InvalidLabeledNlriLength
-                        | ParserError::TruncatedLabeledNlri
-                        | ParserError::TruncatedPrefix
-                        | ParserError::MaxLabelStackDepthExceeded
-                        | ParserError::PeerMaxLabelsExceeded
-                        | ParserError::InvalidPrefix => {
-                            if self.parser.options.show_warnings {
-                                warn!("parser warn: labeled NLRI parsing error: {:?}", e.error);
-                            }
-                            continue;
-                        }
+                    if handle_record_parse_error(&mut self.parser, e.error, e.bytes) {
+                        continue;
                     }
+                    None
                 }
             };
         }

src/parser/iters/mod.rs

@@ -25,7 +25,7 @@ pub use diagnostic::{
     DissectedDiagnosticEvent, DissectingDiagnosticIterator,
 };
 pub use fallible::{FallibleElemIterator, FallibleRecordIterator};
-pub use raw::RawRecordIterator;
+pub use raw::{FilteredRawRecordIterator, RawRecordIterator};
 pub use recovery::{
     RecoveringElemIterator, RecoveringRecordIterator, RecoveryConfig, RecoveryError, RecoveryEvent,
     RecoveryEvidence, RecoveryGap,
@@ -36,12 +36,13 @@ pub use update::{
     UpdateIterator,
 };
 
+use crate::error::ParserError;
 use crate::models::BgpElem;
 use crate::models::{MrtMessage, MrtRecord, TableDumpV2Message};
 use crate::parser::BgpkitParser;
 use crate::RawMrtRecord;
 use crate::{Elementor, Filter, Filterable};
-use log::debug;
+use log::{debug, error, warn};
 use std::io::Read;
 use std::path::Path;
 
@@ -75,6 +76,63 @@ pub(crate) fn record_matches_filters(
     elems.iter().any(|element| element.match_filters(filters))
 }
 
+/// Shared body-parse error policy for record-producing iterators.
+///
+/// Mirrors the historical `RecordIterator` behavior: warnings honor
+/// `disable_warnings()`, core dumps are written for recoverable classes,
+/// and a fatal `ParseError` with core dumps enabled stops the iterator so
+/// a later failure cannot overwrite the dump. Returns `true` to continue
+/// iterating, `false` to stop.
+pub(crate) fn handle_record_parse_error<R>(
+    parser: &mut crate::parser::BgpkitParser<R>,
+    error: ParserError,
+    bytes: Option<Vec<u8>>,
+) -> bool {
+    match error {
+        ParserError::TruncatedMsg(err_str) | ParserError::Unsupported(err_str) => {
+            if parser.options.show_warnings {
+                warn!("parser warn: {}", err_str);
+            }
+            write_mrt_core_dump(parser.core_dump, bytes);
+            true
+        }
+        ParserError::ParseError(err_str) => {
+            error!("parser error: {}", err_str);
+            write_mrt_core_dump(parser.core_dump, bytes);
+            // stop after writing the dump so later failures don't overwrite it
+            !parser.core_dump
+        }
+        ParserError::EofExpected => {
+            // normal end of file
+            false
+        }
+        ParserError::IoError(err) | ParserError::EofError(err) => {
+            // when reaching IO error, stop iterating
+            error!("{:?}", err);
+            write_mrt_core_dump(parser.core_dump, bytes);
+            false
+        }
+        #[cfg(feature = "oneio")]
+        ParserError::OneIoError(_) => false,
+        ParserError::FilterError(_) => {
+            // this should not happen at this stage
+            false
+        }
+        // Labeled NLRI parsing errors - treat as malformed and skip
+        ParserError::InvalidLabeledNlriLength
+        | ParserError::TruncatedLabeledNlri
+        | ParserError::TruncatedPrefix
+        | ParserError::MaxLabelStackDepthExceeded
+        | ParserError::PeerMaxLabelsExceeded
+        | ParserError::InvalidPrefix => {
+            if parser.options.show_warnings {
+                warn!("parser warn: labeled NLRI parsing error: {:?}", error);
+            }
+            true
+        }
+    }
+}
+
 pub(crate) fn write_mrt_core_dump(enabled: bool, bytes: Option<Vec<u8>>) {
     write_mrt_core_dump_to_path(enabled, bytes, "mrt_core_dump");
 }
@@ -114,6 +172,34 @@ impl<R> BgpkitParser<R> {
         RawRecordIterator::new(self)
     }
 
+    /// Creates an iterator over raw MRT records with record-level filter
+    /// semantics applied.
+    ///
+    /// Like [`into_raw_record_iter`](Self::into_raw_record_iter), but only
+    /// records passing the parser's filters are yielded (same semantics as
+    /// [`into_record_iter`](Self::into_record_iter): filters match on the
+    /// elem projection, so no-elem records such as KEEPALIVEs are dropped
+    /// while filters are active, and the `PeerIndexTable` always passes).
+    /// The yielded records carry their original wire bytes — no
+    /// re-encoding — which is what byte-exact consumers (hex output,
+    /// re-dissection) need. Every record body is parsed once inside the
+    /// iterator and the parsed record is yielded alongside, so consumers
+    /// do not parse the bytes twice; parse failures follow the same
+    /// variant-aware diagnostics as the record iterator.
+    ///
+    /// # Example
+    /// ```no_run
+    /// use bgpkit_parser::BgpkitParser;
+    ///
+    /// let parser = BgpkitParser::new("updates.mrt").unwrap();
+    /// for (raw, _) in parser.into_filtered_raw_record_iter() {
+    ///     println!("{}", raw.raw_bytes().len());
+    /// }
+    /// ```
+    pub fn into_filtered_raw_record_iter(self) -> FilteredRawRecordIterator<R> {
+        FilteredRawRecordIterator::new(self)
+    }
+
     /// Creates an opt-in iterator that reports skipped byte ranges while recovering MRT framing.
     ///
     /// Recovery never reconstructs a damaged record. It scans for a structurally valid boundary,