From 27ad1afaada18959c8c8ffd5de16bb1490cd1d5f Mon Sep 17 00:00:00 2001 From: Loic Nageleisen Date: Wed, 29 Jul 2026 17:27:10 +0200 Subject: [PATCH 1/6] feat(data-pipeline): add meta struct blob setter Expose a C setter that copies an opaque binary value into a span's `meta_struct` map. Native tracers can now supply pre-encoded MessagePack without transferring ownership or requiring Rust to interpret it. Reject null span handles and non-UTF-8 keys. Reusing a key replaces its value, and the copied bytes remain owned by the span after the call. --- libdd-data-pipeline-ffi/src/tracer.rs | 95 ++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/libdd-data-pipeline-ffi/src/tracer.rs b/libdd-data-pipeline-ffi/src/tracer.rs index 9489527ad6..d47ec37c70 100644 --- a/libdd-data-pipeline-ffi/src/tracer.rs +++ b/libdd-data-pipeline-ffi/src/tracer.rs @@ -13,9 +13,9 @@ use crate::error::{ExporterError, ExporterErrorCode as ErrorCode}; use crate::response::ExporterResponse; use crate::trace_exporter::TraceExporter; use crate::{catch_panic, gen_error}; -use libdd_common_ffi::slice::AsBytes; +use libdd_common_ffi::slice::{AsBytes, ByteSlice}; use libdd_common_ffi::CharSlice; -use libdd_tinybytes::BytesString; +use libdd_tinybytes::{Bytes, BytesString}; use libdd_trace_utils::span::v04::SpanBytes; use std::ptr::NonNull; @@ -192,6 +192,38 @@ pub unsafe extern "C" fn ddog_tracer_span_set_metric( ) } +/// Add or overwrite a structured metadata entry (`meta_struct`) on the span. +/// +/// The `key` and opaque binary `value` are copied into the span. The value is +/// not interpreted or validated as MessagePack. +/// +/// # Safety +/// +/// `handle` must be a valid pointer to a `TracerSpan`. `key` must point to +/// valid UTF-8 memory, and `value` must point to valid memory for its length. +#[no_mangle] +pub unsafe extern "C" fn ddog_tracer_span_set_meta_struct_blob( + handle: Option<&mut TracerSpan>, + key: CharSlice, + value: ByteSlice, +) -> Option> { + catch_panic!( + if let Some(span) = handle { + let key = match charslice_to_bytesstring(key) { + Ok(s) => s, + Err(e) => return Some(e), + }; + span.0 + .meta_struct + .insert(key, Bytes::copy_from_slice(value.as_bytes())); + None + } else { + gen_error!(ErrorCode::InvalidArgument) + }, + gen_error!(ErrorCode::Panic) + ) +} + // --------------------------------------------------------------------------- // TracerTraceChunks // --------------------------------------------------------------------------- @@ -407,6 +439,10 @@ mod tests { CharSlice::from_bytes(s.as_bytes()) } + fn bs(bytes: &[u8]) -> ByteSlice<'_> { + ByteSlice::from(bytes) + } + fn make_minimal_span() -> Box { unsafe { let mut handle = MaybeUninit::>::uninit(); @@ -465,6 +501,7 @@ mod tests { assert_eq!(span.0.error, 0); assert!(span.0.meta.is_empty()); assert!(span.0.metrics.is_empty()); + assert!(span.0.meta_struct.is_empty()); assert!(span.0.span_links.is_empty()); assert!(span.0.span_events.is_empty()); @@ -527,6 +564,36 @@ mod tests { } } + #[test] + fn set_meta_struct_blob_inserts_binary_entries() { + unsafe { + let mut span = make_minimal_span(); + let value = b"\x82\xa6nested\x92\xc3\xc0\xa3raw\xc4\x03\x00\xff\x80"; + + let err = + ddog_tracer_span_set_meta_struct_blob(Some(&mut *span), cs("_dd.stack"), bs(value)); + assert!(err.is_none()); + + assert_eq!(span.0.meta_struct.get("_dd.stack").unwrap().as_ref(), value); + + ddog_tracer_span_free(span); + } + } + + #[test] + fn set_meta_struct_blob_overwrites_existing_key() { + unsafe { + let mut span = make_minimal_span(); + + ddog_tracer_span_set_meta_struct_blob(Some(&mut *span), cs("k"), bs(b"first")); + ddog_tracer_span_set_meta_struct_blob(Some(&mut *span), cs("k"), bs(b"second")); + + assert_eq!(span.0.meta_struct.get("k").unwrap().as_ref(), b"second"); + + ddog_tracer_span_free(span); + } + } + #[test] fn set_meta_null_handle_returns_error() { unsafe { @@ -545,6 +612,30 @@ mod tests { } } + #[test] + fn set_meta_struct_blob_null_handle_returns_error() { + unsafe { + let err = ddog_tracer_span_set_meta_struct_blob(None, cs("k"), bs(b"value")); + assert!(err.is_some()); + ddog_trace_exporter_error_free(err); + } + } + + #[test] + fn set_meta_struct_blob_invalid_key_returns_error() { + unsafe { + let mut span = make_minimal_span(); + let key = CharSlice::from_bytes(&[0xff]); + + let err = ddog_tracer_span_set_meta_struct_blob(Some(&mut *span), key, bs(b"value")); + assert!(err.is_some()); + assert!(span.0.meta_struct.is_empty()); + ddog_trace_exporter_error_free(err); + + ddog_tracer_span_free(span); + } + } + #[test] fn new_with_empty_strings_succeeds() { unsafe { From 52a0ff69a660a9185d1bd08e3b5b6f12f893b936 Mon Sep 17 00:00:00 2001 From: Loic Nageleisen Date: Wed, 29 Jul 2026 18:47:08 +0200 Subject: [PATCH 2/6] feat(data-pipeline): encode structured values Expose a flat preorder token ABI that encodes nil, scalar, string, binary, array, and map values into an owned MessagePack blob. This lets C callers build structured metadata without implementing MessagePack or sharing the lifetime of their input buffers. Reject malformed token streams, invalid kinds, booleans, UTF-8, counts, and nesting deeper than 64 levels. Successful output contains exactly one value and remains valid until explicitly freed. --- libdd-data-pipeline-ffi/cbindgen.toml | 11 + libdd-data-pipeline-ffi/src/lib.rs | 1 + .../src/structured_value.rs | 451 ++++++++++++++++++ 3 files changed, 463 insertions(+) create mode 100644 libdd-data-pipeline-ffi/src/structured_value.rs diff --git a/libdd-data-pipeline-ffi/cbindgen.toml b/libdd-data-pipeline-ffi/cbindgen.toml index d3e36b4945..c65c8e983b 100644 --- a/libdd-data-pipeline-ffi/cbindgen.toml +++ b/libdd-data-pipeline-ffi/cbindgen.toml @@ -14,6 +14,17 @@ typedef struct ddog_TraceExporter ddog_TraceExporter; typedef struct ddog_TracerSpan ddog_TracerSpan; typedef struct ddog_TracerTraceChunks ddog_TracerTraceChunks; typedef struct ddog_TraceExporterCancelToken ddog_TraceExporterCancelToken; +typedef enum ddog_TracerValueKind { + DDOG_TRACER_VALUE_NIL = 0, + DDOG_TRACER_VALUE_BOOL = 1, + DDOG_TRACER_VALUE_I64 = 2, + DDOG_TRACER_VALUE_U64 = 3, + DDOG_TRACER_VALUE_F64 = 4, + DDOG_TRACER_VALUE_STRING = 5, + DDOG_TRACER_VALUE_BINARY = 6, + DDOG_TRACER_VALUE_ARRAY = 7, + DDOG_TRACER_VALUE_MAP = 8, +} ddog_TracerValueKind; """ [export] diff --git a/libdd-data-pipeline-ffi/src/lib.rs b/libdd-data-pipeline-ffi/src/lib.rs index c8f594f391..8f6505b1fa 100644 --- a/libdd-data-pipeline-ffi/src/lib.rs +++ b/libdd-data-pipeline-ffi/src/lib.rs @@ -8,6 +8,7 @@ mod error; mod response; +mod structured_value; mod trace_exporter; mod tracer; diff --git a/libdd-data-pipeline-ffi/src/structured_value.rs b/libdd-data-pipeline-ffi/src/structured_value.rs new file mode 100644 index 0000000000..61dcc8a033 --- /dev/null +++ b/libdd-data-pipeline-ffi/src/structured_value.rs @@ -0,0 +1,451 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::catch_panic; +use crate::error::{ExporterError, ExporterErrorCode as ErrorCode}; +#[cfg(all(feature = "catch_panic", panic = "unwind"))] +use crate::gen_error; +use libdd_common_ffi::slice::{AsBytes, ByteSlice, Slice}; +use std::ptr::NonNull; + +const DDOG_TRACER_VALUE_NIL: u8 = 0; +const DDOG_TRACER_VALUE_BOOL: u8 = 1; +const DDOG_TRACER_VALUE_I64: u8 = 2; +const DDOG_TRACER_VALUE_U64: u8 = 3; +const DDOG_TRACER_VALUE_F64: u8 = 4; +const DDOG_TRACER_VALUE_STRING: u8 = 5; +const DDOG_TRACER_VALUE_BINARY: u8 = 6; +const DDOG_TRACER_VALUE_ARRAY: u8 = 7; +const DDOG_TRACER_VALUE_MAP: u8 = 8; + +const MAX_DEPTH: u32 = 64; + +/// One value in a flat preorder representation of a structured value. +/// +/// Scalar tokens use the corresponding scalar field. String and binary tokens +/// use `bytes`. Array and map tokens use `child_count`; a map is followed by +/// two values per entry (key, then value). All other fields must be ignored. +/// Integer `kind` constants are used instead of a C enum so malformed tags can +/// be rejected without constructing an invalid Rust enum discriminant. +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct TracerValueToken<'a> { + pub kind: u8, + pub bool_value: u8, + pub child_count: u32, + pub i64_value: i64, + pub u64_value: u64, + pub f64_value: f64, + pub bytes: ByteSlice<'a>, +} + +/// Opaque owned MessagePack blob produced from structured-value tokens. +pub struct TracerEncodedValue(Vec); + +fn invalid_input(message: &str) -> Box { + Box::new(ExporterError::new(ErrorCode::InvalidInput, message)) +} + +fn write_len( + output: &mut Vec, + len: u32, + fix_base: u8, + fix_max: u32, + marker16: u8, + marker32: u8, +) { + if len <= fix_max { + output.push(fix_base | len as u8); + } else if u16::try_from(len).is_ok() { + output.push(marker16); + output.extend_from_slice(&(len as u16).to_be_bytes()); + } else { + output.push(marker32); + output.extend_from_slice(&len.to_be_bytes()); + } +} + +fn write_u64(output: &mut Vec, value: u64) { + if value <= 0x7f { + output.push(value as u8); + } else if value <= u8::MAX as u64 { + output.extend_from_slice(&[0xcc, value as u8]); + } else if value <= u16::MAX as u64 { + output.push(0xcd); + output.extend_from_slice(&(value as u16).to_be_bytes()); + } else if value <= u32::MAX as u64 { + output.push(0xce); + output.extend_from_slice(&(value as u32).to_be_bytes()); + } else { + output.push(0xcf); + output.extend_from_slice(&value.to_be_bytes()); + } +} + +fn write_i64(output: &mut Vec, value: i64) { + if value >= 0 { + write_u64(output, value as u64); + } else if value >= -32 { + output.push(value as i8 as u8); + } else if value >= i8::MIN as i64 { + output.extend_from_slice(&[0xd0, value as i8 as u8]); + } else if value >= i16::MIN as i64 { + output.push(0xd1); + output.extend_from_slice(&(value as i16).to_be_bytes()); + } else if value >= i32::MIN as i64 { + output.push(0xd2); + output.extend_from_slice(&(value as i32).to_be_bytes()); + } else { + output.push(0xd3); + output.extend_from_slice(&value.to_be_bytes()); + } +} + +fn write_bytes(output: &mut Vec, bytes: &[u8], string: bool) -> Result<(), Box> { + let len = u32::try_from(bytes.len()) + .map_err(|_| invalid_input("structured value byte string exceeds u32::MAX"))?; + if string { + std::str::from_utf8(bytes) + .map_err(|_| invalid_input("structured value string is not valid UTF-8"))?; + if len <= 31 { + output.push(0xa0 | len as u8); + } else if len <= u8::MAX as u32 { + output.extend_from_slice(&[0xd9, len as u8]); + } else if len <= u16::MAX as u32 { + output.push(0xda); + output.extend_from_slice(&(len as u16).to_be_bytes()); + } else { + output.push(0xdb); + output.extend_from_slice(&len.to_be_bytes()); + } + } else if len <= u8::MAX as u32 { + output.extend_from_slice(&[0xc4, len as u8]); + } else if len <= u16::MAX as u32 { + output.push(0xc5); + output.extend_from_slice(&(len as u16).to_be_bytes()); + } else { + output.push(0xc6); + output.extend_from_slice(&len.to_be_bytes()); + } + output.extend_from_slice(bytes); + Ok(()) +} + +fn encode_one( + tokens: &[TracerValueToken<'_>], + index: &mut usize, + depth: u32, + output: &mut Vec, +) -> Result<(), Box> { + let token = tokens + .get(*index) + .ok_or_else(|| invalid_input("structured value container is missing child tokens"))?; + *index += 1; + + match token.kind { + DDOG_TRACER_VALUE_NIL => output.push(0xc0), + DDOG_TRACER_VALUE_BOOL => match token.bool_value { + 0 => output.push(0xc2), + 1 => output.push(0xc3), + _ => return Err(invalid_input("structured value boolean must be 0 or 1")), + }, + DDOG_TRACER_VALUE_I64 => write_i64(output, token.i64_value), + DDOG_TRACER_VALUE_U64 => write_u64(output, token.u64_value), + DDOG_TRACER_VALUE_F64 => { + output.push(0xcb); + output.extend_from_slice(&token.f64_value.to_be_bytes()); + } + DDOG_TRACER_VALUE_STRING | DDOG_TRACER_VALUE_BINARY => { + let bytes = token + .bytes + .try_as_bytes() + .map_err(|_| invalid_input("structured value contains an invalid byte slice"))?; + write_bytes(output, bytes, token.kind == DDOG_TRACER_VALUE_STRING)?; + } + DDOG_TRACER_VALUE_ARRAY | DDOG_TRACER_VALUE_MAP => { + if depth >= MAX_DEPTH { + return Err(invalid_input( + "structured value exceeds maximum depth of 64", + )); + } + let values = if token.kind == DDOG_TRACER_VALUE_MAP { + token + .child_count + .checked_mul(2) + .ok_or_else(|| invalid_input("structured value map child count overflows"))? + } else { + token.child_count + }; + if token.kind == DDOG_TRACER_VALUE_ARRAY { + write_len(output, token.child_count, 0x90, 15, 0xdc, 0xdd); + } else { + write_len(output, token.child_count, 0x80, 15, 0xde, 0xdf); + } + for _ in 0..values { + encode_one(tokens, index, depth + 1, output)?; + } + } + _ => { + return Err(invalid_input( + "structured value contains an unknown token kind", + )) + } + } + Ok(()) +} + +/// Encode one flat preorder structured value as an owned MessagePack blob. +/// +/// On success, `out_handle` receives an owned blob that must be freed with +/// [`ddog_tracer_encoded_value_free`]. The input is fully validated and must +/// contain exactly one value. +/// +/// # Safety +/// +/// `tokens` and every byte slice referenced by its tokens must remain valid for +/// this call. `out_handle` must point to writable memory for a +/// `Box`. +#[no_mangle] +pub unsafe extern "C" fn ddog_tracer_encode_value( + tokens: Slice>, + out_handle: NonNull>, +) -> Option> { + catch_panic!( + { + let inner = || -> Result<(), Box> { + let tokens = tokens + .try_as_slice() + .map_err(|_| invalid_input("structured value token slice is invalid"))?; + if tokens.is_empty() { + return Err(invalid_input("structured value token slice is empty")); + } + let mut output = Vec::new(); + let mut index = 0; + encode_one(tokens, &mut index, 0, &mut output)?; + if index != tokens.len() { + return Err(invalid_input("structured value has trailing tokens")); + } + out_handle + .as_ptr() + .write(Box::new(TracerEncodedValue(output))); + Ok(()) + }; + inner().err() + }, + gen_error!(ErrorCode::Panic) + ) +} + +/// Borrow the bytes in an encoded value. The slice is valid until the blob is +/// freed. +#[no_mangle] +pub extern "C" fn ddog_tracer_encoded_value_as_slice( + value: Option<&TracerEncodedValue>, +) -> ByteSlice<'_> { + value + .map(|value| ByteSlice::from(value.0.as_slice())) + .unwrap_or_default() +} + +/// Free an encoded structured value and its bytes. +#[no_mangle] +pub extern "C" fn ddog_tracer_encoded_value_free(value: Option>) { + drop(value); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::MaybeUninit; + + fn token<'a>(kind: u8) -> TracerValueToken<'a> { + TracerValueToken { + kind, + bool_value: 0, + child_count: 0, + i64_value: 0, + u64_value: 0, + f64_value: 0.0, + bytes: ByteSlice::empty(), + } + } + + unsafe fn encode(tokens: &[TracerValueToken<'_>]) -> Result, Box> { + let mut handle = MaybeUninit::>::uninit(); + let out = NonNull::new(handle.as_mut_ptr()).unwrap(); + if let Some(error) = ddog_tracer_encode_value(Slice::from(tokens), out) { + return Err(error); + } + let blob = handle.assume_init(); + let bytes = ddog_tracer_encoded_value_as_slice(Some(&blob)) + .as_bytes() + .to_vec(); + ddog_tracer_encoded_value_free(Some(blob)); + Ok(bytes) + } + + #[test] + fn encodes_all_supported_values() { + let mut map = token(DDOG_TRACER_VALUE_MAP); + map.child_count = 7; + let mut bools = token(DDOG_TRACER_VALUE_ARRAY); + bools.child_count = 2; + let mut false_token = token(DDOG_TRACER_VALUE_BOOL); + false_token.bool_value = 0; + let mut true_token = token(DDOG_TRACER_VALUE_BOOL); + true_token.bool_value = 1; + let mut signed = token(DDOG_TRACER_VALUE_I64); + signed.i64_value = i64::MIN; + let mut unsigned = token(DDOG_TRACER_VALUE_U64); + unsigned.u64_value = u64::MAX; + let mut float = token(DDOG_TRACER_VALUE_F64); + float.f64_value = 1.25; + let mut binary = token(DDOG_TRACER_VALUE_BINARY); + binary.bytes = ByteSlice::from(&b"\0\xff"[..]); + + let strings = [ + "nil", "bools", "signed", "unsigned", "float", "string", "binary", "hello", + ]; + let string_tokens: Vec<_> = strings + .iter() + .map(|value| { + let mut t = token(DDOG_TRACER_VALUE_STRING); + t.bytes = ByteSlice::from(value.as_bytes()); + t + }) + .collect(); + let tokens = [ + map, + string_tokens[0], + token(DDOG_TRACER_VALUE_NIL), + string_tokens[1], + bools, + false_token, + true_token, + string_tokens[2], + signed, + string_tokens[3], + unsigned, + string_tokens[4], + float, + string_tokens[5], + string_tokens[7], + string_tokens[6], + binary, + ]; + + let encoded = unsafe { encode(&tokens).unwrap() }; + let mut expected = vec![ + 0x87, 0xa3, b'n', b'i', b'l', 0xc0, 0xa5, b'b', b'o', b'o', b'l', b's', 0x92, 0xc2, + 0xc3, 0xa6, b's', b'i', b'g', b'n', b'e', b'd', 0xd3, + ]; + expected.extend_from_slice(&i64::MIN.to_be_bytes()); + expected.extend_from_slice(&[0xa8, b'u', b'n', b's', b'i', b'g', b'n', b'e', b'd', 0xcf]); + expected.extend_from_slice(&u64::MAX.to_be_bytes()); + expected.extend_from_slice(&[0xa5, b'f', b'l', b'o', b'a', b't', 0xcb]); + expected.extend_from_slice(&1.25f64.to_be_bytes()); + expected.extend_from_slice(&[ + 0xa6, b's', b't', b'r', b'i', b'n', b'g', 0xa5, b'h', b'e', b'l', b'l', b'o', 0xa6, + b'b', b'i', b'n', b'a', b'r', b'y', 0xc4, 0x02, 0x00, 0xff, + ]); + assert_eq!(encoded, expected); + } + + #[test] + fn rejects_empty_missing_and_trailing_tokens() { + assert!(unsafe { encode(&[]) }.is_err()); + + let mut array = token(DDOG_TRACER_VALUE_ARRAY); + array.child_count = 1; + assert!(unsafe { encode(&[array]) }.is_err()); + assert!( + unsafe { encode(&[token(DDOG_TRACER_VALUE_NIL), token(DDOG_TRACER_VALUE_NIL),]) } + .is_err() + ); + } + + #[test] + fn rejects_invalid_kinds_booleans_and_utf8() { + assert!(unsafe { encode(&[token(255)]) }.is_err()); + + let mut boolean = token(DDOG_TRACER_VALUE_BOOL); + boolean.bool_value = 2; + assert!(unsafe { encode(&[boolean]) }.is_err()); + + let mut string = token(DDOG_TRACER_VALUE_STRING); + string.bytes = ByteSlice::from(&b"\xff"[..]); + assert!(unsafe { encode(&[string]) }.is_err()); + } + + #[test] + fn rejects_excessive_depth_and_map_count_overflow() { + let mut array = token(DDOG_TRACER_VALUE_ARRAY); + array.child_count = 1; + let mut tokens = vec![array; MAX_DEPTH as usize + 1]; + tokens.push(token(DDOG_TRACER_VALUE_NIL)); + assert!(unsafe { encode(&tokens) }.is_err()); + + let mut map = token(DDOG_TRACER_VALUE_MAP); + map.child_count = u32::MAX; + assert!(unsafe { encode(&[map]) }.is_err()); + } + + #[test] + fn encodes_length_boundaries() { + for (kind, marker32, marker255, marker256, marker65536) in [ + (DDOG_TRACER_VALUE_STRING, 0xd9, 0xd9, 0xda, 0xdb), + (DDOG_TRACER_VALUE_BINARY, 0xc4, 0xc4, 0xc5, 0xc6), + ] { + for (len, expected) in [ + (32, vec![marker32, 32]), + (255, vec![marker255, 255]), + (256, vec![marker256, 1, 0]), + (65_536, vec![marker65536, 0, 1, 0, 0]), + ] { + let bytes = vec![b'a'; len]; + let mut value = token(kind); + value.bytes = ByteSlice::from(bytes.as_slice()); + let encoded = unsafe { encode(&[value]).unwrap() }; + assert_eq!(&encoded[..expected.len()], expected); + assert_eq!(encoded.len(), expected.len() + len); + } + } + + let bytes31 = [b'a'; 31]; + let mut string31 = token(DDOG_TRACER_VALUE_STRING); + string31.bytes = ByteSlice::from(&bytes31[..]); + assert_eq!(unsafe { encode(&[string31]).unwrap() }[0], 0xbf); + + for (kind, count, header, values_per_entry) in [ + (DDOG_TRACER_VALUE_ARRAY, 15, vec![0x9f], 1), + (DDOG_TRACER_VALUE_ARRAY, 16, vec![0xdc, 0, 16], 1), + (DDOG_TRACER_VALUE_MAP, 15, vec![0x8f], 2), + (DDOG_TRACER_VALUE_MAP, 16, vec![0xde, 0, 16], 2), + ] { + let mut container = token(kind); + container.child_count = count; + let mut tokens = vec![container]; + tokens.extend(std::iter::repeat_n( + token(DDOG_TRACER_VALUE_NIL), + count as usize * values_per_entry, + )); + let encoded = unsafe { encode(&tokens).unwrap() }; + assert_eq!(&encoded[..header.len()], header); + } + } + + #[test] + fn rejects_invalid_token_slice() { + let tokens = unsafe { Slice::from_raw_parts(std::ptr::null(), 1) }; + let mut handle = MaybeUninit::>::uninit(); + let out = NonNull::new(handle.as_mut_ptr()).unwrap(); + let error = unsafe { ddog_tracer_encode_value(tokens, out) }; + assert!(error.is_some()); + } + + #[test] + fn null_blob_access_is_empty_and_free_is_safe() { + assert!(ddog_tracer_encoded_value_as_slice(None).is_empty()); + ddog_tracer_encoded_value_free(None); + } +} From 1c4db0d9e2f1839f8deff101f2c719d425825f3f Mon Sep 17 00:00:00 2001 From: Loic Nageleisen Date: Wed, 29 Jul 2026 19:33:24 +0200 Subject: [PATCH 3/6] test(data-pipeline): verify owned value blobs Exercise the generated C ABI from the packaged trace exporter example. Mutate the caller's source buffers after encoding and compare the blob with fixed MessagePack bytes to prove that output ownership is independent of the input lifetime. Mirror the ownership check in Rust and document that token byte slices are borrowed only for the synchronous encoding call. --- examples/ffi/trace_exporter.c | 44 +++++++++++++++++++ .../src/structured_value.rs | 38 +++++++++++++--- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/examples/ffi/trace_exporter.c b/examples/ffi/trace_exporter.c index 38277aabf6..d938bd0a45 100644 --- a/examples/ffi/trace_exporter.c +++ b/examples/ffi/trace_exporter.c @@ -58,8 +58,52 @@ int log_init(const char* log_path) { return 0; } +int verify_structured_value_encoder(void) { + uint8_t text[] = "stable"; + uint8_t binary[] = {0x00, 0xff}; + ddog_TracerValueToken tokens[] = { + {.kind = DDOG_TRACER_VALUE_ARRAY, .child_count = 2}, + { + .kind = DDOG_TRACER_VALUE_STRING, + .bytes = {.ptr = text, .len = sizeof(text) - 1}, + }, + { + .kind = DDOG_TRACER_VALUE_BINARY, + .bytes = {.ptr = binary, .len = sizeof(binary)}, + }, + }; + ddog_TracerEncodedValue *blob = NULL; + ddog_TraceExporterError *err = ddog_tracer_encode_value( + (ddog_Slice_TracerValueToken){ + .ptr = tokens, + .len = sizeof(tokens) / sizeof(tokens[0]), + }, + &blob); + if (err) { + handle_error(err); + return 1; + } + + memset(text, 'x', sizeof(text) - 1); + memset(binary, 'x', sizeof(binary)); + static const uint8_t expected[] = { + 0x92, 0xa6, 's','t','a','b','l','e', 0xc4, 0x02, 0x00, 0xff, + }; + ddog_ByteSlice encoded = ddog_tracer_encoded_value_as_slice(blob); + int matches = encoded.len == sizeof(expected) && + memcmp(encoded.ptr, expected, sizeof(expected)) == 0; + ddog_tracer_encoded_value_free(blob); + if (!matches) { + fprintf(stderr, "Structured value encoder did not return an owned blob\n"); + return 1; + } + return 0; +} + int main(int argc, char** argv) { + if (verify_structured_value_encoder() != 0) return 1; + // Initialize logger with optional path from command line const char* log_path = (argc > 1) ? argv[1] : NULL; if (log_init(log_path) != 0) { diff --git a/libdd-data-pipeline-ffi/src/structured_value.rs b/libdd-data-pipeline-ffi/src/structured_value.rs index 61dcc8a033..ae0e94c7cc 100644 --- a/libdd-data-pipeline-ffi/src/structured_value.rs +++ b/libdd-data-pipeline-ffi/src/structured_value.rs @@ -198,7 +198,8 @@ fn encode_one( /// /// On success, `out_handle` receives an owned blob that must be freed with /// [`ddog_tracer_encoded_value_free`]. The input is fully validated and must -/// contain exactly one value. +/// contain exactly one value. Token byte slices are borrowed only for this +/// synchronous call; the returned blob does not retain them. /// /// # Safety /// @@ -271,12 +272,7 @@ mod tests { } unsafe fn encode(tokens: &[TracerValueToken<'_>]) -> Result, Box> { - let mut handle = MaybeUninit::>::uninit(); - let out = NonNull::new(handle.as_mut_ptr()).unwrap(); - if let Some(error) = ddog_tracer_encode_value(Slice::from(tokens), out) { - return Err(error); - } - let blob = handle.assume_init(); + let blob = encode_blob(tokens)?; let bytes = ddog_tracer_encoded_value_as_slice(Some(&blob)) .as_bytes() .to_vec(); @@ -284,6 +280,17 @@ mod tests { Ok(bytes) } + unsafe fn encode_blob( + tokens: &[TracerValueToken<'_>], + ) -> Result, Box> { + let mut handle = MaybeUninit::>::uninit(); + let out = NonNull::new(handle.as_mut_ptr()).unwrap(); + if let Some(error) = ddog_tracer_encode_value(Slice::from(tokens), out) { + return Err(error); + } + Ok(handle.assume_init()) + } + #[test] fn encodes_all_supported_values() { let mut map = token(DDOG_TRACER_VALUE_MAP); @@ -448,4 +455,21 @@ mod tests { assert!(ddog_tracer_encoded_value_as_slice(None).is_empty()); ddog_tracer_encoded_value_free(None); } + + #[test] + fn returned_blob_does_not_borrow_token_bytes() { + let mut backing = b"stable".to_vec(); + let blob = { + let mut string = token(DDOG_TRACER_VALUE_STRING); + string.bytes = ByteSlice::from(backing.as_slice()); + unsafe { encode_blob(&[string]).unwrap() } + }; + + backing.fill(b'x'); + assert_eq!( + ddog_tracer_encoded_value_as_slice(Some(&blob)).as_bytes(), + b"\xa6stable" + ); + ddog_tracer_encoded_value_free(Some(blob)); + } } From 14b8478f9c31e0b1d403686059764ae9c2daaf16 Mon Sep 17 00:00:00 2001 From: Edmund Kump Date: Wed, 5 Aug 2026 16:39:26 -0400 Subject: [PATCH 4/6] replace hand rolled msgpack writing with rmp_writer, which we use everywhere else in libdatadog --- Cargo.lock | 1 + libdd-data-pipeline-ffi/Cargo.toml | 1 + .../src/structured_value.rs | 140 ++++++------------ 3 files changed, 44 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c9dd90cd8..a566c28748 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3023,6 +3023,7 @@ dependencies = [ "libdd-shared-runtime", "libdd-tinybytes", "libdd-trace-utils", + "rmp", "rmp-serde", "tokio-util", "tracing", diff --git a/libdd-data-pipeline-ffi/Cargo.toml b/libdd-data-pipeline-ffi/Cargo.toml index 01370a859c..6bc3f09e35 100644 --- a/libdd-data-pipeline-ffi/Cargo.toml +++ b/libdd-data-pipeline-ffi/Cargo.toml @@ -38,5 +38,6 @@ libdd-shared-runtime = { version = "2.0.0", path = "../libdd-shared-runtime" } libdd-common-ffi = { path = "../libdd-common-ffi", default-features = false } libdd-tinybytes = { path = "../libdd-tinybytes" } libdd-trace-utils = { path = "../libdd-trace-utils" } +rmp = { version = "0.8.14", default-features = false } tokio-util = "0.7.11" tracing.workspace = true \ No newline at end of file diff --git a/libdd-data-pipeline-ffi/src/structured_value.rs b/libdd-data-pipeline-ffi/src/structured_value.rs index ae0e94c7cc..9834bb5ee1 100644 --- a/libdd-data-pipeline-ffi/src/structured_value.rs +++ b/libdd-data-pipeline-ffi/src/structured_value.rs @@ -6,6 +6,10 @@ use crate::error::{ExporterError, ExporterErrorCode as ErrorCode}; #[cfg(all(feature = "catch_panic", panic = "unwind"))] use crate::gen_error; use libdd_common_ffi::slice::{AsBytes, ByteSlice, Slice}; +use rmp::encode::{ + write_array_len, write_bin, write_bool, write_f64, write_map_len, write_nil, write_sint, + write_str, write_uint, +}; use std::ptr::NonNull; const DDOG_TRACER_VALUE_NIL: u8 = 0; @@ -46,89 +50,21 @@ fn invalid_input(message: &str) -> Box { Box::new(ExporterError::new(ErrorCode::InvalidInput, message)) } -fn write_len( - output: &mut Vec, - len: u32, - fix_base: u8, - fix_max: u32, - marker16: u8, - marker32: u8, -) { - if len <= fix_max { - output.push(fix_base | len as u8); - } else if u16::try_from(len).is_ok() { - output.push(marker16); - output.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - output.push(marker32); - output.extend_from_slice(&len.to_be_bytes()); - } -} - -fn write_u64(output: &mut Vec, value: u64) { - if value <= 0x7f { - output.push(value as u8); - } else if value <= u8::MAX as u64 { - output.extend_from_slice(&[0xcc, value as u8]); - } else if value <= u16::MAX as u64 { - output.push(0xcd); - output.extend_from_slice(&(value as u16).to_be_bytes()); - } else if value <= u32::MAX as u64 { - output.push(0xce); - output.extend_from_slice(&(value as u32).to_be_bytes()); - } else { - output.push(0xcf); - output.extend_from_slice(&value.to_be_bytes()); - } +/// Adapts an `rmp` write failure into an exporter error. Writing into a `Vec` +/// cannot actually fail, so this only exists to satisfy the fallible `rmp` API. +fn encoding_failed(_: E) -> Box { + Box::new(ExporterError::new( + ErrorCode::Internal, + "structured value encoding failed", + )) } -fn write_i64(output: &mut Vec, value: i64) { - if value >= 0 { - write_u64(output, value as u64); - } else if value >= -32 { - output.push(value as i8 as u8); - } else if value >= i8::MIN as i64 { - output.extend_from_slice(&[0xd0, value as i8 as u8]); - } else if value >= i16::MIN as i64 { - output.push(0xd1); - output.extend_from_slice(&(value as i16).to_be_bytes()); - } else if value >= i32::MIN as i64 { - output.push(0xd2); - output.extend_from_slice(&(value as i32).to_be_bytes()); - } else { - output.push(0xd3); - output.extend_from_slice(&value.to_be_bytes()); - } -} - -fn write_bytes(output: &mut Vec, bytes: &[u8], string: bool) -> Result<(), Box> { - let len = u32::try_from(bytes.len()) - .map_err(|_| invalid_input("structured value byte string exceeds u32::MAX"))?; - if string { - std::str::from_utf8(bytes) - .map_err(|_| invalid_input("structured value string is not valid UTF-8"))?; - if len <= 31 { - output.push(0xa0 | len as u8); - } else if len <= u8::MAX as u32 { - output.extend_from_slice(&[0xd9, len as u8]); - } else if len <= u16::MAX as u32 { - output.push(0xda); - output.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - output.push(0xdb); - output.extend_from_slice(&len.to_be_bytes()); - } - } else if len <= u8::MAX as u32 { - output.extend_from_slice(&[0xc4, len as u8]); - } else if len <= u16::MAX as u32 { - output.push(0xc5); - output.extend_from_slice(&(len as u16).to_be_bytes()); - } else { - output.push(0xc6); - output.extend_from_slice(&len.to_be_bytes()); - } - output.extend_from_slice(bytes); - Ok(()) +/// `rmp`'s string and binary writers narrow the length to `u32` internally, so +/// reject anything longer up front rather than emit a truncated length prefix. +fn ensure_byte_len_fits(bytes: &[u8]) -> Result<(), Box> { + u32::try_from(bytes.len()) + .map(|_| ()) + .map_err(|_| invalid_input("structured value byte string exceeds u32::MAX")) } fn encode_one( @@ -143,24 +79,35 @@ fn encode_one( *index += 1; match token.kind { - DDOG_TRACER_VALUE_NIL => output.push(0xc0), - DDOG_TRACER_VALUE_BOOL => match token.bool_value { - 0 => output.push(0xc2), - 1 => output.push(0xc3), - _ => return Err(invalid_input("structured value boolean must be 0 or 1")), - }, - DDOG_TRACER_VALUE_I64 => write_i64(output, token.i64_value), - DDOG_TRACER_VALUE_U64 => write_u64(output, token.u64_value), - DDOG_TRACER_VALUE_F64 => { - output.push(0xcb); - output.extend_from_slice(&token.f64_value.to_be_bytes()); + DDOG_TRACER_VALUE_NIL => write_nil(output).map_err(encoding_failed)?, + DDOG_TRACER_VALUE_BOOL => { + let value = match token.bool_value { + 0 => false, + 1 => true, + _ => return Err(invalid_input("structured value boolean must be 0 or 1")), + }; + write_bool(output, value).map_err(encoding_failed)?; + } + DDOG_TRACER_VALUE_I64 => { + write_sint(output, token.i64_value).map_err(encoding_failed)?; + } + DDOG_TRACER_VALUE_U64 => { + write_uint(output, token.u64_value).map_err(encoding_failed)?; } + DDOG_TRACER_VALUE_F64 => write_f64(output, token.f64_value).map_err(encoding_failed)?, DDOG_TRACER_VALUE_STRING | DDOG_TRACER_VALUE_BINARY => { let bytes = token .bytes .try_as_bytes() .map_err(|_| invalid_input("structured value contains an invalid byte slice"))?; - write_bytes(output, bytes, token.kind == DDOG_TRACER_VALUE_STRING)?; + ensure_byte_len_fits(bytes)?; + if token.kind == DDOG_TRACER_VALUE_STRING { + let text = std::str::from_utf8(bytes) + .map_err(|_| invalid_input("structured value string is not valid UTF-8"))?; + write_str(output, text).map_err(encoding_failed)?; + } else { + write_bin(output, bytes).map_err(encoding_failed)?; + } } DDOG_TRACER_VALUE_ARRAY | DDOG_TRACER_VALUE_MAP => { if depth >= MAX_DEPTH { @@ -169,18 +116,15 @@ fn encode_one( )); } let values = if token.kind == DDOG_TRACER_VALUE_MAP { + write_map_len(output, token.child_count).map_err(encoding_failed)?; token .child_count .checked_mul(2) .ok_or_else(|| invalid_input("structured value map child count overflows"))? } else { + write_array_len(output, token.child_count).map_err(encoding_failed)?; token.child_count }; - if token.kind == DDOG_TRACER_VALUE_ARRAY { - write_len(output, token.child_count, 0x90, 15, 0xdc, 0xdd); - } else { - write_len(output, token.child_count, 0x80, 15, 0xde, 0xdf); - } for _ in 0..values { encode_one(tokens, index, depth + 1, output)?; } From b3952ec6220134db871171c1d5adac83730aecb4 Mon Sep 17 00:00:00 2001 From: Edmund Kump Date: Wed, 5 Aug 2026 17:27:33 -0400 Subject: [PATCH 5/6] return InvalidArgument for malformed structured-value slices to be consistent with rest of FFI API --- libdd-data-pipeline-ffi/src/structured_value.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/libdd-data-pipeline-ffi/src/structured_value.rs b/libdd-data-pipeline-ffi/src/structured_value.rs index 9834bb5ee1..efffdc7d01 100644 --- a/libdd-data-pipeline-ffi/src/structured_value.rs +++ b/libdd-data-pipeline-ffi/src/structured_value.rs @@ -50,6 +50,10 @@ fn invalid_input(message: &str) -> Box { Box::new(ExporterError::new(ErrorCode::InvalidInput, message)) } +fn invalid_argument(message: &str) -> Box { + Box::new(ExporterError::new(ErrorCode::InvalidArgument, message)) +} + /// Adapts an `rmp` write failure into an exporter error. Writing into a `Vec` /// cannot actually fail, so this only exists to satisfy the fallible `rmp` API. fn encoding_failed(_: E) -> Box { @@ -99,7 +103,7 @@ fn encode_one( let bytes = token .bytes .try_as_bytes() - .map_err(|_| invalid_input("structured value contains an invalid byte slice"))?; + .map_err(|_| invalid_argument("structured value contains an invalid byte slice"))?; ensure_byte_len_fits(bytes)?; if token.kind == DDOG_TRACER_VALUE_STRING { let text = std::str::from_utf8(bytes) @@ -160,7 +164,7 @@ pub unsafe extern "C" fn ddog_tracer_encode_value( let inner = || -> Result<(), Box> { let tokens = tokens .try_as_slice() - .map_err(|_| invalid_input("structured value token slice is invalid"))?; + .map_err(|_| invalid_argument("structured value token slice is invalid"))?; if tokens.is_empty() { return Err(invalid_input("structured value token slice is empty")); } @@ -391,7 +395,7 @@ mod tests { let mut handle = MaybeUninit::>::uninit(); let out = NonNull::new(handle.as_mut_ptr()).unwrap(); let error = unsafe { ddog_tracer_encode_value(tokens, out) }; - assert!(error.is_some()); + assert_eq!(error.as_ref().unwrap().code, ErrorCode::InvalidArgument); } #[test] From a6fe6f2e5302b73824d915fb71a82eeb1dc1d88a Mon Sep 17 00:00:00 2001 From: Edmund Kump Date: Wed, 5 Aug 2026 17:34:57 -0400 Subject: [PATCH 6/6] cover the structured-value encoder error path in the FFI example --- examples/ffi/trace_exporter.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/examples/ffi/trace_exporter.c b/examples/ffi/trace_exporter.c index 2c0128c909..5859f4556e 100644 --- a/examples/ffi/trace_exporter.c +++ b/examples/ffi/trace_exporter.c @@ -123,9 +123,38 @@ int verify_structured_value_encoder(void) { return 0; } +int verify_structured_value_encoder_rejects_invalid(void) { + ddog_TracerValueToken tokens[] = { + {.kind = 255}, + }; + ddog_TracerEncodedValue *blob = NULL; + ddog_TraceExporterError *err = ddog_tracer_encode_value( + (ddog_Slice_TracerValueToken){ + .ptr = tokens, + .len = sizeof(tokens) / sizeof(tokens[0]), + }, + &blob); + if (err == NULL) { + fprintf(stderr, "Structured value encoder accepted an unknown token kind\n"); + ddog_tracer_encoded_value_free(blob); + return 1; + } + ddog_trace_exporter_error_free(err); + + // The encoder writes the out-handle only on success, so a rejected input + // leaves the caller's NULL blob untouched and there is nothing to free. + if (blob != NULL) { + fprintf(stderr, "Structured value encoder wrote a blob on the error path\n"); + ddog_tracer_encoded_value_free(blob); + return 1; + } + return 0; +} + int main(int argc, char** argv) { if (verify_structured_value_encoder() != 0) return 1; + if (verify_structured_value_encoder_rejects_invalid() != 0) return 1; // Initialize logger with optional path from command line const char* log_path = (argc > 1) ? argv[1] : NULL;