diff --git a/e2e-tests/tests/backtrace_execution.rs b/e2e-tests/tests/backtrace_execution.rs index 9ba04c68..01ca5ca7 100644 --- a/e2e-tests/tests/backtrace_execution.rs +++ b/e2e-tests/tests/backtrace_execution.rs @@ -570,7 +570,10 @@ async fn test_hot_backtrace_compact_unwind_rows_cover_call_sites() -> anyhow::Re Ok(()) } +// Keep verifier-heavy backtrace loads out of the same kernel and shared +// container sandbox at the same time. Pure DWARF analysis tests stay parallel. #[tokio::test] +#[serial(backtrace_execution)] async fn test_special_stack_and_program_counter_registers_are_printable() -> anyhow::Result<()> { init(); @@ -610,6 +613,7 @@ trace hot_bt_probe { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_full_unwinds_complete_user_stack() -> anyhow::Result<()> { init(); @@ -653,6 +657,7 @@ trace dummy_touch { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_unwinds_register_return_address_rule() -> anyhow::Result<()> { init(); @@ -698,6 +703,7 @@ trace register_ra_leaf { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_preserves_same_value_frame_pointer_rule() -> anyhow::Result<()> { init(); @@ -743,6 +749,7 @@ trace same_value_rbp_leaf { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_undefined_return_address_completes() -> anyhow::Result<()> { init(); @@ -814,6 +821,7 @@ trace undefined_ra_tail_leaf { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_preserves_undefined_frame_pointer_state() -> anyhow::Result<()> { init(); @@ -887,6 +895,7 @@ trace undefined_rbp_tail_leaf { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_rejects_undefined_return_address_register() -> anyhow::Result<()> { init(); @@ -926,6 +935,51 @@ trace undefined_rbp_ra_inline_leaf { } #[tokio::test] +#[serial(backtrace_execution)] +async fn test_hot_backtrace_accepts_available_zero_return_address_register() -> anyhow::Result<()> { + init(); + + let script = r#" +trace zero_rbp_tail_leaf { + print "ZERO_RBP_TAIL_STACK"; + bt full; +} +"#; + let (count, stdout, stderr) = run_hot_backtrace_with_depth(script, 8).await?; + if count == 0 && stderr.contains("BPF_PROG_LOAD") { + return Ok(()); + } + let block = matching_backtrace_block_with_ordered_patterns_after( + &stdout, + &stderr, + "ZERO_RBP_TAIL_STACK", + 8, + "a tail-call stack whose return address uses an available zero-valued register", + &[ + "#0 zero_rbp_tail_leaf", + "#1 zero_rbp_tail_level_1", + "#2 zero_rbp_tail_level_2", + "#3 zero_rbp_tail_level_3", + "#4 zero_rbp_tail_level_4", + "#5 zero_rbp_tail_boundary", + ], + )?; + assert!( + block.contains("backtrace: complete, 6 frames (max 8)"), + "an available zero-valued return-address register should terminate cleanly\nBLOCK:\n{block}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}" + ); + assert!( + !block.contains("#6 ") + && !block.contains("stopped:") + && !block.contains("required-register-unavailable"), + "zero must not be confused with an unavailable register\nBLOCK:\n{block}" + ); + + Ok(()) +} + +#[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_runtime_unsupported_cfi_is_reported() -> anyhow::Result<()> { init(); @@ -1029,6 +1083,7 @@ trace unsupported_cfi_tail_leaf { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_defaults_to_max_depth_128() -> anyhow::Result<()> { init(); @@ -1060,6 +1115,7 @@ trace dummy_touch { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_depth_from_config_file() -> anyhow::Result<()> { init(); @@ -1106,6 +1162,7 @@ trace dummy_touch { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_depth_one_stops_after_current_frame() -> anyhow::Result<()> { init(); @@ -1135,6 +1192,7 @@ trace dummy_touch { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_non_power_of_two_depth_keeps_frame_slots_ordered() -> anyhow::Result<()> { init(); @@ -1177,6 +1235,7 @@ trace dummy_touch { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_multiple_deep_backtrace_statements_use_tail_calls() -> anyhow::Result<()> { init(); @@ -1221,6 +1280,7 @@ trace dummy_touch { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_conditional_tail_call_backtrace_ignores_skipped_slots() -> anyhow::Result<()> { init(); @@ -1284,6 +1344,7 @@ trace dummy_touch { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_default_bt_and_backtrace_alias_render_distinct_modes() -> anyhow::Result<()> { init(); @@ -1331,6 +1392,7 @@ trace dummy_touch { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_inline_and_noinline_bt_modes_control_inline_frames() -> anyhow::Result<()> { init(); @@ -1377,6 +1439,7 @@ trace inline_callsite_program.c:43 { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_non_pie_pid_backtrace_uses_load_bias_for_runtime_cfi_rows() -> anyhow::Result<()> { init(); @@ -1416,6 +1479,7 @@ trace dummy_touch { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_cross_module_backtrace_resolves_so_and_exe_frames() -> anyhow::Result<()> { init(); @@ -1482,7 +1546,7 @@ trace cross_module_lib_leaf { } #[tokio::test] -#[serial(backtrace_t_mode)] +#[serial(backtrace_execution)] async fn test_t_mode_cross_module_backtrace_resolves_so_and_exe_frames() -> anyhow::Result<()> { init(); if skip_if_nested_t_mode_unsupported() { @@ -1549,7 +1613,7 @@ trace cross_module_lib_leaf { } #[tokio::test] -#[serial(backtrace_t_mode)] +#[serial(backtrace_execution)] async fn test_t_mode_multiple_backtrace_traces_share_cross_module_cfi() -> anyhow::Result<()> { init(); if skip_if_nested_t_mode_unsupported() { @@ -1647,7 +1711,7 @@ trace cross_module_lib_leaf { } #[tokio::test] -#[serial(backtrace_dlopen)] +#[serial(backtrace_execution)] async fn test_pid_backtrace_reports_frame_from_library_loaded_by_dlopen() -> anyhow::Result<()> { init(); @@ -1731,7 +1795,7 @@ trace dlopen_main_callback { } #[tokio::test] -#[serial(backtrace_t_mode, backtrace_dlopen)] +#[serial(backtrace_execution)] async fn test_t_mode_backtrace_unwinds_library_loaded_by_dlopen() -> anyhow::Result<()> { init(); if skip_if_nested_t_mode_unsupported() { @@ -1802,7 +1866,7 @@ trace dlopen_main_callback { } #[tokio::test] -#[serial(backtrace_t_mode, backtrace_dlopen)] +#[serial(backtrace_execution)] async fn test_t_mode_backtrace_unwinds_shared_library_loaded_by_dlopen() -> anyhow::Result<()> { init(); if skip_if_nested_t_mode_unsupported() { @@ -1883,6 +1947,7 @@ trace dlopen_lib_leaf { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_full_renders_function_parameters() -> anyhow::Result<()> { init(); @@ -1943,6 +2008,7 @@ async fn test_function_parameters_use_signature_lookup_path() -> anyhow::Result< } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_raw_renders_debug_metadata() -> anyhow::Result<()> { init(); @@ -1972,6 +2038,7 @@ trace dummy_touch { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_hot_backtrace_full_keeps_up_with_raw() -> anyhow::Result<()> { init(); @@ -2012,6 +2079,7 @@ trace hot_bt_probe { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_backtrace_depth_128_loads_with_tail_calls() -> anyhow::Result<()> { init(); @@ -2055,6 +2123,7 @@ trace hot_bt_probe { } #[tokio::test] +#[serial(backtrace_execution)] async fn test_deep_full_backtrace_stays_warm_under_event_load() -> anyhow::Result<()> { init(); diff --git a/e2e-tests/tests/fixtures/backtrace_hot_program/Makefile b/e2e-tests/tests/fixtures/backtrace_hot_program/Makefile index 719606c1..17e15ae2 100644 --- a/e2e-tests/tests/fixtures/backtrace_hot_program/Makefile +++ b/e2e-tests/tests/fixtures/backtrace_hot_program/Makefile @@ -7,11 +7,12 @@ REGISTER_RA_OBJ ?= backtrace_register_ra.o SAME_VALUE_RBP_OBJ ?= backtrace_same_value_rbp.o UNDEFINED_RA_OBJ ?= backtrace_undefined_ra.o UNDEFINED_RBP_OBJ ?= backtrace_undefined_rbp.o +ZERO_RBP_OBJ ?= backtrace_zero_rbp.o UNSUPPORTED_CFI_OBJ ?= backtrace_unsupported_cfi.o all: $(BINARY) -$(BINARY): $(OBJ) $(REGISTER_RA_OBJ) $(SAME_VALUE_RBP_OBJ) $(UNDEFINED_RA_OBJ) $(UNDEFINED_RBP_OBJ) $(UNSUPPORTED_CFI_OBJ) +$(BINARY): $(OBJ) $(REGISTER_RA_OBJ) $(SAME_VALUE_RBP_OBJ) $(UNDEFINED_RA_OBJ) $(UNDEFINED_RBP_OBJ) $(ZERO_RBP_OBJ) $(UNSUPPORTED_CFI_OBJ) $(CC) $(CFLAGS) -o $@ $^ $(OBJ): backtrace_hot_program.c @@ -29,10 +30,13 @@ $(UNDEFINED_RA_OBJ): backtrace_undefined_ra.S $(UNDEFINED_RBP_OBJ): backtrace_undefined_rbp.S $(CC) $(CFLAGS) -c -o $@ $< +$(ZERO_RBP_OBJ): backtrace_zero_rbp.S + $(CC) $(CFLAGS) -c -o $@ $< + $(UNSUPPORTED_CFI_OBJ): backtrace_unsupported_cfi.S $(CC) $(CFLAGS) -c -o $@ $< -backtrace_hot_program_nopie: backtrace_hot_program_nopie.o $(REGISTER_RA_OBJ) $(SAME_VALUE_RBP_OBJ) $(UNDEFINED_RA_OBJ) $(UNDEFINED_RBP_OBJ) $(UNSUPPORTED_CFI_OBJ) +backtrace_hot_program_nopie: backtrace_hot_program_nopie.o $(REGISTER_RA_OBJ) $(SAME_VALUE_RBP_OBJ) $(UNDEFINED_RA_OBJ) $(UNDEFINED_RBP_OBJ) $(ZERO_RBP_OBJ) $(UNSUPPORTED_CFI_OBJ) $(CC) $(BASE_CFLAGS) -no-pie -o $@ $^ backtrace_hot_program_nopie.o: backtrace_hot_program.c diff --git a/e2e-tests/tests/fixtures/backtrace_hot_program/backtrace_hot_program.c b/e2e-tests/tests/fixtures/backtrace_hot_program/backtrace_hot_program.c index 72fd8abf..002a77b0 100644 --- a/e2e-tests/tests/fixtures/backtrace_hot_program/backtrace_hot_program.c +++ b/e2e-tests/tests/fixtures/backtrace_hot_program/backtrace_hot_program.c @@ -187,6 +187,38 @@ __attribute__((noinline)) void undefined_rbp_tail_level_2(uint64_t value) extern void undefined_rbp_tail_outer(uint64_t value); +__attribute__((noinline)) void zero_rbp_tail_leaf(uint64_t value) +{ + hot_sink += value; + asm volatile("" ::: "memory"); +} + +__attribute__((noinline)) void zero_rbp_tail_level_1(uint64_t value) +{ + zero_rbp_tail_leaf(value + 1); + asm volatile("" ::: "memory"); +} + +__attribute__((noinline)) void zero_rbp_tail_level_2(uint64_t value) +{ + zero_rbp_tail_level_1(value + 1); + asm volatile("" ::: "memory"); +} + +__attribute__((noinline)) void zero_rbp_tail_level_3(uint64_t value) +{ + zero_rbp_tail_level_2(value + 1); + asm volatile("" ::: "memory"); +} + +__attribute__((noinline)) void zero_rbp_tail_level_4(uint64_t value) +{ + zero_rbp_tail_level_3(value + 1); + asm volatile("" ::: "memory"); +} + +extern void zero_rbp_tail_boundary(uint64_t value); + __attribute__((noinline)) void unsupported_cfi_inline_leaf(uint64_t value) { hot_sink += value; @@ -243,6 +275,7 @@ int main(void) undefined_rbp_inline_outer(i); undefined_rbp_ra_inline_outer(i); undefined_rbp_tail_outer(i); + zero_rbp_tail_boundary(i); unsupported_cfi_inline_caller(i); unsupported_cfi_tail_caller(i); usleep(1000); diff --git a/e2e-tests/tests/fixtures/backtrace_hot_program/backtrace_zero_rbp.S b/e2e-tests/tests/fixtures/backtrace_hot_program/backtrace_zero_rbp.S new file mode 100644 index 00000000..081beebb --- /dev/null +++ b/e2e-tests/tests/fixtures/backtrace_hot_program/backtrace_zero_rbp.S @@ -0,0 +1,22 @@ + .text + .globl zero_rbp_tail_boundary + .type zero_rbp_tail_boundary, @function +zero_rbp_tail_boundary: + .cfi_startproc + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + # RBP is available even though its value is zero. The return-address rule + # uses it to terminate this synthetic stack cleanly at a tail-call boundary. + xorl %ebp, %ebp + .cfi_register %rip, %rbp + call zero_rbp_tail_level_4 + .cfi_restore %rip + popq %rbp + .cfi_restore %rbp + .cfi_def_cfa_offset 8 + ret + .cfi_endproc + .size zero_rbp_tail_boundary, .-zero_rbp_tail_boundary + + .section .note.GNU-stack,"",@progbits diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/frame_recovery.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/frame_recovery.rs index 2bcad5fa..039bae67 100644 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace/frame_recovery.rs +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/frame_recovery.rs @@ -41,15 +41,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let cfa_base = self.select_register_state(row.cfa_register, state, "bt_cfa_base")?; - let cfa_base_available = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - cfa_base, - self.context.i64_type().const_zero(), - "bt_cfa_base_available", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_base_available = + self.select_register_availability(row.cfa_register, state, "bt_cfa_base_available")?; let cfa = self .builder .build_int_add(cfa_base, row.cfa_offset, "bt_runtime_cfa") @@ -84,15 +77,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .builder .build_or(ra_is_register, ra_is_same, "bt_ra_register_like") .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let ra_register_available = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - ra_from_register, - self.context.i64_type().const_zero(), - "bt_ra_register_available", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ra_register_available = + self.select_register_availability(row.ra_register, state, "bt_ra_register_available")?; let ra_source_available = self .builder .build_select::, _>( @@ -172,6 +158,9 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let next_ip = next_ip.as_basic_value().into_int_value(); let next_rbp = self.recover_rbp_from_runtime_row(row, cfa, state, scratch)?; let error_code = self.load_i16(scratch.next_error_code_ptr, "bt_next_error_code_value")?; + // This recovery is inlined for every captured frame. Keep availability + // branchless so it does not add another verifier control-flow path at + // every frame in scripts with multiple backtraces. let error_code = self .builder .build_select::, _>( @@ -206,7 +195,18 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .builder .build_phi(self.context.i64_type(), "bt_next_rbp_phi") .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - rbp_phi.add_incoming(&[(&state.rbp, terminal_block), (&next_rbp, recovered_block)]); + rbp_phi.add_incoming(&[ + (&state.rbp, terminal_block), + (&next_rbp.value, recovered_block), + ]); + let rbp_available_phi = self + .builder + .build_phi(self.context.bool_type(), "bt_next_rbp_available_phi") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + rbp_available_phi.add_incoming(&[ + (&state.rbp_available, terminal_block), + (&next_rbp.available, recovered_block), + ]); let error_phi = self .builder .build_phi(self.context.i16_type(), "bt_next_error_phi") @@ -220,6 +220,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { ip: ip_phi.as_basic_value().into_int_value(), rsp: rsp_phi.as_basic_value().into_int_value(), rbp: rbp_phi.as_basic_value().into_int_value(), + rbp_available: rbp_available_phi.as_basic_value().into_int_value(), error_code: error_phi.as_basic_value().into_int_value(), }) } @@ -457,7 +458,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { cfa: IntValue<'ctx>, state: BtRegisterState<'ctx>, scratch: &BtScratch<'ctx>, - ) -> Result> { + ) -> Result> { let is_at = self.is_recovery_kind( row.rbp_kind, crate::BACKTRACE_RECOVERY_AT_CFA_OFFSET, @@ -481,6 +482,10 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { MemoryAccessSize::U64, "bt_rbp_read", )?; + let rbp_read_succeeded = self + .builder + .build_not(rbp_read_failed, "bt_rbp_read_succeeded") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.store_backtrace_error_code_if( scratch.next_error_code_ptr, rbp_read_failed, @@ -490,6 +495,9 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.builder .build_store(scratch.next_rbp_ptr, rbp_from_memory.into_int_value()) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.next_rbp_available_ptr, rbp_read_succeeded) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder .build_unconditional_branch(join_block) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; @@ -501,6 +509,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let rbp_from_register = self.select_register_state(row.rbp_register, state, "bt_rbp_reg")?; + let rbp_register_available = + self.select_register_availability(row.rbp_register, state, "bt_rbp_reg_available")?; let rbp_is_val = self.is_recovery_kind( row.rbp_kind, crate::BACKTRACE_RECOVERY_VAL_CFA_OFFSET, @@ -540,15 +550,41 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { ) .map_err(|e| CodeGenError::LLVMError(e.to_string()))? .into_int_value(); + let rbp_value_available = self + .builder + .build_select::, _>( + rbp_is_val, + self.context.bool_type().const_all_ones().into(), + self.context.bool_type().const_zero().into(), + "bt_rbp_val_available", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let rbp_non_at_available = self + .builder + .build_select::, _>( + rbp_is_register_like, + rbp_register_available.into(), + rbp_value_available.into(), + "bt_rbp_non_at_available", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); self.builder .build_store(scratch.next_rbp_ptr, rbp_non_at) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.next_rbp_available_ptr, rbp_non_at_available) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder .build_unconditional_branch(join_block) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder.position_at_end(join_block); - self.load_i64(scratch.next_rbp_ptr, "bt_next_rbp_value") + Ok(BtRecoveredRegister { + value: self.load_i64(scratch.next_rbp_ptr, "bt_next_rbp_value")?, + available: self.load_bool(scratch.next_rbp_available_ptr, "bt_next_rbp_available")?, + }) } pub(super) fn select_register_state( @@ -600,6 +636,34 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { .map_err(|e| CodeGenError::LLVMError(e.to_string())) } + pub(super) fn select_register_availability( + &self, + register: IntValue<'ctx>, + state: BtRegisterState<'ctx>, + name: &str, + ) -> Result> { + let is_rbp = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + register, + self.context + .i16_type() + .const_int(X86_64_DWARF_RBP as u64, false), + &format!("{name}_is_rbp"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_select::, _>( + is_rbp, + state.rbp_available.into(), + self.context.bool_type().const_all_ones().into(), + name, + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + pub(super) fn is_recovery_kind( &self, kind: IntValue<'ctx>, diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/inline.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/inline.rs index d47f1843..e30157ab 100644 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace/inline.rs +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/inline.rs @@ -127,6 +127,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let ip_ptr = self.build_entry_alloca(i64_type, "bt_state_ip")?; let rsp_ptr = self.build_entry_alloca(i64_type, "bt_state_rsp")?; let rbp_ptr = self.build_entry_alloca(i64_type, "bt_state_rbp")?; + let rbp_available_ptr = + self.build_entry_alloca(self.context.bool_type(), "bt_state_rbp_available")?; let module_bias_ptr = self.build_entry_alloca(i64_type, "bt_state_module_bias")?; let module_cookie_ptr = self.build_entry_alloca(i64_type, "bt_state_module_cookie")?; let module_found_ptr = @@ -161,6 +163,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { ip: self.load_i64(ip_ptr, "bt_initial_current_ip")?, rsp: self.load_i64(rsp_ptr, "bt_initial_current_rsp")?, rbp: self.load_i64(rbp_ptr, "bt_initial_current_rbp")?, + rbp_available: self.context.bool_type().const_all_ones(), }; let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; let validation = self.validate_backtrace_next_frame(state, next)?; @@ -234,6 +237,9 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.builder .build_store(rbp_ptr, next.rbp) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_available_ptr, next.rbp_available) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder .build_store(module_bias_ptr, frame_module.bias) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; @@ -315,6 +321,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { ip: self.load_i64(ip_ptr, "bt_current_ip")?, rsp: self.load_i64(rsp_ptr, "bt_current_rsp")?, rbp: self.load_i64(rbp_ptr, "bt_current_rbp")?, + rbp_available: self.load_bool(rbp_available_ptr, "bt_current_rbp_available")?, }; let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; @@ -395,6 +402,9 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.builder .build_store(rbp_ptr, next.rbp) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_available_ptr, next.rbp_available) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder .build_store(module_bias_ptr, frame_module.bias) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/mod.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/mod.rs index 218860f1..b09ea2d8 100644 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace/mod.rs +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/mod.rs @@ -68,6 +68,7 @@ struct RuntimeBtRowScratch<'ctx> { struct BtScratch<'ctx> { row: RuntimeBtRowScratch<'ctx>, next_rbp_ptr: PointerValue<'ctx>, + next_rbp_available_ptr: PointerValue<'ctx>, next_error_code_ptr: PointerValue<'ctx>, } @@ -76,6 +77,7 @@ struct BtRegisterState<'ctx> { ip: IntValue<'ctx>, rsp: IntValue<'ctx>, rbp: IntValue<'ctx>, + rbp_available: IntValue<'ctx>, } #[derive(Clone, Copy)] @@ -83,9 +85,16 @@ struct BtNextFrame<'ctx> { ip: IntValue<'ctx>, rsp: IntValue<'ctx>, rbp: IntValue<'ctx>, + rbp_available: IntValue<'ctx>, error_code: IntValue<'ctx>, } +#[derive(Clone, Copy)] +struct BtRecoveredRegister<'ctx> { + value: IntValue<'ctx>, + available: IntValue<'ctx>, +} + struct BtFrameValidation<'ctx> { valid: IntValue<'ctx>, complete: IntValue<'ctx>, @@ -149,7 +158,7 @@ mod tests { } #[test] - fn runtime_return_address_reads_are_guarded_by_the_memory_rule() { + fn runtime_return_address_reads_are_guarded_without_branching_on_availability() { let context = inkwell::context::Context::create(); let opts = CompileOptions::default(); let mut ctx = @@ -157,7 +166,10 @@ mod tests { let i8_type = context.i8_type(); let i16_type = context.i16_type(); let i64_type = context.i64_type(); - let fn_type = i64_type.fn_type(&[i8_type.into(), i64_type.into()], false); + let fn_type = i64_type.fn_type( + &[i8_type.into(), i64_type.into(), context.bool_type().into()], + false, + ); let function = ctx.module.add_function("bt_ra_recovery", fn_type, None); let entry = context.append_basic_block(function, "entry"); ctx.builder.position_at_end(entry); @@ -165,7 +177,7 @@ mod tests { let row = RuntimeBtUnwindRow { found: context.bool_type().const_all_ones(), unsupported: context.bool_type().const_zero(), - cfa_register: i16_type.const_int(X86_64_DWARF_RSP as u64, false), + cfa_register: i16_type.const_int(X86_64_DWARF_RBP as u64, false), cfa_offset: i64_type.const_int(8, false), ra_kind: function .get_nth_param(0) @@ -184,6 +196,10 @@ mod tests { .get_nth_param(1) .expect("RBP parameter") .into_int_value(), + rbp_available: function + .get_nth_param(2) + .expect("RBP availability parameter") + .into_int_value(), }; let scratch = ctx.allocate_backtrace_scratch().expect("backtrace scratch"); let next = ctx @@ -209,6 +225,11 @@ mod tests { let memory_block = &ir[memory_start..non_memory_start]; let non_memory_block = &ir[non_memory_start..join_start]; + assert!( + !ir.contains("br i1 %bt_required_registers_available"), + "register availability must remain branchless to avoid multiplying verifier paths across backtrace statements\nIR:\n{ir}" + ); + assert!( ir.contains("br i1 %bt_ra_at_kind, label %bt_ra_from_memory, label %bt_ra_non_memory"), "return-address recovery should branch on the memory rule before probing\nIR:\n{ir}" @@ -224,8 +245,9 @@ mod tests { ); assert!( ir.contains("bt_ra_register_available") - && ir.contains("bt_required_registers_available"), - "return-address register availability should participate in frame recovery\nIR:\n{ir}" + && ir.contains("bt_required_registers_available") + && ir.contains("bt_required_register_unavailable_code"), + "return-address register availability should select the frame recovery error\nIR:\n{ir}" ); } diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/tail_call.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/tail_call.rs index cf6741ba..b3d0d118 100644 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace/tail_call.rs +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/tail_call.rs @@ -163,6 +163,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let ip_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_ip")?; let rsp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rsp")?; let rbp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rbp")?; + let rbp_available_ptr = + self.build_entry_alloca(self.context.bool_type(), "bt_tail_prefix_rbp_available")?; let module_bias_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_module_bias")?; let module_cookie_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_module_cookie")?; @@ -183,6 +185,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { ip: raw_ip, rsp: initial_rsp, rbp: initial_rbp, + rbp_available: self.context.bool_type().const_all_ones(), }; let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; let validation = self.validate_backtrace_next_frame(state, next)?; @@ -256,6 +259,9 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.builder .build_store(rbp_ptr, next.rbp) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_available_ptr, next.rbp_available) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder .build_store(module_bias_ptr, frame_module.bias) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; @@ -330,6 +336,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { ip: self.load_i64(ip_ptr, "bt_tail_prefix_current_ip")?, rsp: self.load_i64(rsp_ptr, "bt_tail_prefix_current_rsp")?, rbp: self.load_i64(rbp_ptr, "bt_tail_prefix_current_rbp")?, + rbp_available: self + .load_bool(rbp_available_ptr, "bt_tail_prefix_current_rbp_available")?, }; let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; let validation = self.validate_backtrace_next_frame(state, next)?; @@ -410,6 +418,9 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { self.builder .build_store(rbp_ptr, next.rbp) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_available_ptr, next.rbp_available) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; self.builder .build_store(module_bias_ptr, frame_module.bias) .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; @@ -484,6 +495,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { let tail_ip = self.load_i64(ip_ptr, "bt_tail_state_prefix_ip")?; let tail_rsp = self.load_i64(rsp_ptr, "bt_tail_state_prefix_rsp")?; let tail_rbp = self.load_i64(rbp_ptr, "bt_tail_state_prefix_rbp")?; + let tail_rbp_available = + self.load_bool(rbp_available_ptr, "bt_tail_state_prefix_rbp_available")?; let tail_module_bias = self.load_i64(module_bias_ptr, "bt_tail_state_module_bias")?; let tail_module_cookie = self.load_i64(module_cookie_ptr, "bt_tail_state_module_cookie")?; let tail_module_found = self.load_bool(module_found_ptr, "bt_tail_state_module_found")?; @@ -505,6 +518,14 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { tail_rbp, "bt_state_rbp", )?; + let tail_rbp_available_u8 = + self.bool_to_u8(tail_rbp_available, "bt_state_rbp_available_u8")?; + self.store_u8_value( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_AVAILABLE_OFFSET, + tail_rbp_available_u8, + "bt_state_rbp_available", + )?; self.store_state_i64( state_ptr, crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, @@ -1040,6 +1061,20 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, "bt_step_current_rbp", )?; + let current_rbp_available_u8 = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_AVAILABLE_OFFSET, + "bt_step_current_rbp_available_u8", + )?; + let current_rbp_available = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + current_rbp_available_u8, + self.context.i8_type().const_zero(), + "bt_step_current_rbp_available", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; let module_bias = self.load_row_i64( state_ptr, crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, @@ -1133,6 +1168,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { ip: current_ip, rsp: current_rsp, rbp: current_rbp, + rbp_available: current_rbp_available, }; let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, scratch)?; let validation = self.validate_backtrace_next_frame(state, next)?; @@ -1222,6 +1258,14 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { next.rbp, "bt_step_state_next_rbp", )?; + let next_rbp_available_u8 = + self.bool_to_u8(next.rbp_available, "bt_step_next_rbp_available_u8")?; + self.store_u8_value( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_AVAILABLE_OFFSET, + next_rbp_available_u8, + "bt_step_state_next_rbp_available", + )?; self.store_state_i64( state_ptr, crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/unwind_rows.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/unwind_rows.rs index 3d47300e..7f812e41 100644 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace/unwind_rows.rs +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/unwind_rows.rs @@ -89,6 +89,8 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { rbp_offset_ptr: self.build_entry_alloca(i64_type, "bt_row_rbp_offset")?, }, next_rbp_ptr: self.build_entry_alloca(i64_type, "bt_next_rbp")?, + next_rbp_available_ptr: self + .build_entry_alloca(self.context.bool_type(), "bt_next_rbp_available")?, next_error_code_ptr: self.build_entry_alloca(i16_type, "bt_next_error_code")?, }) } diff --git a/ghostscope-compiler/src/lib.rs b/ghostscope-compiler/src/lib.rs index dfc39aff..94f87315 100644 --- a/ghostscope-compiler/src/lib.rs +++ b/ghostscope-compiler/src/lib.rs @@ -23,22 +23,23 @@ pub use ghostscope_protocol::bpf_abi::{ BACKTRACE_RECOVERY_AT_CFA_OFFSET, BACKTRACE_RECOVERY_REGISTER, BACKTRACE_RECOVERY_SAME_VALUE, BACKTRACE_RECOVERY_UNDEFINED, BACKTRACE_RECOVERY_VAL_CFA_OFFSET, BACKTRACE_TAIL_NO_NEXT_SLOT, BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, - BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, - BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, - BACKTRACE_TAIL_STATE_FLAGS_OFFSET, BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, - BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, - BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, - BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, - BACKTRACE_TAIL_STATE_SIZE, BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, - BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, - BACKTRACE_UNWIND_ROW_PC_END_OFFSET, BACKTRACE_UNWIND_ROW_PC_START_OFFSET, - BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, - BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, - BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, - BACKTRACE_UNWIND_ROW_SIZE, BACKTRACE_UNWIND_ROW_UNSUPPORTED_CFI, - BACKTRACE_UNWIND_WORDS_PER_ROW, BACKTRACE_UNWIND_WORD_CFA_OFFSET, BACKTRACE_UNWIND_WORD_PC_END, - BACKTRACE_UNWIND_WORD_PC_START, BACKTRACE_UNWIND_WORD_RA_OFFSET, - BACKTRACE_UNWIND_WORD_RBP_OFFSET, BACKTRACE_UNWIND_WORD_REGISTERS, + BACKTRACE_TAIL_STATE_CURRENT_RBP_AVAILABLE_OFFSET, BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, + BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, + BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, BACKTRACE_TAIL_STATE_FLAGS_OFFSET, + BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, + BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, + BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, + BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, BACKTRACE_TAIL_STATE_SIZE, + BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, + BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, BACKTRACE_UNWIND_ROW_PC_END_OFFSET, + BACKTRACE_UNWIND_ROW_PC_START_OFFSET, BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, + BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, + BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, + BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, BACKTRACE_UNWIND_ROW_SIZE, + BACKTRACE_UNWIND_ROW_UNSUPPORTED_CFI, BACKTRACE_UNWIND_WORDS_PER_ROW, + BACKTRACE_UNWIND_WORD_CFA_OFFSET, BACKTRACE_UNWIND_WORD_PC_END, BACKTRACE_UNWIND_WORD_PC_START, + BACKTRACE_UNWIND_WORD_RA_OFFSET, BACKTRACE_UNWIND_WORD_RBP_OFFSET, + BACKTRACE_UNWIND_WORD_REGISTERS, }; #[derive(Debug, thiserror::Error)] @@ -611,5 +612,6 @@ mod tests { assert_eq!(BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, 53); assert_eq!(BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, 54); assert_eq!(BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, 56); + assert_eq!(BACKTRACE_TAIL_STATE_CURRENT_RBP_AVAILABLE_OFFSET, 57); } } diff --git a/ghostscope-protocol/src/bpf_abi.rs b/ghostscope-protocol/src/bpf_abi.rs index edc57f1e..d603d7b1 100644 --- a/ghostscope-protocol/src/bpf_abi.rs +++ b/ghostscope-protocol/src/bpf_abi.rs @@ -293,6 +293,7 @@ pub struct BacktraceTailCallState { pub active_slot: u8, pub error_code: u16, pub next_slot: u8, + pub current_rbp_available: u8, } pub const BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET: usize = @@ -325,6 +326,8 @@ pub const BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET: usize = std::mem::offset_of!(BacktraceTailCallState, error_code); pub const BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET: usize = std::mem::offset_of!(BacktraceTailCallState, next_slot); +pub const BACKTRACE_TAIL_STATE_CURRENT_RBP_AVAILABLE_OFFSET: usize = + std::mem::offset_of!(BacktraceTailCallState, current_rbp_available); pub const BACKTRACE_TAIL_STATE_SIZE: usize = std::mem::size_of::(); pub const BACKTRACE_TAIL_NO_NEXT_SLOT: u8 = u8::MAX; @@ -496,5 +499,6 @@ mod tests { assert_eq!(BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, 53); assert_eq!(BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, 54); assert_eq!(BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, 56); + assert_eq!(BACKTRACE_TAIL_STATE_CURRENT_RBP_AVAILABLE_OFFSET, 57); } } diff --git a/ghostscope-protocol/src/lib.rs b/ghostscope-protocol/src/lib.rs index e16069ea..7de31235 100644 --- a/ghostscope-protocol/src/lib.rs +++ b/ghostscope-protocol/src/lib.rs @@ -58,25 +58,25 @@ pub use bpf_abi::{ BACKTRACE_RECOVERY_REGISTER, BACKTRACE_RECOVERY_SAME_VALUE, BACKTRACE_RECOVERY_UNDEFINED, BACKTRACE_RECOVERY_VAL_CFA_OFFSET, BACKTRACE_TAIL_NO_NEXT_SLOT, BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, - BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, - BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, - BACKTRACE_TAIL_STATE_FLAGS_OFFSET, BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, - BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, - BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, - BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, - BACKTRACE_TAIL_STATE_SIZE, BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, - BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, - BACKTRACE_UNWIND_ROW_PC_END_OFFSET, BACKTRACE_UNWIND_ROW_PC_START_OFFSET, - BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, - BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, - BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, - BACKTRACE_UNWIND_ROW_SIZE, BACKTRACE_UNWIND_ROW_UNSUPPORTED_CFI, - BACKTRACE_UNWIND_WORDS_PER_ROW, BACKTRACE_UNWIND_WORD_CFA_OFFSET, BACKTRACE_UNWIND_WORD_PC_END, - BACKTRACE_UNWIND_WORD_PC_START, BACKTRACE_UNWIND_WORD_RA_OFFSET, - BACKTRACE_UNWIND_WORD_RBP_OFFSET, BACKTRACE_UNWIND_WORD_REGISTERS, - PID_ALIAS_VALUE_PROC_PID_OFFSET, PID_ALIAS_VALUE_SIZE, PROC_MODULE_KEY_COOKIE_HI_OFFSET, - PROC_MODULE_KEY_COOKIE_LO_OFFSET, PROC_MODULE_KEY_PAD_OFFSET, PROC_MODULE_KEY_PID_OFFSET, - PROC_MODULE_KEY_SIZE, PROC_MODULE_OFFSETS_VALUE_BASE_OFFSET, + BACKTRACE_TAIL_STATE_CURRENT_RBP_AVAILABLE_OFFSET, BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, + BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, + BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, BACKTRACE_TAIL_STATE_FLAGS_OFFSET, + BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, + BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, + BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, + BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, BACKTRACE_TAIL_STATE_SIZE, + BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, + BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, BACKTRACE_UNWIND_ROW_PC_END_OFFSET, + BACKTRACE_UNWIND_ROW_PC_START_OFFSET, BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, + BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, + BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, + BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, BACKTRACE_UNWIND_ROW_SIZE, + BACKTRACE_UNWIND_ROW_UNSUPPORTED_CFI, BACKTRACE_UNWIND_WORDS_PER_ROW, + BACKTRACE_UNWIND_WORD_CFA_OFFSET, BACKTRACE_UNWIND_WORD_PC_END, BACKTRACE_UNWIND_WORD_PC_START, + BACKTRACE_UNWIND_WORD_RA_OFFSET, BACKTRACE_UNWIND_WORD_RBP_OFFSET, + BACKTRACE_UNWIND_WORD_REGISTERS, PID_ALIAS_VALUE_PROC_PID_OFFSET, PID_ALIAS_VALUE_SIZE, + PROC_MODULE_KEY_COOKIE_HI_OFFSET, PROC_MODULE_KEY_COOKIE_LO_OFFSET, PROC_MODULE_KEY_PAD_OFFSET, + PROC_MODULE_KEY_PID_OFFSET, PROC_MODULE_KEY_SIZE, PROC_MODULE_OFFSETS_VALUE_BASE_OFFSET, PROC_MODULE_OFFSETS_VALUE_BSS_OFFSET, PROC_MODULE_OFFSETS_VALUE_DATA_OFFSET, PROC_MODULE_OFFSETS_VALUE_RODATA_OFFSET, PROC_MODULE_OFFSETS_VALUE_SIZE, PROC_MODULE_OFFSETS_VALUE_SIZE_OFFSET, PROC_MODULE_OFFSETS_VALUE_TEXT_OFFSET,