Support LLVM 23 raw profile format (version 11)

Cargo.toml

@@ -33,6 +33,7 @@ __llvm_19 = []
 __llvm_20 = []
 __llvm_21 = []
 __llvm_22 = []
+__llvm_23 = []
 
 [dependencies]
 anyhow = "1.0.65"

src/instrumentation_profile/raw_profile.rs

@@ -71,6 +71,10 @@ pub struct Header {
     pub bitmap_delta: u64,
     pub num_vtables: u64,
     pub vnames_size: u64,
+    /// Raw profile version 11 (LLVM 23) added uniform-counter fields.
+    pub num_uniform_counters: u64,
+    pub padding_bytes_after_uniform_counters: u64,
+    pub uniform_counters_delta: u64,
 }
 
 impl Header {
@@ -119,12 +123,16 @@ pub struct ProfileData<T> {
     name_ref: u64,
     func_hash: u64,
     counter_ptr: T,
+    /// Raw profile version 11 (LLVM 23).
+    uniform_counter_ptr: Option<T>,
     bitmap_ptr: Option<T>,
     function_addr: T,
     values_ptr_expr: T,
     num_counters: u32,
     /// This might just be two values?
     num_value_sites: [u16; ValueKind::MemOpSize as usize + 1],
+    /// Raw profile version 11 (LLVM 23).
+    offload_device_wave_size: u16,
     num_bitmap_bytes: u32,
 }
 
@@ -343,8 +351,13 @@ where
                 let (bytes, data) = ProfileData::<T>::parse(input, &header)?;
                 debug!("Parsed data section {:?}", data);
                 data_section.push(data);
-                if version_num > 8 {
-                    let (bytes, v) = take(4usize)(bytes)?; // TODO WHAT AM I MISSING HERE?
+                // Struct tail padding. In v9/v10 ProfileData ends on a 4-byte field at offset 60,
+                // so the compiler pads to 64. In v11 (LLVM 23) OffloadDeviceWaveSize shifts
+                // NumBitmapBytes to offset 68, the struct ends at 72 which is already 8-aligned,
+                // and the padding moves INSIDE the struct (2 bytes, handled in ProfileData::parse).
+                // Taking 4 here as well would over-consume and desynchronise every later record.
+                if version_num > 8 && version_num < 11 {
+                    let (bytes, v) = take(4usize)(bytes)?;
                     debug!("Got those padding? bytes {:?}", v);
                     input = bytes;
                 } else {
@@ -392,6 +405,36 @@ where
             debug!("Applying padding bytes after counters");
             let (bytes, _) = take(counters_end)(input)?;
             input = bytes;
+
+            // compiler-rt writes the body as: data, PaddingBytesBeforeCounters, counters,
+            // PaddingBytesAfterCounters, bitmap, PaddingBytesAfterBitmapBytes, uniform counters,
+            // PaddingBytesAfterUniformCounters, names (see the IOVec list in
+            // lprofWriteDataImpl, compiler-rt/lib/profile/InstrProfilingWriter.c).
+            //
+            // The take above stops at the end of PaddingBytesAfterCounters, so everything
+            // between there and the names section has to be stepped over explicitly.
+            // Otherwise the names parser starts reading inside the bitmap or the uniform
+            // counter data and produces garbage symbol names.
+            //
+            // Both sections are zero-sized in the common case, which is why this went
+            // unnoticed: the bitmap is only populated for MC/DC instrumentation, and
+            // NumUniformCounters is 0 unless the uniform counter section is emitted. The
+            // arithmetic below is a no-op in that case.
+            let uniform_counters_size =
+                (header.num_uniform_counters as usize).saturating_mul(std::mem::size_of::<u64>());
+            let pre_names_sections = (header.num_bitmap_bytes as usize)
+                .saturating_add(header.padding_bytes_after_bitmap_bytes as usize)
+                .saturating_add(uniform_counters_size)
+                .saturating_add(header.padding_bytes_after_uniform_counters as usize);
+            if pre_names_sections > 0 {
+                debug!(
+                    "Skipping {} bytes of bitmap and uniform counter data before names",
+                    pre_names_sections
+                );
+                let (bytes, _) = take(pre_names_sections)(input)?;
+                input = bytes;
+            }
+
             let end_length = input.len() - header.names_len as usize;
             let mut names_section = Vec::with_capacity(data_section.len());
             while input.len() > end_length {
@@ -478,6 +521,22 @@ where
                     (bytes, 0, 0)
                 };
 
+            // Raw profile version 11 (LLVM 23) inserts three uint64 fields here, before
+            // NamesSize. Without reading them every subsequent field is off by 24 bytes.
+            let (
+                bytes,
+                num_uniform_counters,
+                padding_bytes_after_uniform_counters,
+                uniform_counters_delta,
+            ) = if (version & !VARIANT_MASKS_ALL) >= 11 {
+                let (bytes, num_uniform_counters) = nom_u64(endianness)(bytes)?;
+                let (bytes, padding_after) = nom_u64(endianness)(bytes)?;
+                let (bytes, uniform_delta) = nom_u64(endianness)(bytes)?;
+                (bytes, num_uniform_counters, padding_after, uniform_delta)
+            } else {
+                (bytes, 0, 0, 0)
+            };
+
             let (bytes, names_len) = nom_u64(endianness)(bytes)?;
             let (bytes, counters_delta) = nom_u64(endianness)(bytes)?;
 
@@ -517,6 +576,9 @@ where
                 bitmap_delta,
                 num_vtables,
                 vnames_size,
+                num_uniform_counters,
+                padding_bytes_after_uniform_counters,
+                uniform_counters_delta,
             };
             debug!("Read header {:?}", result);
             Ok((bytes, result))
@@ -554,6 +616,13 @@ where
         let (bytes, name_ref) = nom_u64(endianness)(bytes)?;
         let (bytes, func_hash) = nom_u64(endianness)(bytes)?;
         let (bytes, counter_ptr) = parse(bytes)?;
+        // v11 (LLVM 23) inserts UniformCounterPtr between CounterPtr and BitmapPtr.
+        let (bytes, uniform_counter_ptr) = if header.version() >= 11 {
+            let (bytes, p) = parse(bytes)?;
+            (bytes, Some(p))
+        } else {
+            (bytes, None)
+        };
         let (bytes, bitmap_ptr) = if header.version() > 8 {
             let (bytes, bitmap_ptr) = parse(bytes)?;
             (bytes, Some(bitmap_ptr))
@@ -565,6 +634,15 @@ where
         let (bytes, num_counters) = nom_u32(endianness)(bytes)?;
         let (bytes, value_0) = nom_u16(endianness)(bytes)?;
         let (bytes, value_1) = nom_u16(endianness)(bytes)?;
+        // v11 (LLVM 23) adds OffloadDeviceWaveSize after NumValueSites[]. In C that leaves the
+        // following uint32 misaligned, so the compiler inserts 2 bytes of padding; skip both.
+        let (bytes, offload_device_wave_size) = if header.version() >= 11 {
+            let (bytes, w) = nom_u16(endianness)(bytes)?;
+            let (bytes, _pad) = nom_u16(endianness)(bytes)?;
+            (bytes, w)
+        } else {
+            (bytes, 0)
+        };
         let (bytes, num_bitmap_bytes) = if header.version() > 8 {
             nom_u32(endianness)(bytes)?
         } else {
@@ -577,11 +655,13 @@ where
                 name_ref,
                 func_hash,
                 counter_ptr,
+                uniform_counter_ptr,
                 bitmap_ptr,
                 function_addr,
                 values_ptr_expr,
                 num_counters,
                 num_value_sites: [value_0, value_1],
+                offload_device_wave_size,
                 num_bitmap_bytes,
             },
         ))

tests/data/profdata/llvm-23/CSIR_profile.proftext

@@ -0,0 +1,11 @@
+# CSIR level Instrumentation Flag
+:csir
+bar
+# Func Hash:
+1152921534274394772
+# Num Counters:
+2
+# Counter Values:
+99938
+62
+

tests/data/profdata/llvm-23/FUnique.proftext

@@ -0,0 +1,30 @@
+# IR level Instrumentation Flag
+:ir
+_Z3barmi
+# Func Hash:
+784007056844089447
+# Num Counters:
+2
+# Counter Values:
+0
+0
+
+main
+# Func Hash:
+784007059655560962
+# Num Counters:
+2
+# Counter Values:
+1
+0
+
+test.c;_ZL3foom.__uniq.276699478366846449772231447066107882794
+# Func Hash:
+1124680652115249575
+# Num Counters:
+3
+# Counter Values:
+0
+0
+0
+

tests/data/profdata/llvm-23/IR_profile.proftext

@@ -0,0 +1,9 @@
+:ir
+main
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+1
+

tests/data/profdata/llvm-23/NoFUnique.proftext

@@ -0,0 +1,30 @@
+# IR level Instrumentation Flag
+:ir
+_Z3barmi
+# Func Hash:
+784007056844089447
+# Num Counters:
+2
+# Counter Values:
+0
+0
+
+main
+# Func Hash:
+784007059655560962
+# Num Counters:
+2
+# Counter Values:
+1
+0
+
+test.c;_ZL3foom
+# Func Hash:
+1124680652115249575
+# Num Counters:
+3
+# Counter Values:
+0
+0
+0
+

tests/data/profdata/llvm-23/bad-hash.proftext

@@ -0,0 +1,4 @@
+function_count_not
+badhash
+1
+1

tests/data/profdata/llvm-23/bar3-1.proftext

@@ -0,0 +1,6 @@
+bar
+3
+3
+1
+2
+3

tests/data/profdata/llvm-23/basic.proftext

@@ -0,0 +1,19 @@
+foo
+10
+2
+499500
+179900
+
+main
+16650
+4
+1
+1000
+1000000
+499500
+
+foo2
+10
+2
+500500
+180100

tests/data/profdata/llvm-23/clang_profile.proftext

@@ -0,0 +1,8 @@
+main
+# Func Hash:
+0
+# Num Counters:
+1
+# Counter Values:
+1
+

tests/data/profdata/llvm-23/counter-mismatch-1.proftext

@@ -0,0 +1,13 @@
+foo
+1024
+1
+0
+
+foo
+1024
+5
+0
+0
+0
+0
+0

tests/data/profdata/llvm-23/counter-mismatch-2.proftext

@@ -0,0 +1,5 @@
+foo
+1024
+2
+0
+0

tests/data/profdata/llvm-23/counter-mismatch-3.proftext

@@ -0,0 +1,6 @@
+foo
+1024
+3
+0
+0
+0

tests/data/profdata/llvm-23/counter-mismatch-4.proftext

@@ -0,0 +1,7 @@
+foo
+1024
+4
+0
+0
+0
+0

tests/data/profdata/llvm-23/cs-sample-preinline-probe.proftext

@@ -0,0 +1,48 @@
+[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]:1467299:11
+ 0: 6
+ 1: 6
+ 3: 287884
+ 4: 287864 _Z3fibi:315608
+ 15: 23
+ !CFGChecksum: 281479271677951
+ !Attributes: 2
+[main:3.1 @ _Z5funcBi:1 @ _Z8funcLeafi]:500853:20
+ 0: 15
+ 1: 15
+ 3: 74946
+ 4: 74941 _Z3fibi:82359
+ 10: 23324
+ 11: 23327 _Z3fibi:25228
+ 15: 11
+ !CFGChecksum: 281479271677951
+  !Attributes: 2
+[external:12 @ main]:154:12
+ 2: 12
+ 3: 10 _Z5funcAi:7
+ 3.1: 10 _Z5funcBi:11
+ !CFGChecksum: 563125815542069
+[main]:154:0
+ 2: 12
+ 3: 18 _Z5funcAi:11
+ 3.1: 18 _Z5funcBi:19
+ !CFGChecksum: 563125815542069
+[external:10 @ _Z5funcBi]:120:10
+ 0: 10
+ 1: 10
+ !CFGChecksum: 563022570642068
+[externalA:17 @ _Z5funcBi]:120:3
+ 0: 3
+ 1: 3
+ !CFGChecksum: 563022570642068
+[main:3.1 @ _Z5funcBi]:120:19
+ 0: 19
+ 1: 19 _Z8funcLeafi:20
+ 3: 12
+ !CFGChecksum: 563022570642068
+ !Attributes: 2
+[main:3 @ _Z5funcAi]:99:11
+ 0: 10
+ 1: 10 _Z8funcLeafi:11
+ 3: 24
+ !CFGChecksum: 844530426352218
+ !Attributes: 2