diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 7669f819fda04..4cd05af741278 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -639,7 +639,7 @@ void amd_uncore_df_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) info.split.aux_data = 0; info.split.num_pmcs = NUM_COUNTERS_NB; info.split.gid = 0; - info.split.cid = topology_logical_package_id(cpu); + info.split.cid = topology_amd_node_id(cpu); if (pmu_version >= 2) { ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES); @@ -899,8 +899,8 @@ void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu) cpuid(EXT_PERFMON_DEBUG_FEATURES, &eax, &ebx.full, &ecx, &edx); info.split.aux_data = ecx; /* stash active mask */ info.split.num_pmcs = ebx.split.num_umc_pmc; - info.split.gid = topology_logical_package_id(cpu); - info.split.cid = topology_logical_package_id(cpu); + info.split.gid = topology_amd_node_id(cpu); + info.split.cid = topology_amd_node_id(cpu); *per_cpu_ptr(uncore->info, cpu) = info; } diff --git a/tools/arch/x86/include/asm/amd/ibs.h b/tools/arch/x86/include/asm/amd/ibs.h index 8f19b05920456..e9662370e690d 100644 --- a/tools/arch/x86/include/asm/amd/ibs.h +++ b/tools/arch/x86/include/asm/amd/ibs.h @@ -67,7 +67,8 @@ union ibs_op_ctl { opmaxcnt_ext:7, /* 20-26: upper 7 bits of periodic op maximum count */ reserved0:5, /* 27-31: reserved */ opcurcnt:27, /* 32-58: periodic op counter current count */ - reserved1:5; /* 59-63: reserved */ + ldlat_thrsh:4, /* 59-62: Load Latency threshold */ + ldlat_en:1; /* 63: Load Latency enabled */ }; }; @@ -98,7 +99,9 @@ union ibs_op_data2 { rmt_node:1, /* 4: destination node */ cache_hit_st:1, /* 5: cache hit state */ data_src_hi:2, /* 6-7: data source high */ - reserved1:56; /* 8-63: reserved */ + strm_st:1, /* 8: streaming store */ + rmt_socket:1, /* 9: remote socket */ + reserved1:54; /* 10-63: reserved */ }; }; diff --git a/tools/perf/Documentation/perf-amd-ibs.txt b/tools/perf/Documentation/perf-amd-ibs.txt new file mode 100644 index 0000000000000..deaf0e5f184b2 --- /dev/null +++ b/tools/perf/Documentation/perf-amd-ibs.txt @@ -0,0 +1,237 @@ +perf-amd-ibs(1) +=============== + +NAME +---- +perf-amd-ibs - Support for AMD Instruction-Based Sampling (IBS) with perf tool + +SYNOPSIS +-------- +[verse] +'perf record' -e ibs_op// +'perf record' -e ibs_fetch// + +DESCRIPTION +----------- + +Instruction-Based Sampling (IBS) provides precise Instruction Pointer (IP) +profiling support on AMD platforms. IBS has two independent components: IBS +Op and IBS Fetch. IBS Op sampling provides information about instruction +execution (micro-op execution to be precise) with details like d-cache +hit/miss, d-TLB hit/miss, cache miss latency, load/store data source, branch +behavior etc. IBS Fetch sampling provides information about instruction fetch +with details like i-cache hit/miss, i-TLB hit/miss, fetch latency etc. IBS is +per-smt-thread i.e. each SMT hardware thread contains standalone IBS units. + +Both, IBS Op and IBS Fetch, are exposed as PMUs by Linux and can be exploited +using the Linux perf utility. The following files will be created at boot time +if IBS is supported by the hardware and kernel. + + /sys/bus/event_source/devices/ibs_op/ + /sys/bus/event_source/devices/ibs_fetch/ + +IBS Op PMU supports two events: cycles and micro ops. IBS Fetch PMU supports +one event: fetch ops. + +IBS PMUs do not have user/kernel filtering capability and thus it requires +CAP_SYS_ADMIN or CAP_PERFMON privilege. + +IBS VS. REGULAR CORE PMU +------------------------ + +IBS gives samples with precise IP, i.e. the IP recorded with IBS sample has +no skid. Whereas the IP recorded by regular core PMU will have some skid +(sample was generated at IP X but perf would record it at IP X+n). Hence, +regular core PMU might not help for profiling with instruction level +precision. Further, IBS provides additional information about the sample in +question. On the other hand, regular core PMU has it's own advantages like +plethora of events, counting mode (less interference), up to 6 parallel +counters, event grouping support, filtering capabilities etc. + +Three regular core PMU events are internally forwarded to IBS Op PMU when +precise_ip attribute is set: + + -e cpu-cycles:p becomes -e ibs_op// + -e r076:p becomes -e ibs_op// + -e r0C1:p becomes -e ibs_op/cnt_ctl=1/ + +EXAMPLES +-------- + +IBS Op PMU +~~~~~~~~~~ + +System-wide profile, cycles event, sampling period: 100000 + + # perf record -e ibs_op// -c 100000 -a + +Per-cpu profile (cpu10), cycles event, sampling period: 100000 + + # perf record -e ibs_op// -c 100000 -C 10 + +Userspace only, per-cpu profile (cpu10), cycles event, sampling period: 100000 + + Zen6 onward (See NOTES): + # perf record -e ibs_op//u -c 100000 -C 10 + + Until Zen5: + # perf record -e ibs_op/swfilt=1/u -c 100000 -C 10 + +Per-cpu profile (cpu10), cycles event, sampling freq: 1000 + + # perf record -e ibs_op// -F 1000 -C 10 + +System-wide profile, uOps event, sampling period: 100000 + + # perf record -e ibs_op/cnt_ctl=1/ -c 100000 -a + +Same command, but also capture IBS register raw dump along with perf sample: + + # perf record -e ibs_op/cnt_ctl=1/ -c 100000 -a --raw-samples + +System-wide profile, uOps event, sampling period: 100000, L3MissOnly (Zen4 onward) + + # perf record -e ibs_op/cnt_ctl=1,l3missonly=1/ -c 100000 -a + +System-wide profile, cycles event, sampling period: 100000, LdLat filtering (Zen5 +onward) + + # perf record -e ibs_op/ldlat=128/ -c 100000 -a + + Supported load latency threshold values are 128 to 2048 (both inclusive). + Latency value which is a multiple of 128 incurs a little less profiling + overhead compared to other values. + +System-wide profile, cycles event, sampling period: 100000, streaming store +filter (Zen6 onward) + + # perf record -e ibs_op/strmst=1/ -c 100000 -a + +Per process(upstream v6.2 onward), uOps event, sampling period: 100000 + + # perf record -e ibs_op/cnt_ctl=1/ -c 100000 -p 1234 + +Per process(upstream v6.2 onward), uOps event, sampling period: 100000 + + # perf record -e ibs_op/cnt_ctl=1/ -c 100000 -- ls + +To analyse recorded profile in aggregate mode + + # perf report + /* Select a line and press 'a' to drill down at instruction level. */ + +To go over each sample + + # perf script + +Raw dump of IBS registers when profiled with --raw-samples + + # perf report -D + /* Look for PERF_RECORD_SAMPLE */ + + Example register raw dump: + + ibs_op_ctl: 000002c30006186a MaxCnt 100000 L3MissOnly 0 En 1 + Val 1 CntCtl 0=cycles CurCnt 707 + IbsOpRip: ffffffff8204aea7 + ibs_op_data: 0000010002550001 CompToRetCtr 1 TagToRetCtr 597 + BrnRet 0 RipInvalid 0 BrnFuse 0 Microcode 1 + ibs_op_data2: 0000000000000013 RmtNode 1 DataSrc 3=DRAM + ibs_op_data3: 0000000031960092 LdOp 0 StOp 1 DcL1TlbMiss 0 + DcL2TlbMiss 0 DcL1TlbHit2M 1 DcL1TlbHit1G 0 DcL2TlbHit2M 0 + DcMiss 1 DcMisAcc 0 DcWcMemAcc 0 DcUcMemAcc 0 DcLockedOp 0 + DcMissNoMabAlloc 0 DcLinAddrValid 1 DcPhyAddrValid 1 + DcL2TlbHit1G 0 L2Miss 1 SwPf 0 OpMemWidth 32 bytes + OpDcMissOpenMemReqs 12 DcMissLat 0 TlbRefillLat 0 + IbsDCLinAd: ff110008a5398920 + IbsDCPhysAd: 00000008a5398920 + +IBS applied in a real world usecase + + ~90% regression was observed in tbench with specific scheduler hint + which was counter intuitive. IBS profile of good and bad run captured + using perf helped in identifying exact cause of the problem: + + https://lore.kernel.org/r/20220921063638.2489-1-kprateek.nayak@amd.com + +IBS Fetch PMU +~~~~~~~~~~~~~ + +Similar commands can be used with Fetch PMU as well. + +System-wide profile, fetch ops event, sampling period: 100000 + + # perf record -e ibs_fetch// -c 100000 -a + +Userspace only, system-wide profile, fetch ops event, sampling period: 100000 + + Zen6 onward (See NOTES): + # perf record -e ibs_fetch//u -c 100000 -a + + Until Zen5: + # perf record -e ibs_fetch/swfilt=1/u -c 100000 -a + +System-wide profile, fetch ops event, sampling period: 100000, Random enable + + # perf record -e ibs_fetch/rand_en=1/ -c 100000 -a + + Random enable adds small degree of variability to sample period. This + helps in cases like long running loops where PMU is tagging the same + instruction over and over because of fixed sample period. + +System-wide profile, fetch ops event, sampling period: 10000, fetch latency +filter (Zen6 onward) + + # perf record -e ibs_fetch/fetchlat=128/ -c 10000 -a + + Supported fetch latency threshold values are 128 to 1920 (both inclusive). + Latency value which is a multiple of 128 incurs a little less profiling + overhead compared to other values. + +etc. + +PERF MEM AND PERF C2C +--------------------- + +perf mem is a memory access profiler tool and perf c2c is a shared data +cacheline analyser tool. Both of them internally uses IBS Op PMU on AMD. +Below is a simple example of the perf mem tool. + + # perf mem record -c 100000 -- make + # perf mem report + +A normal perf mem report output will provide detailed memory access profile. +However, it can also be aggregated based on output fields. For example: + + # perf mem report -F mem,sample,snoop + Samples: 3M of event 'ibs_op//', Event count (approx.): 23524876 + Memory access Samples Snoop + N/A 1903343 N/A + L1 hit 1056754 N/A + L2 hit 75231 N/A + L3 hit 9496 HitM + L3 hit 2270 N/A + RAM hit 8710 N/A + Remote node, same socket RAM hit 3241 N/A + Remote core, same node Any cache hit 1572 HitM + Remote core, same node Any cache hit 514 N/A + Remote node, same socket Any cache hit 1216 HitM + Remote node, same socket Any cache hit 350 N/A + Uncached hit 18 N/A + +Please refer to their man page for more detail. + +NOTES +----- +Hardware privilege filtering uses bit 63 to distinguish between kernel +and userspace addresses. Hardware privilege filtering is not supported +on 32-bit systems. Also, the bit 63 convention is not universal and can +fail in specific environments, such as, using 64-bit host IBS to profile +a 32-bit guest, using 64-bit host IBS to profile non-Linux 64-bit guests +that do not adhere to the bit 63 privilege standard etc. + +SEE ALSO +-------- + +linkperf:perf-record[1], linkperf:perf-script[1], linkperf:perf-report[1], +linkperf:perf-mem[1], linkperf:perf-c2c[1] diff --git a/tools/perf/Documentation/perf.txt b/tools/perf/Documentation/perf.txt index ba3df49c169d3..fa20f54e347bb 100644 --- a/tools/perf/Documentation/perf.txt +++ b/tools/perf/Documentation/perf.txt @@ -77,7 +77,8 @@ linkperf:perf-stat[1], linkperf:perf-top[1], linkperf:perf-record[1], linkperf:perf-report[1], linkperf:perf-list[1] -linkperf:perf-annotate[1],linkperf:perf-archive[1],linkperf:perf-arm-spe[1], +linkperf:perf-amd-ibs[1], linkperf:perf-annotate[1], +linkperf:perf-archive[1], linkperf:perf-arm-spe[1], linkperf:perf-bench[1], linkperf:perf-buildid-cache[1], linkperf:perf-buildid-list[1], linkperf:perf-c2c[1], linkperf:perf-config[1], linkperf:perf-data[1], linkperf:perf-diff[1], diff --git a/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json b/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json index 03cb039434dee..71b883776f37b 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/amdzen6/floating-point.json @@ -212,7 +212,7 @@ { "EventName": "fp_ops_ret_by_type.scalar_logical", "EventCode": "0x0a", - "BriefDescription": "Retired scalar floating-point move uops.", + "BriefDescription": "Retired scalar floating-point logical uops.", "UMask": "0x0d" }, { @@ -665,6 +665,12 @@ "BriefDescription": "Retired 256-bit packed floating-point shuffle uops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "fp_pack_ops_ret.fp256_bfloat", + "EventCode": "0x0c", + "BriefDescription": "Retired 256-bit packed floating-point bfloat uops.", + "UMask": "0xc0" + }, { "EventName": "fp_pack_ops_ret.fp256_logical", "EventCode": "0x0c", @@ -758,7 +764,7 @@ { "EventName": "fp_pack_int_ops_ret.int128_vnni", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer VNNI ops.", + "BriefDescription": "Retired 128-bit packed integer VNNI uops.", "UMask": "0x0c" }, { @@ -803,12 +809,30 @@ "BriefDescription": "Retired 256-bit packed integer multiply-accumulate uops.", "UMask": "0x40" }, + { + "EventName": "fp_pack_int_ops_ret.int256_aes", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer AES uops.", + "UMask": "0x50" + }, + { + "EventName": "fp_pack_int_ops_ret.int256_sha", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer SHA uops.", + "UMask": "0x60" + }, { "EventName": "fp_pack_int_ops_ret.int256_cmp", "EventCode": "0x0d", "BriefDescription": "Retired 256-bit packed integer compare uops.", "UMask": "0x70" }, + { + "EventName": "fp_pack_int_ops_ret.int256_cvt", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer convert or pack uops.", + "UMask": "0x80" + }, { "EventName": "fp_pack_int_ops_ret.int256_shift", "EventCode": "0x0d", @@ -1083,19 +1107,19 @@ "EventName": "fp_nsq_read_stalls.fp_prf", "EventCode": "0x13", "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free physical register file (FP-PRF) entries.", - "UMask": "0x0e" + "UMask": "0x02" }, { "EventName": "fp_nsq_read_stalls.k_prf", "EventCode": "0x13", "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free mask physical register file (K-PRF) entries.", - "UMask": "0x0e" + "UMask": "0x04" }, { "EventName": "fp_nsq_read_stalls.fp_sq", "EventCode": "0x13", "BriefDescription": "Cycles when reads of the NSQ and writes to the floating-point or SIMD schedulers are stalled due to insufficient free scheduler entries.", - "UMask": "0x0e" + "UMask": "0x08" }, { "EventName": "fp_nsq_read_stalls.all", diff --git a/tools/perf/tests/shell/amd-ibs-swfilt.sh b/tools/perf/tests/shell/amd-ibs-swfilt.sh new file mode 100755 index 0000000000000..5d17dc624cbe6 --- /dev/null +++ b/tools/perf/tests/shell/amd-ibs-swfilt.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# AMD IBS software filtering + +cpu_family() { + grep -m1 '^cpu family[[:space:]]*:' /proc/cpuinfo \ + | awk -F: '{gsub(/^[ \t]+/, "", $2); print $2}' +} + +cpu_model() { + grep -m1 '^model[[:space:]]*:' /proc/cpuinfo \ + | awk -F: '{gsub(/^[ \t]+/, "", $2); print $2}' +} + +# IBS PMUs does not advertize privilege filtering capability. Rely +# on Family / Model check. +hw_priv_filter_supported() { + family=$(cpu_family) + model=$(cpu_model) + + if (( family > 0x1a )) || + { (( family == 0x1a )) && + { { (( model >= 0x50 && model <= 0x5f )) || + (( model >= 0x80 && model <= 0xaf )) || + (( model >= 0xc0 && model <= 0xcf )); }; }; }; then + return 0 # True + else + return 1 # False + fi +} + +ParanoidAndNotRoot() { + [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] +} + +echo "check availability of IBS swfilt" + +# check if IBS PMU is available +if [ ! -d /sys/bus/event_source/devices/ibs_op ]; then + echo "[SKIP] IBS PMU does not exist" + exit 2 +fi + +# check if IBS PMU has swfilt format +if [ ! -f /sys/bus/event_source/devices/ibs_op/format/swfilt ]; then + echo "[SKIP] IBS PMU does not have swfilt" + exit 2 +fi + +echo "run perf record with modifier and swfilt" +err=0 + +# setting any modifiers should fail +if ! hw_priv_filter_supported; then + perf record -B -e ibs_op//u -o /dev/null true 2> /dev/null + if [ $? -eq 0 ]; then + echo "[FAIL] IBS PMU should not accept exclude_kernel" + exit 1 + fi +fi + +# setting it with swfilt should be fine +perf record -B -e ibs_op/swfilt/u -o /dev/null true +if [ $? -ne 0 ]; then + echo "[FAIL] IBS op PMU cannot handle swfilt for exclude_kernel" + exit 1 +fi + +if ! ParanoidAndNotRoot 1 +then + # setting it with swfilt=1 should be fine + perf record -B -e ibs_op/swfilt=1/k -o /dev/null true + if [ $? -ne 0 ]; then + echo "[FAIL] IBS op PMU cannot handle swfilt for exclude_user" + exit 1 + fi +else + echo "[SKIP] not root and perf_event_paranoid too high for exclude_user" + err=2 +fi + +# check ibs_fetch PMU as well +perf record -B -e ibs_fetch/swfilt/u -o /dev/null true +if [ $? -ne 0 ]; then + echo "[FAIL] IBS fetch PMU cannot handle swfilt for exclude_kernel" + exit 1 +fi + +# check system wide recording +if ! ParanoidAndNotRoot 0 +then + perf record -aB --synth=no -e ibs_op/swfilt/k -o /dev/null true + if [ $? -ne 0 ]; then + echo "[FAIL] IBS op PMU cannot handle swfilt in system-wide mode" + exit 1 + fi +else + echo "[SKIP] not root and perf_event_paranoid too high for system-wide/exclude_user" + err=2 +fi + +echo "check number of samples with swfilt" + +kernel_sample=$(perf record -e ibs_op/swfilt/u -o- true | perf script -i- -F misc | grep -c ^K) +if [ ${kernel_sample} -ne 0 ]; then + echo "[FAIL] unexpected kernel samples: " ${kernel_sample} + exit 1 +fi + +if ! ParanoidAndNotRoot 1 +then + user_sample=$(perf record -e ibs_fetch/swfilt/k -o- true | perf script -i- -F misc | grep -c ^U) + if [ ${user_sample} -ne 0 ]; then + echo "[FAIL] unexpected user samples: " ${user_sample} + exit 1 + fi +else + echo "[SKIP] not root and perf_event_paranoid too high for exclude_user" + err=2 +fi + +exit $err diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c index 456ce64ad822c..eaddf75688ba1 100644 --- a/tools/perf/util/amd-sample-raw.c +++ b/tools/perf/util/amd-sample-raw.c @@ -19,6 +19,33 @@ static u32 cpu_family, cpu_model, ibs_fetch_type, ibs_op_type; static bool zen4_ibs_extensions; +static bool ldlat_cap; +static bool dtlb_pgsize_cap; +static bool rmtsocket_cap; +static bool strmst_cap; + +/* + * Status fields of IBS_FETCH_CTL and IBS_FETCH_CTL_EXT are valid only if + * IBS_FETCH_CTL[PhyAddrValid] is set. + */ +static int fetch_ctl_depends_on_phy_addr_valid(void) +{ + static int depends = -1; /* -1: Don't know, 1: Yes, 0: No */ + + if (depends != -1) + return depends; + + depends = 0; + if (cpu_family > 0x1a || + (cpu_family == 0x1a && ( + (cpu_model >= 0x50 && cpu_model <= 0x5f) || + (cpu_model >= 0x80 && cpu_model <= 0xaf) || + (cpu_model >= 0xc0 && cpu_model <= 0xcf)))) { + depends = 1; + } + + return depends; +} static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) { @@ -41,6 +68,18 @@ static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) const char *ic_miss_str = NULL; const char *l1tlb_pgsz_str = NULL; char l3_miss_str[sizeof(" L3MissOnly _ FetchOcMiss _ FetchL3Miss _")] = ""; + char l3_miss_only_str[sizeof(" L3MissOnly _")] = ""; + + if (fetch_ctl_depends_on_phy_addr_valid() && !reg.phy_addr_valid) { + snprintf(l3_miss_only_str, sizeof(l3_miss_only_str), + " L3MissOnly %d", reg.l3_miss_only); + + printf("ibs_fetch_ctl:\t%016llx MaxCnt %7d Cnt %7d En %d Val %d Comp %d " + "PhyAddrValid 0 RandEn %d%s\n", reg.val, reg.fetch_maxcnt << 4, + reg.fetch_cnt << 4, reg.fetch_en, reg.fetch_val, reg.fetch_comp, + reg.rand_en, l3_miss_only_str); + return; + } if (cpu_family == 0x19 && cpu_model < 0x10) { /* @@ -70,22 +109,31 @@ static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) l3_miss_str); } -static void pr_ic_ibs_extd_ctl(union ic_ibs_extd_ctl reg) +static void pr_ic_ibs_extd_ctl(union ibs_fetch_ctl fetch_ctl, union ic_ibs_extd_ctl reg) { + if (fetch_ctl_depends_on_phy_addr_valid() && !fetch_ctl.phy_addr_valid) + return; + printf("ic_ibs_ext_ctl:\t%016llx IbsItlbRefillLat %3d\n", reg.val, reg.itlb_refill_lat); } static void pr_ibs_op_ctl(union ibs_op_ctl reg) { char l3_miss_only[sizeof(" L3MissOnly _")] = ""; + char ldlat[sizeof(" LdLatThrsh __ LdLatEn _")] = ""; if (zen4_ibs_extensions) snprintf(l3_miss_only, sizeof(l3_miss_only), " L3MissOnly %d", reg.l3_miss_only); - printf("ibs_op_ctl:\t%016llx MaxCnt %9d%s En %d Val %d CntCtl %d=%s CurCnt %9d\n", + if (ldlat_cap) { + snprintf(ldlat, sizeof(ldlat), " LdLatThrsh %2d LdLatEn %d", + reg.ldlat_thrsh, reg.ldlat_en); + } + + printf("ibs_op_ctl:\t%016llx MaxCnt %9d%s En %d Val %d CntCtl %d=%s CurCnt %9d%s\n", reg.val, ((reg.opmaxcnt_ext << 16) | reg.opmaxcnt) << 4, l3_miss_only, reg.op_en, reg.op_val, reg.cnt_ctl, - reg.cnt_ctl ? "uOps" : "cycles", reg.opcurcnt); + reg.cnt_ctl ? "uOps" : "cycles", reg.opcurcnt, ldlat); } static void pr_ibs_op_data(union ibs_op_data reg) @@ -118,8 +166,16 @@ static void pr_ibs_op_data2_extended(union ibs_op_data2 reg) /* 13 to 31 are reserved. Avoid printing them. */ }; int data_src = (reg.data_src_hi << 3) | reg.data_src_lo; + char rmtsocket[sizeof("RmtSocket _ ")] = ""; + char strmst[sizeof("StrmSt _ ")] = ""; + + if (rmtsocket_cap) + snprintf(rmtsocket, sizeof(rmtsocket), "RmtSocket %d ", reg.rmt_socket); + if (strmst_cap) + snprintf(strmst, sizeof(strmst), "StrmSt %d ", reg.strm_st); - printf("ibs_op_data2:\t%016llx %sRmtNode %d%s\n", reg.val, + printf("ibs_op_data2:\t%016llx %s%s%sRmtNode %d%s\n", reg.val, + rmtsocket, strmst, (data_src == 1 || data_src == 2 || data_src == 5) ? (reg.cache_hit_st ? "CacheHitSt 1=O-State " : "CacheHitSt 0=M-state ") : "", reg.rmt_node, @@ -138,8 +194,16 @@ static void pr_ibs_op_data2_default(union ibs_op_data2 reg) " DataSrc 6=(reserved)", " DataSrc 7=Other" }; + char rmtsocket[sizeof("RmtSocket _ ")] = ""; + char strmst[sizeof("StrmSt _ ")] = ""; - printf("ibs_op_data2:\t%016llx %sRmtNode %d%s\n", reg.val, + if (rmtsocket_cap) + snprintf(rmtsocket, sizeof(rmtsocket), "RmtSocket %d ", reg.rmt_socket); + if (strmst_cap) + snprintf(strmst, sizeof(strmst), "StrmSt %d ", reg.strm_st); + + printf("ibs_op_data2:\t%016llx %s%s%sRmtNode %d%s\n", reg.val, + rmtsocket, strmst, reg.data_src_lo == 2 ? (reg.cache_hit_st ? "CacheHitSt 1=O-State " : "CacheHitSt 0=M-state ") : "", reg.rmt_node, data_src_str[reg.data_src_lo]); @@ -154,9 +218,21 @@ static void pr_ibs_op_data2(union ibs_op_data2 reg) static void pr_ibs_op_data3(union ibs_op_data3 reg) { - char l2_miss_str[sizeof(" L2Miss _")] = ""; - char op_mem_width_str[sizeof(" OpMemWidth _____ bytes")] = ""; + static const char * const dc_page_sizes[] = { + " 4K", + " 2M", + " 1G", + " ??", + }; char op_dc_miss_open_mem_reqs_str[sizeof(" OpDcMissOpenMemReqs __")] = ""; + char dc_l1_l2tlb_miss_str[sizeof(" DcL1TlbMiss _ DcL2TlbMiss _")] = ""; + char dc_l1tlb_hit_str[sizeof(" DcL1TlbHit2M _ DcL1TlbHit1G _")] = ""; + char op_mem_width_str[sizeof(" OpMemWidth _____ bytes")] = ""; + char tlb_refill_lat_str[sizeof(" TlbRefillLat _____")] = ""; + char dc_l2tlb_hit_2m_str[sizeof(" DcL2TlbHit2M _")] = ""; + char dc_l2tlb_hit_1g_str[sizeof(" DcL2TlbHit1G _")] = ""; + char dc_page_size_str[sizeof(" DcPageSize ____")] = ""; + char l2_miss_str[sizeof(" L2Miss _")] = ""; /* * Erratum #1293 @@ -172,16 +248,46 @@ static void pr_ibs_op_data3(union ibs_op_data3 reg) snprintf(op_mem_width_str, sizeof(op_mem_width_str), " OpMemWidth %2d bytes", 1 << (reg.op_mem_width - 1)); - printf("ibs_op_data3:\t%016llx LdOp %d StOp %d DcL1TlbMiss %d DcL2TlbMiss %d " - "DcL1TlbHit2M %d DcL1TlbHit1G %d DcL2TlbHit2M %d DcMiss %d DcMisAcc %d " - "DcWcMemAcc %d DcUcMemAcc %d DcLockedOp %d DcMissNoMabAlloc %d DcLinAddrValid %d " - "DcPhyAddrValid %d DcL2TlbHit1G %d%s SwPf %d%s%s DcMissLat %5d TlbRefillLat %5d\n", - reg.val, reg.ld_op, reg.st_op, reg.dc_l1tlb_miss, reg.dc_l2tlb_miss, - reg.dc_l1tlb_hit_2m, reg.dc_l1tlb_hit_1g, reg.dc_l2tlb_hit_2m, reg.dc_miss, - reg.dc_mis_acc, reg.dc_wc_mem_acc, reg.dc_uc_mem_acc, reg.dc_locked_op, - reg.dc_miss_no_mab_alloc, reg.dc_lin_addr_valid, reg.dc_phy_addr_valid, - reg.dc_l2_tlb_hit_1g, l2_miss_str, reg.sw_pf, op_mem_width_str, - op_dc_miss_open_mem_reqs_str, reg.dc_miss_lat, reg.tlb_refill_lat); + if (dtlb_pgsize_cap) { + if (reg.dc_phy_addr_valid) { + int idx = (reg.dc_l1tlb_hit_1g << 1) | reg.dc_l1tlb_hit_2m; + + snprintf(dc_l1_l2tlb_miss_str, sizeof(dc_l1_l2tlb_miss_str), + " DcL1TlbMiss %d DcL2TlbMiss %d", + reg.dc_l1tlb_miss, reg.dc_l2tlb_miss); + snprintf(dc_page_size_str, sizeof(dc_page_size_str), + " DcPageSize %4s", dc_page_sizes[idx]); + } + } else { + snprintf(dc_l1_l2tlb_miss_str, sizeof(dc_l1_l2tlb_miss_str), + " DcL1TlbMiss %d DcL2TlbMiss %d", + reg.dc_l1tlb_miss, reg.dc_l2tlb_miss); + snprintf(dc_l1tlb_hit_str, sizeof(dc_l1tlb_hit_str), + " DcL1TlbHit2M %d DcL1TlbHit1G %d", + reg.dc_l1tlb_hit_2m, reg.dc_l1tlb_hit_1g); + snprintf(dc_l2tlb_hit_2m_str, sizeof(dc_l2tlb_hit_2m_str), + " DcL2TlbHit2M %d", reg.dc_l2tlb_hit_2m); + snprintf(dc_l2tlb_hit_1g_str, sizeof(dc_l2tlb_hit_1g_str), + " DcL2TlbHit1G %d", reg.dc_l2_tlb_hit_1g); + } + + /* Use !zen4_ibs_extensions as a proxy for Zen3 and earlier */ + if (!zen4_ibs_extensions || reg.dc_phy_addr_valid) { + snprintf(tlb_refill_lat_str, sizeof(tlb_refill_lat_str), + " TlbRefillLat %5d", reg.tlb_refill_lat); + } + + printf("ibs_op_data3:\t%016llx LdOp %d StOp %d%s%s%s DcMiss %d DcMisAcc %d " + "DcWcMemAcc %d DcUcMemAcc %d DcLockedOp %d DcMissNoMabAlloc %d " + "DcLinAddrValid %d DcPhyAddrValid %d%s%s SwPf %d%s%s " + "DcMissLat %5d%s\n", + reg.val, reg.ld_op, reg.st_op, dc_l1_l2tlb_miss_str, + dtlb_pgsize_cap ? dc_page_size_str : dc_l1tlb_hit_str, + dc_l2tlb_hit_2m_str, reg.dc_miss, reg.dc_mis_acc, reg.dc_wc_mem_acc, + reg.dc_uc_mem_acc, reg.dc_locked_op, reg.dc_miss_no_mab_alloc, + reg.dc_lin_addr_valid, reg.dc_phy_addr_valid, dc_l2tlb_hit_1g_str, + l2_miss_str, reg.sw_pf, op_mem_width_str, op_dc_miss_open_mem_reqs_str, + reg.dc_miss_lat, tlb_refill_lat_str); } /* @@ -210,8 +316,12 @@ static void amd_dump_ibs_op(struct perf_sample *sample) pr_ibs_op_data3(*op_data3); if (op_data3->dc_lin_addr_valid) printf("IbsDCLinAd:\t%016llx\n", *(rip + 4)); - if (op_data3->dc_phy_addr_valid) + + /* Use !zen4_ibs_extensions as a proxy for Zen3 and earlier */ + if (op_data3->dc_phy_addr_valid && *(rip + 5) && + (!zen4_ibs_extensions || op_data3->dc_lin_addr_valid)) { printf("IbsDCPhysAd:\t%016llx\n", *(rip + 5)); + } if (op_data->op_brn_ret && *(rip + 6)) printf("IbsBrTarget:\t%016llx\n", *(rip + 6)); } @@ -231,7 +341,7 @@ static void amd_dump_ibs_fetch(struct perf_sample *sample) printf("IbsFetchLinAd:\t%016llx\n", *addr++); if (fetch_ctl->phy_addr_valid) printf("IbsFetchPhysAd:\t%016llx\n", *addr); - pr_ic_ibs_extd_ctl(*extd_ctl); + pr_ic_ibs_extd_ctl(*fetch_ctl, *extd_ctl); } /* @@ -331,6 +441,18 @@ bool evlist__has_amd_ibs(struct evlist *evlist) if (perf_env__find_pmu_cap(env, "ibs_op", "zen4_ibs_extensions")) zen4_ibs_extensions = 1; + if (perf_env__find_pmu_cap(env, "ibs_op", "ldlat")) + ldlat_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "dtlb_pgsize")) + dtlb_pgsize_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "rmtsocket")) + rmtsocket_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "strmst")) + strmst_cap = 1; + if (ibs_fetch_type || ibs_op_type) { if (!cpu_family) parse_cpuid(env);