diff --git a/MAINTAINERS b/MAINTAINERS index b413d5c97d413..b98c117048c7f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -22003,6 +22003,7 @@ Q: https://patchwork.kernel.org/project/linux-pm/list/ B: https://bugzilla.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git turbostat F: tools/power/x86/turbostat/ +F: tools/testing/selftests/turbostat/ TW5864 VIDEO4LINUX DRIVER M: Bluecherry Maintainers diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 9fd6bf8c7e9b0..083148877523e 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -7325,7 +7325,7 @@ __init int intel_pmu_init(void) case INTEL_FAM6_ATOM_SILVERMONT_D: case INTEL_FAM6_ATOM_SILVERMONT_MID: case INTEL_FAM6_ATOM_AIRMONT: - case INTEL_FAM6_ATOM_AIRMONT_MID: + case INTEL_FAM6_ATOM_SILVERMONT_MID2: memcpy(hw_cache_event_ids, slm_hw_cache_event_ids, sizeof(hw_cache_event_ids)); memcpy(hw_cache_extra_regs, slm_hw_cache_extra_regs, diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h index eacddbc9e4d7f..129a193820fef 100644 --- a/arch/x86/include/asm/intel-family.h +++ b/arch/x86/include/asm/intel-family.h @@ -199,6 +199,11 @@ #define INTEL_PANTHERLAKE_L IFM(6, 0xCC) +#define INTEL_WILDCATLAKE_L IFM(6, 0xD5) + +#define INTEL_NOVALAKE IFM(18, 0x01) +#define INTEL_NOVALAKE_L IFM(18, 0x03) + /* "Small Core" Processors (Atom/E-Core) */ #define INTEL_FAM6_ATOM_BONNELL 0x1C /* Diamondville, Pineview */ @@ -219,11 +224,11 @@ #define INTEL_ATOM_SILVERMONT_D IFM(6, 0x4D) /* Avaton, Rangely */ #define INTEL_FAM6_ATOM_SILVERMONT_MID 0x4A /* Merriefield */ #define INTEL_ATOM_SILVERMONT_MID IFM(6, 0x4A) /* Merriefield */ +#define INTEL_FAM6_ATOM_SILVERMONT_MID2 0x5A /* Anniedale */ +#define INTEL_ATOM_SILVERMONT_MID2 IFM(6, 0x5A) /* Anniedale */ #define INTEL_FAM6_ATOM_AIRMONT 0x4C /* Cherry Trail, Braswell */ #define INTEL_ATOM_AIRMONT IFM(6, 0x4C) /* Cherry Trail, Braswell */ -#define INTEL_FAM6_ATOM_AIRMONT_MID 0x5A /* Moorefield */ -#define INTEL_ATOM_AIRMONT_MID IFM(6, 0x5A) /* Moorefield */ #define INTEL_FAM6_ATOM_AIRMONT_NP 0x75 /* Lightning Mountain */ #define INTEL_ATOM_AIRMONT_NP IFM(6, 0x75) /* Lightning Mountain */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index da6a6299696b2..f7c17cd0c8e03 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1168,7 +1168,7 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { VULNWL_INTEL(CORE_YONAH, NO_SSB), - VULNWL_INTEL(ATOM_AIRMONT_MID, NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT), + VULNWL_INTEL(ATOM_SILVERMONT_MID2, NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT), VULNWL_INTEL(ATOM_AIRMONT_NP, NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT), VULNWL_INTEL(ATOM_GOLDMONT, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO), diff --git a/arch/x86/kernel/tsc_msr.c b/arch/x86/kernel/tsc_msr.c index 6555a857a1e6e..0f1c2ea419800 100644 --- a/arch/x86/kernel/tsc_msr.c +++ b/arch/x86/kernel/tsc_msr.c @@ -152,7 +152,7 @@ static const struct x86_cpu_id tsc_msr_cpu_ids[] = { X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &freq_desc_byt), X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &freq_desc_tng), X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &freq_desc_cht), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT_MID, &freq_desc_ann), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID2, &freq_desc_ann), X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT_NP, &freq_desc_lgm), {} }; diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index c105030372102..19d8d7f8bef1a 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -1285,7 +1285,7 @@ static const struct x86_cpu_id rapl_ids[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &rapl_defaults_byt), X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &rapl_defaults_cht), X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &rapl_defaults_tng), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT_MID, &rapl_defaults_ann), + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID2, &rapl_defaults_ann), X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &rapl_defaults_core), diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h index 487ef5846c248..c57ec9dfd32d3 100644 --- a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h +++ b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h @@ -229,10 +229,10 @@ int v4l2_get_acpi_sensor_info(struct device *dev, char **module_id_str); #define IS_BYT __IS_SOC(INTEL_FAM6_ATOM_SILVERMONT) #define IS_CHT __IS_SOC(INTEL_FAM6_ATOM_AIRMONT) #define IS_MRFD __IS_SOC(INTEL_FAM6_ATOM_SILVERMONT_MID) -#define IS_MOFD __IS_SOC(INTEL_FAM6_ATOM_AIRMONT_MID) +#define IS_MOFD __IS_SOC(INTEL_FAM6_ATOM_SILVERMONT_MID2) /* Both CHT and MOFD come with ISP2401 */ #define IS_ISP2401 __IS_SOCS(INTEL_FAM6_ATOM_AIRMONT, \ - INTEL_FAM6_ATOM_AIRMONT_MID) + INTEL_FAM6_ATOM_SILVERMONT_MID2) #endif /* ATOMISP_PLATFORM_H_ */ diff --git a/tools/power/x86/turbostat/Makefile b/tools/power/x86/turbostat/Makefile index 92e139b9c792d..3946d5254a1fd 100644 --- a/tools/power/x86/turbostat/Makefile +++ b/tools/power/x86/turbostat/Makefile @@ -3,6 +3,8 @@ CC = $(CROSS_COMPILE)gcc BUILD_OUTPUT := $(CURDIR) PREFIX ?= /usr DESTDIR ?= +DAY := $(shell date +%Y.%m.%d) +SNAPSHOT = turbostat-$(DAY) ifeq ("$(origin O)", "command line") BUILD_OUTPUT := $(O) @@ -12,6 +14,7 @@ turbostat : turbostat.c override CFLAGS += -O2 -Wall -Wextra -I../../../include override CFLAGS += -DMSRHEADER='"../../../../arch/x86/include/asm/msr-index.h"' override CFLAGS += -DINTEL_FAMILY_HEADER='"../../../../arch/x86/include/asm/intel-family.h"' +override CFLAGS += -DBUILD_BUG_HEADER='"../../../../include/linux/build_bug.h"' override CFLAGS += -D_FILE_OFFSET_BITS=64 override CFLAGS += -D_FORTIFY_SOURCE=2 @@ -22,9 +25,34 @@ override CFLAGS += -D_FORTIFY_SOURCE=2 .PHONY : clean clean : @rm -f $(BUILD_OUTPUT)/turbostat + @rm -f $(SNAPSHOT).tar.gz install : turbostat - install -d $(DESTDIR)$(PREFIX)/bin + install -d $(DESTDIR)$(PREFIX)/bin install $(BUILD_OUTPUT)/turbostat $(DESTDIR)$(PREFIX)/bin/turbostat - install -d $(DESTDIR)$(PREFIX)/share/man/man8 + install -d $(DESTDIR)$(PREFIX)/share/man/man8 install -m 644 turbostat.8 $(DESTDIR)$(PREFIX)/share/man/man8 + +snapshot: turbostat + @rm -rf $(SNAPSHOT) + @mkdir $(SNAPSHOT) + @cp turbostat Makefile turbostat.c turbostat.8 ../../../../arch/x86/include/asm/intel-family.h $(SNAPSHOT) + + @sed -e 's/^#include /#include "bits.h"/' ../../../../arch/x86/include/asm/msr-index.h > $(SNAPSHOT)/msr-index.h + @echo '#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))' >> $(SNAPSHOT)/msr-index.h + @echo "#define BIT(x) (1 << (x))" > $(SNAPSHOT)/bits.h + @echo "#define BIT_ULL(nr) (1ULL << (nr))" >> $(SNAPSHOT)/bits.h + @echo "#define GENMASK(h, l) (((~0UL) << (l)) & (~0UL >> (sizeof(long) * 8 - 1 - (h))))" >> $(SNAPSHOT)/bits.h + @echo "#define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (sizeof(long long) * 8 - 1 - (h))))" >> $(SNAPSHOT)/bits.h + + @echo '#define BUILD_BUG_ON(cond) do { enum { compile_time_check ## __COUNTER__ = 1/(!(cond)) }; } while (0)' > $(SNAPSHOT)/build_bug.h + @echo '#define __must_be_array(arr) 0' >> $(SNAPSHOT)/build_bug.h + + @echo PWD=. > $(SNAPSHOT)/Makefile + @echo "CFLAGS += -DMSRHEADER='\"msr-index.h\"'" >> $(SNAPSHOT)/Makefile + @echo "CFLAGS += -DINTEL_FAMILY_HEADER='\"intel-family.h\"'" >> $(SNAPSHOT)/Makefile + @echo "CFLAGS += -DBUILD_BUG_HEADER='\"build_bug.h\"'" >> $(SNAPSHOT)/Makefile + @sed -e's/.*MSRHEADER.*//' -e's/.*INTEL_FAMILY_HEADER.*//' -e's/.*BUILD_BUG_HEADER.*//' Makefile >> $(SNAPSHOT)/Makefile + + @rm -f $(SNAPSHOT).tar.gz + tar cvzf $(SNAPSHOT).tar.gz $(SNAPSHOT) diff --git a/tools/power/x86/turbostat/turbostat.8 b/tools/power/x86/turbostat/turbostat.8 index 1ba6340d3b3da..1551fcdbfd8a0 100644 --- a/tools/power/x86/turbostat/turbostat.8 +++ b/tools/power/x86/turbostat/turbostat.8 @@ -28,10 +28,16 @@ name as necessary to disambiguate it from others is necessary. Note that option .PP \fB--add attributes\fP add column with counter having specified 'attributes'. The 'location' attribute is required, all others are optional. .nf - location: {\fBmsrDDD\fP | \fBmsr0xXXX\fP | \fB/sys/path...\fP} + location: {\fBmsrDDD\fP | \fBmsr0xXXX\fP | \fB/sys/path...\fP | \fBperf//\fP} msrDDD is a decimal offset, eg. msr16 msr0xXXX is a hex offset, eg. msr0x10 /sys/path... is an absolute path to a sysfs attribute + is a perf device from /sys/bus/event_source/devices/ eg. cstate_core + On Intel hybrid platforms, instead of one "cpu" perf device there are two, "cpu_core" and "cpu_atom" devices for P and E cores respectively. + Turbostat, in this case, allow user to use "cpu" device and will automatically detect the type of a CPU and translate it to "cpu_core" and "cpu_atom" accordingly. + For a complete example see "ADD PERF COUNTER EXAMPLE #2 (using virtual "cpu" device)". + is a perf event for given device from /sys/bus/event_source/devices//events/ eg. c1-residency + perf/cstate_core/c1-residency would then use /sys/bus/event_source/devices/cstate_core/events/c1-residency scope: {\fBcpu\fP | \fBcore\fP | \fBpackage\fP} sample and print the counter for every cpu, core, or package. @@ -41,10 +47,11 @@ name as necessary to disambiguate it from others is necessary. Note that option MSRs are read as 64-bits, u32 truncates the displayed value to 32-bits. default: u64 - format: {\fBraw\fP | \fBdelta\fP | \fBpercent\fP} + format: {\fBraw\fP | \fBdelta\fP | \fBpercent\fP | \fBaverage\fP} 'raw' shows the MSR contents in hex. 'delta' shows the difference in values during the measurement interval. 'percent' shows the delta as a percentage of the cycles elapsed. + 'average' similar to raw, but also averaged for node/package summaries (or when using -S). default: delta name: "name_string" @@ -52,6 +59,39 @@ name as necessary to disambiguate it from others is necessary. Note that option as the column header. .fi .PP +\fB--add pmt,[attr_name=attr_value, ...]\fP add column with a PMT (Intel Platform Monitoring Technology) counter in a similar way to --add option above, but require PMT metadata to be supplied to correctly read and display the counter. The metadata can be found in the Intel PMT XML files, hosted at https://github.com/intel/Intel-PMT. For a complete example see "ADD PMT COUNTER EXAMPLE". +.nf + name="name_string" + For column header. + + type={\fBraw\fP} + 'raw' shows the counter contents in hex. + default: raw + + format={\fBraw\fP | \fBdelta\fP} + 'raw' shows the counter contents in hex. + 'delta' shows the difference in values during the measurement interval. + default: raw + + domain={\fBcpu%u\fP | \fBcore%u\fP | \fBpackage%u\fP} + 'cpu' per cpu/thread counter. + 'core' per core counter. + 'package' per package counter. + '%u' denotes id of the domain that the counter is associated with. For example core4 would mean that the counter is associated with core number 4. + + offset=\fB%u\fP + '%u' offset within the PMT MMIO region. + + lsb=\fB%u\fP + '%u' least significant bit within the 64 bit value read from 'offset'. Together with 'msb', used to form a read mask. + + msb=\fB%u\fP + '%u' most significant bit within the 64 bit value read from 'offset'. Together with 'lsb', used to form a read mask. + + guid=\fB%x\fP + '%x' hex identifier of the PMT MMIO region. +.fi +.PP \fB--cpu cpu-set\fP limit output to system summary plus the specified cpu-set. If cpu-set is the string "core", then the system summary plus the first CPU in each core are printed -- eg. subsequent HT siblings are not printed. Or if cpu-set is the string "package", then the system summary plus the first CPU in each package is printed. Otherwise, the system summary plus the specified set of CPUs are printed. The cpu-set is ordered from low to high, comma delimited with ".." and "-" permitted to denote a range. eg. 1,2,8,14..17,21-44 .PP \fB--hide column\fP do not show the specified built-in columns. May be invoked multiple times, or with a comma-separated list of column names. @@ -61,12 +101,16 @@ The column name "all" can be used to enable all disabled-by-default built-in cou .PP \fB--show column\fP show only the specified built-in columns. May be invoked multiple times, or with a comma-separated list of column names. .PP -\fB--show CATEGORY --hide CATEGORY\fP Show and hide also accept a single CATEGORY of columns: "all", "topology", "idle", "frequency", "power", "sysfs", "other". +\fB--show CATEGORY --hide CATEGORY\fP Show and hide also accept a comma-separated-list of CATEGORIES of columns: "all", "topology", "idle", "frequency", "power", "cpuidle", "hwidle", "swidle", "cache", "llc", "other". "idle" (enabled by default), includes "hwidle" and "pct_idle". "cpuidle" (default disabled) includes cpuidle software invocation counters. "swidle" includes "cpuidle" plus "pct_idle". "hwidle" includes only hardware based idle residency counters. Older versions of turbostat used the term "sysfs" for what is now "swidle". .PP \fB--Dump\fP displays the raw counter values. .PP \fB--quiet\fP Do not decode and print the system configuration header information. .PP +\fB--no-msr\fP Disable all the uses of the MSR driver. +.PP +\fB--no-perf\fP Disable all the uses of the perf API. +.PP \fB--interval seconds\fP overrides the default 5.0 second measurement interval. .PP \fB--num_iterations num\fP number of the measurement iterations. @@ -93,7 +137,7 @@ displays the statistics gathered since it was forked. The system configuration dump (if --quiet is not used) is followed by statistics. The first row of the statistics labels the content of each column (below). The second row of statistics is the system summary line. The system summary line has a '-' in the columns for the Package, Core, and CPU. The contents of the system summary line depends on the type of column. Columns that count items (eg. IRQ) show the sum across all CPUs in the system. Columns that show a percentage show the average across all CPUs in the system. Columns that dump raw MSR values simply show 0 in the summary. After the system summary row, each row describes a specific Package/Core/CPU. Note that if the --cpu parameter is used to limit which specific CPUs are displayed, turbostat will still collect statistics for all CPUs in the system and will still show the system summary for all CPUs in the system. .SH COLUMN DESCRIPTIONS .PP -\fBusec\fP For each CPU, the number of microseconds elapsed during counter collection, including thread migration -- if any. This counter is disabled by default, and is enabled with "--enable usec", or --debug. On the summary row, usec refers to the total elapsed time to collect the counters on all cpus. +\fBusec\fP For each CPU, the number of microseconds elapsed during counter collection, including thread migration -- if any. This counter is disabled by default, and is enabled with "--enable usec", or --debug. On the summary row, usec refers to the total elapsed time to snapshot the procfs/sysfs and collect the counters on all cpus. .PP \fBTime_Of_Day_Seconds\fP For each CPU, the gettimeofday(2) value (seconds.subsec since Epoch) when the counters ending the measurement interval were collected. This column is disabled by default, and can be enabled with "--enable Time_Of_Day_Seconds" or "--debug". On the summary row, Time_Of_Day_Seconds refers to the timestamp following collection of counters on the last CPU. .PP @@ -115,19 +159,45 @@ The system configuration dump (if --quiet is not used) is followed by statistics .PP \fBSMI\fP The number of System Management Interrupts serviced CPU during the measurement interval. While this counter is actually per-CPU, SMI are triggered on all processors, so the number should be the same for all CPUs. .PP -\fBC1, C2, C3...\fP The number times Linux requested the C1, C2, C3 idle state during the measurement interval. The system summary line shows the sum for all CPUs. These are C-state names as exported in /sys/devices/system/cpu/cpu*/cpuidle/state*/name. While their names are generic, their attributes are processor specific. They the system description section of output shows what MWAIT sub-states they are mapped to on each system. +\fBLLCkRPS\fP Last Level Cache Thousands of References Per Second. For CPUs with an L3 LLC, this is the number of references that CPU made to the L3 (and the number of misses that CPU made to it's L2). For CPUs with an L2 LLC, this is the number of references to the L2 (and the number of misses to the CPU's L1). The system summary row shows the sum for all CPUs. In both cases, the value displayed is the actual value divided by 1000 in the interest of usually fitting into 8 columns. .PP -\fBC1%, C2%, C3%\fP The residency percentage that Linux requested C1, C2, C3.... The system summary is the average of all CPUs in the system. Note that these are software, reflecting what was requested. The hardware counters reflect what was actually achieved. +\fBLLC%hit\fP Last Level Cache Hit Rate %. Hit Rate Percent = 100.0 * (References - Misses)/References. The system summary row shows the weighted average for all CPUs (100.0 * (Sum_References - Sum_Misses)/Sum_References). .PP -\fBCPU%c1, CPU%c3, CPU%c6, CPU%c7\fP show the percentage residency in hardware core idle states. These numbers are from hardware residency counters. +\fBC1, C2, C3...\fP The number times Linux requested the C1, C2, C3 idle state during the measurement interval. The system summary line shows the sum for all CPUs. These are C-state names as exported in /sys/devices/system/cpu/cpu*/cpuidle/state*/name. While their names are generic, their attributes are processor specific. They the system description section of output shows what MWAIT sub-states they are mapped to on each system. These counters are in the "cpuidle" group, which is disabled, by default. +.PP +\fBC1+, C2+, C3+...\fP The idle governor idle state misprediction statistics. Inidcates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a deeper idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/below file. These counters are in the "cpuidle" group, which is disabled, by default. +.PP +\fBC1-, C2-, C3-...\fP The idle governor idle state misprediction statistics. Inidcates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a shallower idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/above file. These counters are in the "cpuidle" group, which is disabled, by default. +.PP +\fBC1%, C2%, C3%\fP The residency percentage that Linux requested C1, C2, C3.... The system summary is the average of all CPUs in the system. Note that these are software, reflecting what was requested. The hardware counters reflect what was actually achieved. These counters are in the "pct_idle" group, which is enabled by default. +.PP +\fBCPU%c1, CPU%c3, CPU%c6, CPU%c7\fP show the percentage residency in hardware core idle states. These numbers are from hardware residency counters and are in the "hwidle" group, which is enabled, by default. .PP \fBCoreTmp\fP Degrees Celsius reported by the per-core Digital Thermal Sensor. .PP \fBPkgTmp\fP Degrees Celsius reported by the per-package Package Thermal Monitor. .PP -\fBGFX%rc6\fP The percentage of time the GPU is in the "render C6" state, rc6, during the measurement interval. From /sys/class/drm/card0/power/rc6_residency_ms. +\fBCoreThr\fP Core Thermal Throttling events during the measurement interval. Note that events since boot can be find in /sys/devices/system/cpu/cpu*/thermal_throttle/* +.PP +\fBGFX%rc6\fP The percentage of time the GPU is in the "render C6" state, rc6, during the measurement interval. From /sys/class/drm/card0/power/rc6_residency_ms or /sys/class/drm/card0/gt/gt0/rc6_residency_ms or /sys/class/drm/card0/device/tile0/gtN/gtidle/idle_residency_ms depending on the graphics driver being used. +.PP +\fBGFXMHz\fP Instantaneous snapshot of what sysfs presents at the end of the measurement interval. From /sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz or /sys/class/drm/card0/gt_cur_freq_mhz or /sys/class/drm/card0/gt/gt0/rps_cur_freq_mhz or /sys/class/drm/card0/device/tile0/gtN/freq0/cur_freq depending on the graphics driver being used. +.PP +\fBGFXAMHz\fP Instantaneous snapshot of what sysfs presents at the end of the measurement interval. From /sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz or /sys/class/drm/card0/gt_act_freq_mhz or /sys/class/drm/card0/gt/gt0/rps_act_freq_mhz or /sys/class/drm/card0/device/tile0/gtN/freq0/act_freq depending on the graphics driver being used. +.PP +\fBSAM%mc6\fP The percentage of time the SA Media is in the "module C6" state, mc6, during the measurement interval. From /sys/class/drm/card0/gt/gt1/rc6_residency_ms or /sys/class/drm/card0/device/tile0/gtN/gtidle/idle_residency_ms depending on the graphics driver being used. +.PP +\fBSAMMHz\fP Instantaneous snapshot of what sysfs presents at the end of the measurement interval. From /sys/class/drm/card0/gt/gt1/rps_cur_freq_mhz or /sys/class/drm/card0/device/tile0/gtN/freq0/cur_freq depending on the graphics driver being used. +.PP +\fBSAMAMHz\fP Instantaneous snapshot of what sysfs presents at the end of the measurement interval. From /sys/class/drm/card0/gt/gt1/rps_act_freq_mhz or /sys/class/drm/card0/device/tile0/gtN/freq0/act_freq depending on the graphics driver being used. +.PP +\fBTotl%C0\fP Weighted percentage of time that CPUs are busy. If N CPUs are busy during an interval, the percentage is N * 100%. +.PP +\fBAny%C0\fP Percentage of time that at least one CPU is busy. .PP -\fBGFXMHz\fP Instantaneous snapshot of what sysfs presents at the end of the measurement interval. From /sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz. +\fBGFX%C0\fP Percentage of time that at least one GFX compute engine is busy. +.PP +\fBCPUGFX%\fP Percentage of time that at least one CPU is busy at the same time as at least one Graphics compute enginer is busy. .PP \fBPkg%pc2, Pkg%pc3, Pkg%pc6, Pkg%pc7\fP percentage residency in hardware package idle states. These numbers are from hardware residency counters. .PP @@ -139,11 +209,17 @@ The system configuration dump (if --quiet is not used) is followed by statistics .PP \fBRAMWatt\fP Watts consumed by the DRAM DIMMS -- available only on server processors. .PP +\fBSysWatt\fP Watts consumed by the whole platform (RAPL PSYS). +.PP \fBPKG_%\fP percent of the interval that RAPL throttling was active on the Package. Note that the system summary is the sum of the package throttling time, and thus may be higher than 100% on a multi-package system. Note that the meaning of this field is model specific. For example, some hardware increments this counter when RAPL responds to thermal limits, but does not increment this counter when RAPL responds to power limits. Comparing PkgWatt and PkgTmp to system limits is necessary. .PP \fBRAM_%\fP percent of the interval that RAPL throttling was active on DRAM. .PP -\fBUncMHz\fP uncore MHz, instantaneous sample. +\fBUncMHz\fP per-package uncore MHz, instantaneous sample. +.PP +\fBUMHz1.0\fP per-package uncore MHz for pm_domain=1 and fabric_cluster=0, instantaneous sample. System summary is the average of all packages. +Intel Granite Rapids systems use pm_domains 0-2 for CPUs, and 3-4 for IO, with cluster always 0. +For the "--show" and "--hide" options, use "UncMHz" to operate on all UMHz*.* as a group. .SH TOO MUCH INFORMATION EXAMPLE By default, turbostat dumps all possible information -- a system configuration header, followed by columns for all counters. This is ideal for remote debugging, use the "--out" option to save everything to a text file, and get that file to the expert helping you debug. @@ -306,7 +382,7 @@ available on all processors. Here we limit turbostat to showing just the CPU number for cpu0 - cpu3. We add a counter showing the 32-bit raw value of MSR 0x199 (MSR_IA32_PERF_CTL), labeling it with the column header, "PRF_CTRL", and display it only once, -afte the conclusion of a 0.1 second sleep. +after the conclusion of a 0.1 second sleep. .nf sudo ./turbostat --quiet --cpu 0-3 --show CPU --add msr0x199,u32,raw,PRF_CTRL sleep .1 0.101604 sec @@ -319,6 +395,77 @@ CPU PRF_CTRL .fi +.SH ADD PERF COUNTER EXAMPLE +Here we limit turbostat to showing just the CPU number for cpu0 - cpu3. +We add a counter showing time spent in C1 core cstate, +labeling it with the column header, "pCPU%c1", and display it only once, +after the conclusion of 0.1 second sleep. +We also show CPU%c1 built-in counter that should show similar values. +.nf +sudo ./turbostat --quiet --cpu 0-3 --show CPU,CPU%c1 --add perf/cstate_core/c1-residency,cpu,delta,percent,pCPU%c1 sleep .1 +0.102448 sec +CPU pCPU%c1 CPU%c1 +- 34.89 34.89 +0 45.99 45.99 +1 45.94 45.94 +2 23.83 23.83 +3 23.84 23.84 + +.fi + +.SH ADD PERF COUNTER EXAMPLE #2 (using virtual cpu device) +Here we run on hybrid, Meteor Lake platform. +We limit turbostat to show output for just cpu0 (pcore) and cpu4 (ecore). +We add a counter showing number of L3 cache misses, using virtual "cpu" device, +labeling it with the column header, "VCMISS". +We add a counter showing number of L3 cache misses, using virtual "cpu_core" device, +labeling it with the column header, "PCMISS". This will fail on ecore cpu4. +We add a counter showing number of L3 cache misses, using virtual "cpu_atom" device, +labeling it with the column header, "ECMISS". This will fail on pcore cpu0. +We display it only once, after the conclusion of 0.1 second sleep. +.nf +sudo ./turbostat --quiet --cpu 0,4 --show CPU --add perf/cpu/cache-misses,cpu,delta,VCMISS --add perf/cpu_core/cache-misses,cpu,delta,PCMISS --add perf/cpu_atom/cache-misses,cpu,delta,ECMISS sleep 5 +turbostat: added_perf_counters_init_: perf/cpu_atom/cache-misses: failed to open counter on cpu0 +turbostat: added_perf_counters_init_: perf/cpu_core/cache-misses: failed to open counter on cpu4 +5.001207 sec +CPU ECMISS PCMISS VCMISS +- 41586506 46291219 87877749 +4 83173012 0 83173040 +0 0 92582439 92582458 +.fi + +.SH ADD PMT COUNTER EXAMPLE +Here we limit turbostat to showing just the CPU number 0. +We add two counters, showing crystal clock count and the DC6 residency. +All the parameters passed are based on the metadata found in the PMT XML files. + +For the crystal clock count, we +label it with the column header, "XTAL", +we set the type to 'raw', to read the number of clock ticks in hex, +we set the format to 'delta', to display the difference in ticks during the measurement interval, +we set the domain to 'package0', to collect it and associate it with the whole package number 0, +we set the offset to '0', which is a offset of the counter within the PMT MMIO region, +we set the lsb and msb to cover all 64 bits of the read 64 bit value, +and finally we set the guid to '0x1a067102', that identifies the PMT MMIO region to which the 'offset' is applied to read the counter value. + +For the DC6 residency counter, we +label it with the column header, "Die%c6", +we set the type to 'txtal_time', to obtain the percent residency value +we set the format to 'delta', to display the difference in ticks during the measurement interval, +we set the domain to 'package0', to collect it and associate it with the whole package number 0, +we set the offset to '0', which is a offset of the counter within the PMT MMIO region, +we set the lsb and msb to cover all 64 bits of the read 64 bit value, +and finally we set the guid to '0x1a067102', that identifies the PMT MMIO region to which the 'offset' is applied to read the counter value. + +.nf +sudo ./turbostat --quiet --cpu 0 --show CPU --add pmt,name=XTAL,type=raw,format=delta,domain=package0,offset=0,lsb=0,msb=63,guid=0x1a067102 --add pmt,name=Die%c6,type=txtal_time,format=delta,domain=package0,offset=120,lsb=0,msb=63,guid=0x1a067102 +0.104352 sec +CPU XTAL Die%c6 +- 0x0000006d4d957ca7 0.00 +0 0x0000006d4d957ca7 0.00 +0.102448 sec +.fi + .SH INPUT For interval-mode, turbostat will immediately end the current interval @@ -389,14 +536,40 @@ that they count at TSC rate, which is true on all processors tested to date. Volume 3B: System Programming Guide" https://www.intel.com/products/processor/manuals/ +.SH RUN THE LATEST VERSION +If turbostat complains that it doesn't recognize your processor, +please try the latest version. + +The latest version of turbostat does not require the latest version of the Linux kernel. +However, some features, such as perf(1) counters, do require kernel support. + +The latest turbostat release is available in the upstream Linux Kernel source tree. +eg. "git pull https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" +and run make in tools/power/x86/turbostat/. + +n.b. "make install" will update your system manually, but a distro update may subsequently downgrade your turbostat to an older version. +For this reason, manually installing to /usr/local/bin may be what you want. + +Note that turbostat/Makefile has a "make snapshot" target, which will create a tar file +that can build without a local kernel source tree. + +If the upstream version isn't new enough, the development tree can be found here: +"git pull https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git turbostat" + +If the development tree doesn't work, please contact the author via chat, +or via email with the word "turbostat" on the Subject line. + .SH FILES .ta .nf +/sys/bus/event_source/devices/ /dev/cpu/*/msr +/sys/class/intel_pmt/ +/sys/devices/system/cpu/ .fi .SH "SEE ALSO" -msr(4), vmstat(8) +perf(1), msr(4), vmstat(8) .PP .SH AUTHOR .nf diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 6fad5fbfb0f96..3af45302eee19 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -3,13 +3,41 @@ * turbostat -- show CPU frequency and C-state residency * on modern Intel and AMD processors. * - * Copyright (c) 2023 Intel Corporation. + * Copyright (c) 2025 Intel Corporation. * Len Brown */ #define _GNU_SOURCE #include MSRHEADER + +// copied from arch/x86/include/asm/cpu_device_id.h +#define VFM_MODEL_BIT 0 +#define VFM_FAMILY_BIT 8 +#define VFM_VENDOR_BIT 16 +#define VFM_RSVD_BIT 24 + +#define VFM_MODEL_MASK GENMASK(VFM_FAMILY_BIT - 1, VFM_MODEL_BIT) +#define VFM_FAMILY_MASK GENMASK(VFM_VENDOR_BIT - 1, VFM_FAMILY_BIT) +#define VFM_VENDOR_MASK GENMASK(VFM_RSVD_BIT - 1, VFM_VENDOR_BIT) + +#define VFM_MODEL(vfm) (((vfm) & VFM_MODEL_MASK) >> VFM_MODEL_BIT) +#define VFM_FAMILY(vfm) (((vfm) & VFM_FAMILY_MASK) >> VFM_FAMILY_BIT) +#define VFM_VENDOR(vfm) (((vfm) & VFM_VENDOR_MASK) >> VFM_VENDOR_BIT) + +#define VFM_MAKE(_vendor, _family, _model) ( \ + ((_model) << VFM_MODEL_BIT) | \ + ((_family) << VFM_FAMILY_BIT) | \ + ((_vendor) << VFM_VENDOR_BIT) \ +) +// end copied section + +#define CPUID_LEAF_MODEL_ID 0x1A +#define CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT 24 + +#define X86_VENDOR_INTEL 0 + #include INTEL_FAMILY_HEADER +#include BUILD_BUG_HEADER #include #include #include @@ -19,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +65,9 @@ #include #include #include +#include +#include +#include #define UNUSED(x) (void)(x) @@ -52,17 +84,55 @@ */ #define NAME_BYTES 20 #define PATH_BYTES 128 +#define PERF_NAME_BYTES 128 #define MAX_NOFILE 0x8000 +#define COUNTER_KIND_PERF_PREFIX "perf/" +#define COUNTER_KIND_PERF_PREFIX_LEN strlen(COUNTER_KIND_PERF_PREFIX) +#define PERF_DEV_NAME_BYTES 32 +#define PERF_EVT_NAME_BYTES 32 + +#define INTEL_ECORE_TYPE 0x20 +#define INTEL_PCORE_TYPE 0x40 + +#define ROUND_UP_TO_PAGE_SIZE(n) (((n) + 0x1000UL-1UL) & ~(0x1000UL-1UL)) + enum counter_scope { SCOPE_CPU, SCOPE_CORE, SCOPE_PACKAGE }; -enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC }; -enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT }; +enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC, COUNTER_K2M }; +enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT, FORMAT_AVERAGE }; +enum counter_source { COUNTER_SOURCE_NONE, COUNTER_SOURCE_PERF, COUNTER_SOURCE_MSR }; + +struct perf_counter_info { + struct perf_counter_info *next; + + /* How to open the counter / What counter it is. */ + char device[PERF_DEV_NAME_BYTES]; + char event[PERF_EVT_NAME_BYTES]; + + /* How to show/format the counter. */ + char name[PERF_NAME_BYTES]; + unsigned int width; + enum counter_scope scope; + enum counter_type type; + enum counter_format format; + double scale; + + /* For reading the counter. */ + int *fd_perf_per_domain; + size_t num_domains; +}; + +struct sysfs_path { + char path[PATH_BYTES]; + int id; + struct sysfs_path *next; +}; struct msr_counter { unsigned int msr_num; char name[NAME_BYTES]; - char path[PATH_BYTES]; + struct sysfs_path *sp; unsigned int width; enum counter_type type; enum counter_format format; @@ -72,150 +142,349 @@ struct msr_counter { #define FLAGS_SHOW (1 << 1) #define SYSFS_PERCPU (1 << 1) }; +static int use_android_msr_path; struct msr_counter bic[] = { - { 0x0, "usec", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Time_Of_Day_Seconds", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Package", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Node", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Avg_MHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Busy%", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Bzy_MHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "TSC_MHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "IRQ", "", 0, 0, 0, NULL, 0 }, - { 0x0, "SMI", "", 32, 0, FORMAT_DELTA, NULL, 0 }, - { 0x0, "sysfs", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU%c1", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU%c3", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU%c6", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU%c7", "", 0, 0, 0, NULL, 0 }, - { 0x0, "ThreadC", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CoreTmp", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CoreCnt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "PkgTmp", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFX%rc6", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFXMHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc2", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc3", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc6", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc7", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc8", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg%pc9", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pk%pc10", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU%LPI", "", 0, 0, 0, NULL, 0 }, - { 0x0, "SYS%LPI", "", 0, 0, 0, NULL, 0 }, - { 0x0, "PkgWatt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CorWatt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFXWatt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "PkgCnt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "RAMWatt", "", 0, 0, 0, NULL, 0 }, - { 0x0, "PKG_%", "", 0, 0, 0, NULL, 0 }, - { 0x0, "RAM_%", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Pkg_J", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Cor_J", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFX_J", "", 0, 0, 0, NULL, 0 }, - { 0x0, "RAM_J", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Mod%c6", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Totl%C0", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Any%C0", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFX%C0", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPUGFX%", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Core", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CPU", "", 0, 0, 0, NULL, 0 }, - { 0x0, "APIC", "", 0, 0, 0, NULL, 0 }, - { 0x0, "X2APIC", "", 0, 0, 0, NULL, 0 }, - { 0x0, "Die", "", 0, 0, 0, NULL, 0 }, - { 0x0, "GFXAMHz", "", 0, 0, 0, NULL, 0 }, - { 0x0, "IPC", "", 0, 0, 0, NULL, 0 }, - { 0x0, "CoreThr", "", 0, 0, 0, NULL, 0 }, - { 0x0, "UncMHz", "", 0, 0, 0, NULL, 0 }, + { 0x0, "usec", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Time_Of_Day_Seconds", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Package", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Node", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Avg_MHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Busy%", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Bzy_MHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "TSC_MHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "IRQ", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SMI", NULL, 32, 0, FORMAT_DELTA, NULL, 0 }, + { 0x0, "cpuidle", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%c1", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%c3", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%c6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%c7", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "ThreadC", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CoreTmp", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CoreCnt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "PkgTmp", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFX%rc6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFXMHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc2", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc3", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc7", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc8", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg%pc9", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pk%pc10", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%LPI", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SYS%LPI", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "PkgWatt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CorWatt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFXWatt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "PkgCnt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "RAMWatt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "PKG_%", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "RAM_%", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Pkg_J", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Cor_J", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFX_J", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "RAM_J", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Mod%c6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Totl%C0", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Any%C0", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFX%C0", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPUGFX%", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Core", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "APIC", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "X2APIC", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Die", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "L3", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "GFXAMHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "IPC", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CoreThr", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "UncMHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SAM%mc6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SAMMHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SAMAMHz", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Die%c6", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "SysWatt", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Sys_J", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "NMI", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "CPU%c1e", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "pct_idle", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "LLCkRPS", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "LLC%hit", NULL, 0, 0, 0, NULL, 0 }, +}; + +/* n.b. bic_names must match the order in bic[], above */ +enum bic_names { + BIC_USEC, + BIC_TOD, + BIC_Package, + BIC_Node, + BIC_Avg_MHz, + BIC_Busy, + BIC_Bzy_MHz, + BIC_TSC_MHz, + BIC_IRQ, + BIC_SMI, + BIC_cpuidle, + BIC_CPU_c1, + BIC_CPU_c3, + BIC_CPU_c6, + BIC_CPU_c7, + BIC_ThreadC, + BIC_CoreTmp, + BIC_CoreCnt, + BIC_PkgTmp, + BIC_GFX_rc6, + BIC_GFXMHz, + BIC_Pkgpc2, + BIC_Pkgpc3, + BIC_Pkgpc6, + BIC_Pkgpc7, + BIC_Pkgpc8, + BIC_Pkgpc9, + BIC_Pkgpc10, + BIC_CPU_LPI, + BIC_SYS_LPI, + BIC_PkgWatt, + BIC_CorWatt, + BIC_GFXWatt, + BIC_PkgCnt, + BIC_RAMWatt, + BIC_PKG__, + BIC_RAM__, + BIC_Pkg_J, + BIC_Cor_J, + BIC_GFX_J, + BIC_RAM_J, + BIC_Mod_c6, + BIC_Totl_c0, + BIC_Any_c0, + BIC_GFX_c0, + BIC_CPUGFX, + BIC_Core, + BIC_CPU, + BIC_APIC, + BIC_X2APIC, + BIC_Die, + BIC_L3, + BIC_GFXACTMHz, + BIC_IPC, + BIC_CORE_THROT_CNT, + BIC_UNCORE_MHZ, + BIC_SAM_mc6, + BIC_SAMMHz, + BIC_SAMACTMHz, + BIC_Diec6, + BIC_SysWatt, + BIC_Sys_J, + BIC_NMI, + BIC_CPU_c1e, + BIC_pct_idle, + BIC_LLC_RPS, + BIC_LLC_HIT, + MAX_BIC }; -#define MAX_BIC (sizeof(bic) / sizeof(struct msr_counter)) -#define BIC_USEC (1ULL << 0) -#define BIC_TOD (1ULL << 1) -#define BIC_Package (1ULL << 2) -#define BIC_Node (1ULL << 3) -#define BIC_Avg_MHz (1ULL << 4) -#define BIC_Busy (1ULL << 5) -#define BIC_Bzy_MHz (1ULL << 6) -#define BIC_TSC_MHz (1ULL << 7) -#define BIC_IRQ (1ULL << 8) -#define BIC_SMI (1ULL << 9) -#define BIC_sysfs (1ULL << 10) -#define BIC_CPU_c1 (1ULL << 11) -#define BIC_CPU_c3 (1ULL << 12) -#define BIC_CPU_c6 (1ULL << 13) -#define BIC_CPU_c7 (1ULL << 14) -#define BIC_ThreadC (1ULL << 15) -#define BIC_CoreTmp (1ULL << 16) -#define BIC_CoreCnt (1ULL << 17) -#define BIC_PkgTmp (1ULL << 18) -#define BIC_GFX_rc6 (1ULL << 19) -#define BIC_GFXMHz (1ULL << 20) -#define BIC_Pkgpc2 (1ULL << 21) -#define BIC_Pkgpc3 (1ULL << 22) -#define BIC_Pkgpc6 (1ULL << 23) -#define BIC_Pkgpc7 (1ULL << 24) -#define BIC_Pkgpc8 (1ULL << 25) -#define BIC_Pkgpc9 (1ULL << 26) -#define BIC_Pkgpc10 (1ULL << 27) -#define BIC_CPU_LPI (1ULL << 28) -#define BIC_SYS_LPI (1ULL << 29) -#define BIC_PkgWatt (1ULL << 30) -#define BIC_CorWatt (1ULL << 31) -#define BIC_GFXWatt (1ULL << 32) -#define BIC_PkgCnt (1ULL << 33) -#define BIC_RAMWatt (1ULL << 34) -#define BIC_PKG__ (1ULL << 35) -#define BIC_RAM__ (1ULL << 36) -#define BIC_Pkg_J (1ULL << 37) -#define BIC_Cor_J (1ULL << 38) -#define BIC_GFX_J (1ULL << 39) -#define BIC_RAM_J (1ULL << 40) -#define BIC_Mod_c6 (1ULL << 41) -#define BIC_Totl_c0 (1ULL << 42) -#define BIC_Any_c0 (1ULL << 43) -#define BIC_GFX_c0 (1ULL << 44) -#define BIC_CPUGFX (1ULL << 45) -#define BIC_Core (1ULL << 46) -#define BIC_CPU (1ULL << 47) -#define BIC_APIC (1ULL << 48) -#define BIC_X2APIC (1ULL << 49) -#define BIC_Die (1ULL << 50) -#define BIC_GFXACTMHz (1ULL << 51) -#define BIC_IPC (1ULL << 52) -#define BIC_CORE_THROT_CNT (1ULL << 53) -#define BIC_UNCORE_MHZ (1ULL << 54) - -#define BIC_TOPOLOGY (BIC_Package | BIC_Node | BIC_CoreCnt | BIC_PkgCnt | BIC_Core | BIC_CPU | BIC_Die ) -#define BIC_THERMAL_PWR ( BIC_CoreTmp | BIC_PkgTmp | BIC_PkgWatt | BIC_CorWatt | BIC_GFXWatt | BIC_RAMWatt | BIC_PKG__ | BIC_RAM__) -#define BIC_FREQUENCY ( BIC_Avg_MHz | BIC_Busy | BIC_Bzy_MHz | BIC_TSC_MHz | BIC_GFXMHz | BIC_GFXACTMHz | BIC_UNCORE_MHZ) -#define BIC_IDLE ( BIC_sysfs | BIC_CPU_c1 | BIC_CPU_c3 | BIC_CPU_c6 | BIC_CPU_c7 | BIC_GFX_rc6 | BIC_Pkgpc2 | BIC_Pkgpc3 | BIC_Pkgpc6 | BIC_Pkgpc7 | BIC_Pkgpc8 | BIC_Pkgpc9 | BIC_Pkgpc10 | BIC_CPU_LPI | BIC_SYS_LPI | BIC_Mod_c6 | BIC_Totl_c0 | BIC_Any_c0 | BIC_GFX_c0 | BIC_CPUGFX) -#define BIC_OTHER ( BIC_IRQ | BIC_SMI | BIC_ThreadC | BIC_CoreTmp | BIC_IPC) - -#define BIC_DISABLED_BY_DEFAULT (BIC_USEC | BIC_TOD | BIC_APIC | BIC_X2APIC) - -unsigned long long bic_enabled = (0xFFFFFFFFFFFFFFFFULL & ~BIC_DISABLED_BY_DEFAULT); -unsigned long long bic_present = BIC_USEC | BIC_TOD | BIC_sysfs | BIC_APIC | BIC_X2APIC; - -#define DO_BIC(COUNTER_NAME) (bic_enabled & bic_present & COUNTER_NAME) -#define DO_BIC_READ(COUNTER_NAME) (bic_present & COUNTER_NAME) -#define ENABLE_BIC(COUNTER_NAME) (bic_enabled |= COUNTER_NAME) -#define BIC_PRESENT(COUNTER_BIT) (bic_present |= COUNTER_BIT) -#define BIC_NOT_PRESENT(COUNTER_BIT) (bic_present &= ~COUNTER_BIT) -#define BIC_IS_ENABLED(COUNTER_BIT) (bic_enabled & COUNTER_BIT) +void print_bic_set(char *s, cpu_set_t *set) +{ + int i; + + assert(MAX_BIC < CPU_SETSIZE); + + printf("%s:", s); + + for (i = 0; i <= MAX_BIC; ++i) { + + if (CPU_ISSET(i, set)) { + assert(i < MAX_BIC); + printf(" %s", bic[i].name); + } + } + putchar('\n'); +} + +static cpu_set_t bic_group_topology; +static cpu_set_t bic_group_thermal_pwr; +static cpu_set_t bic_group_frequency; +static cpu_set_t bic_group_hw_idle; +static cpu_set_t bic_group_sw_idle; +static cpu_set_t bic_group_idle; +static cpu_set_t bic_group_cache; +static cpu_set_t bic_group_other; +static cpu_set_t bic_group_disabled_by_default; +static cpu_set_t bic_enabled; +static cpu_set_t bic_present; + +/* modify */ +#define BIC_INIT(set) CPU_ZERO(set) + +#define SET_BIC(COUNTER_NUMBER, set) CPU_SET(COUNTER_NUMBER, set) +#define CLR_BIC(COUNTER_NUMBER, set) CPU_CLR(COUNTER_NUMBER, set) + +#define BIC_PRESENT(COUNTER_NUMBER) SET_BIC(COUNTER_NUMBER, &bic_present) +#define BIC_NOT_PRESENT(COUNTER_NUMBER) CPU_CLR(COUNTER_NUMBER, &bic_present) + +/* test */ +#define BIC_IS_ENABLED(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_enabled) +#define DO_BIC_READ(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_present) +#define DO_BIC(COUNTER_NUMBER) (CPU_ISSET(COUNTER_NUMBER, &bic_enabled) && CPU_ISSET(COUNTER_NUMBER, &bic_present)) + +static void bic_set_all(cpu_set_t *set) +{ + int i; + + assert(MAX_BIC < CPU_SETSIZE); + + for (i = 0; i < MAX_BIC; ++i) + SET_BIC(i, set); +} + +/* + * bic_clear_bits() + * clear all the bits from "clr" in "dst" + */ +static void bic_clear_bits(cpu_set_t *dst, cpu_set_t *clr) +{ + int i; + + assert(MAX_BIC < CPU_SETSIZE); + + for (i = 0; i < MAX_BIC; ++i) + if (CPU_ISSET(i, clr)) + CLR_BIC(i, dst); +} + +static void bic_groups_init(void) +{ + BIC_INIT(&bic_group_topology); + SET_BIC(BIC_Package, &bic_group_topology); + SET_BIC(BIC_Node, &bic_group_topology); + SET_BIC(BIC_CoreCnt, &bic_group_topology); + SET_BIC(BIC_PkgCnt, &bic_group_topology); + SET_BIC(BIC_Core, &bic_group_topology); + SET_BIC(BIC_CPU, &bic_group_topology); + SET_BIC(BIC_Die, &bic_group_topology); + SET_BIC(BIC_L3, &bic_group_topology); + + BIC_INIT(&bic_group_thermal_pwr); + SET_BIC(BIC_CoreTmp, &bic_group_thermal_pwr); + SET_BIC(BIC_PkgTmp, &bic_group_thermal_pwr); + SET_BIC(BIC_PkgWatt, &bic_group_thermal_pwr); + SET_BIC(BIC_CorWatt, &bic_group_thermal_pwr); + SET_BIC(BIC_GFXWatt, &bic_group_thermal_pwr); + SET_BIC(BIC_RAMWatt, &bic_group_thermal_pwr); + SET_BIC(BIC_PKG__, &bic_group_thermal_pwr); + SET_BIC(BIC_RAM__, &bic_group_thermal_pwr); + SET_BIC(BIC_SysWatt, &bic_group_thermal_pwr); + + BIC_INIT(&bic_group_frequency); + SET_BIC(BIC_Avg_MHz, &bic_group_frequency); + SET_BIC(BIC_Busy, &bic_group_frequency); + SET_BIC(BIC_Bzy_MHz, &bic_group_frequency); + SET_BIC(BIC_TSC_MHz, &bic_group_frequency); + SET_BIC(BIC_GFXMHz, &bic_group_frequency); + SET_BIC(BIC_GFXACTMHz, &bic_group_frequency); + SET_BIC(BIC_SAMMHz, &bic_group_frequency); + SET_BIC(BIC_SAMACTMHz, &bic_group_frequency); + SET_BIC(BIC_UNCORE_MHZ, &bic_group_frequency); + + BIC_INIT(&bic_group_hw_idle); + SET_BIC(BIC_Busy, &bic_group_hw_idle); + SET_BIC(BIC_CPU_c1, &bic_group_hw_idle); + SET_BIC(BIC_CPU_c3, &bic_group_hw_idle); + SET_BIC(BIC_CPU_c6, &bic_group_hw_idle); + SET_BIC(BIC_CPU_c7, &bic_group_hw_idle); + SET_BIC(BIC_GFX_rc6, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc2, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc3, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc6, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc7, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc8, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc9, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc10, &bic_group_hw_idle); + SET_BIC(BIC_CPU_LPI, &bic_group_hw_idle); + SET_BIC(BIC_SYS_LPI, &bic_group_hw_idle); + SET_BIC(BIC_Mod_c6, &bic_group_hw_idle); + SET_BIC(BIC_Totl_c0, &bic_group_hw_idle); + SET_BIC(BIC_Any_c0, &bic_group_hw_idle); + SET_BIC(BIC_GFX_c0, &bic_group_hw_idle); + SET_BIC(BIC_CPUGFX, &bic_group_hw_idle); + SET_BIC(BIC_SAM_mc6, &bic_group_hw_idle); + SET_BIC(BIC_Diec6, &bic_group_hw_idle); + + BIC_INIT(&bic_group_sw_idle); + SET_BIC(BIC_Busy, &bic_group_sw_idle); + SET_BIC(BIC_cpuidle, &bic_group_sw_idle); + SET_BIC(BIC_pct_idle, &bic_group_sw_idle); + + BIC_INIT(&bic_group_idle); + + CPU_OR(&bic_group_idle, &bic_group_idle, &bic_group_hw_idle); + SET_BIC(BIC_pct_idle, &bic_group_idle); + + BIC_INIT(&bic_group_cache); + SET_BIC(BIC_LLC_RPS, &bic_group_cache); + SET_BIC(BIC_LLC_HIT, &bic_group_cache); + + BIC_INIT(&bic_group_other); + SET_BIC(BIC_IRQ, &bic_group_other); + SET_BIC(BIC_NMI, &bic_group_other); + SET_BIC(BIC_SMI, &bic_group_other); + SET_BIC(BIC_ThreadC, &bic_group_other); + SET_BIC(BIC_CoreTmp, &bic_group_other); + SET_BIC(BIC_IPC, &bic_group_other); + + BIC_INIT(&bic_group_disabled_by_default); + SET_BIC(BIC_USEC, &bic_group_disabled_by_default); + SET_BIC(BIC_TOD, &bic_group_disabled_by_default); + SET_BIC(BIC_cpuidle, &bic_group_disabled_by_default); + SET_BIC(BIC_APIC, &bic_group_disabled_by_default); + SET_BIC(BIC_X2APIC, &bic_group_disabled_by_default); + + BIC_INIT(&bic_enabled); + bic_set_all(&bic_enabled); + bic_clear_bits(&bic_enabled, &bic_group_disabled_by_default); + + BIC_INIT(&bic_present); + SET_BIC(BIC_USEC, &bic_present); + SET_BIC(BIC_TOD, &bic_present); + SET_BIC(BIC_cpuidle, &bic_present); + SET_BIC(BIC_APIC, &bic_present); + SET_BIC(BIC_X2APIC, &bic_present); + SET_BIC(BIC_pct_idle, &bic_present); +} + +/* + * MSR_PKG_CST_CONFIG_CONTROL decoding for pkg_cstate_limit: + * If you change the values, note they are used both in comparisons + * (>= PCL__7) and to index pkg_cstate_limit_strings[]. + */ +#define PCLUKN 0 /* Unknown */ +#define PCLRSV 1 /* Reserved */ +#define PCL__0 2 /* PC0 */ +#define PCL__1 3 /* PC1 */ +#define PCL__2 4 /* PC2 */ +#define PCL__3 5 /* PC3 */ +#define PCL__4 6 /* PC4 */ +#define PCL__6 7 /* PC6 */ +#define PCL_6N 8 /* PC6 No Retention */ +#define PCL_6R 9 /* PC6 Retention */ +#define PCL__7 10 /* PC7 */ +#define PCL_7S 11 /* PC7 Shrink */ +#define PCL__8 12 /* PC8 */ +#define PCL__9 13 /* PC9 */ +#define PCL_10 14 /* PC10 */ +#define PCLUNL 15 /* Unlimited */ char *proc_stat = "/proc/stat"; FILE *outf; int *fd_percpu; int *fd_instr_count_percpu; +int *fd_llc_percpu; struct timeval interval_tv = { 5, 0 }; struct timespec interval_ts = { 5, 0 }; -/* Save original CPU model */ -unsigned int model_orig; - unsigned int num_iterations; unsigned int header_iterations; unsigned int debug; @@ -223,27 +492,22 @@ unsigned int quiet; unsigned int shown; unsigned int sums_need_wide_columns; unsigned int rapl_joules; +unsigned int valid_rapl_msrs; unsigned int summary_only; unsigned int list_header_only; unsigned int dump_only; -unsigned int do_snb_cstates; -unsigned int do_knl_cstates; -unsigned int do_slm_cstates; -unsigned int use_c1_residency_msr; -unsigned int has_aperf; +unsigned int force_load; +unsigned int cpuid_has_aperf_mperf; +unsigned int has_aperf_access; unsigned int has_epb; unsigned int has_turbo; unsigned int is_hybrid; -unsigned int do_irtl_snb; -unsigned int do_irtl_hsw; unsigned int units = 1000000; /* MHz etc */ unsigned int genuine_intel; unsigned int authentic_amd; unsigned int hygon_genuine; unsigned int max_level, max_extended_level; unsigned int has_invariant_tsc; -unsigned int do_nhm_platform_info; -unsigned int no_MSR_MISC_PWR_MGMT; unsigned int aperf_mperf_multiplier = 1; double bclk; double base_hz; @@ -252,5913 +516,9569 @@ double tsc_tweak = 1.0; unsigned int show_pkg_only; unsigned int show_core_only; char *output_buffer, *outp; -unsigned int do_rapl; unsigned int do_dts; unsigned int do_ptm; unsigned int do_ipc; -unsigned long long gfx_cur_rc6_ms; unsigned long long cpuidle_cur_cpu_lpi_us; unsigned long long cpuidle_cur_sys_lpi_us; -unsigned int gfx_cur_mhz; -unsigned int gfx_act_mhz; unsigned int tj_max; unsigned int tj_max_override; -int tcc_offset_bits; double rapl_power_units, rapl_time_units; -double rapl_dram_energy_units, rapl_energy_units; +double rapl_dram_energy_units, rapl_energy_units, rapl_psys_energy_units; double rapl_joule_counter_range; -unsigned int do_core_perf_limit_reasons; -unsigned int has_automatic_cstate_conversion; -unsigned int dis_cstate_prewake; -unsigned int do_gfx_perf_limit_reasons; -unsigned int do_ring_perf_limit_reasons; unsigned int crystal_hz; unsigned long long tsc_hz; int base_cpu; -double discover_bclk(unsigned int family, unsigned int model); unsigned int has_hwp; /* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */ /* IA32_HWP_REQUEST, IA32_HWP_STATUS */ unsigned int has_hwp_notify; /* IA32_HWP_INTERRUPT */ unsigned int has_hwp_activity_window; /* IA32_HWP_REQUEST[bits 41:32] */ unsigned int has_hwp_epp; /* IA32_HWP_REQUEST[bits 31:24] */ unsigned int has_hwp_pkg; /* IA32_HWP_REQUEST_PKG */ -unsigned int has_misc_feature_control; unsigned int first_counter_read = 1; + +static struct timeval procsysfs_tv_begin; + int ignore_stdin; +bool no_msr; +bool no_perf; + +enum gfx_sysfs_idx { + GFX_rc6, + GFX_MHz, + GFX_ACTMHz, + SAM_mc6, + SAM_MHz, + SAM_ACTMHz, + GFX_MAX +}; -#define RAPL_PKG (1 << 0) - /* 0x610 MSR_PKG_POWER_LIMIT */ - /* 0x611 MSR_PKG_ENERGY_STATUS */ -#define RAPL_PKG_PERF_STATUS (1 << 1) - /* 0x613 MSR_PKG_PERF_STATUS */ -#define RAPL_PKG_POWER_INFO (1 << 2) - /* 0x614 MSR_PKG_POWER_INFO */ - -#define RAPL_DRAM (1 << 3) - /* 0x618 MSR_DRAM_POWER_LIMIT */ - /* 0x619 MSR_DRAM_ENERGY_STATUS */ -#define RAPL_DRAM_PERF_STATUS (1 << 4) - /* 0x61b MSR_DRAM_PERF_STATUS */ -#define RAPL_DRAM_POWER_INFO (1 << 5) - /* 0x61c MSR_DRAM_POWER_INFO */ - -#define RAPL_CORES_POWER_LIMIT (1 << 6) - /* 0x638 MSR_PP0_POWER_LIMIT */ -#define RAPL_CORE_POLICY (1 << 7) - /* 0x63a MSR_PP0_POLICY */ - -#define RAPL_GFX (1 << 8) - /* 0x640 MSR_PP1_POWER_LIMIT */ - /* 0x641 MSR_PP1_ENERGY_STATUS */ - /* 0x642 MSR_PP1_POLICY */ - -#define RAPL_CORES_ENERGY_STATUS (1 << 9) - /* 0x639 MSR_PP0_ENERGY_STATUS */ -#define RAPL_PER_CORE_ENERGY (1 << 10) - /* Indicates cores energy collection is per-core, - * not per-package. */ -#define RAPL_AMD_F17H (1 << 11) - /* 0xc0010299 MSR_RAPL_PWR_UNIT */ - /* 0xc001029a MSR_CORE_ENERGY_STAT */ - /* 0xc001029b MSR_PKG_ENERGY_STAT */ -#define RAPL_CORES (RAPL_CORES_ENERGY_STATUS | RAPL_CORES_POWER_LIMIT) -#define TJMAX_DEFAULT 100 +struct gfx_sysfs_info { + FILE *fp; + unsigned int val; + unsigned long long val_ull; +}; -/* MSRs that are not yet in the kernel-provided header. */ -#define MSR_RAPL_PWR_UNIT 0xc0010299 -#define MSR_CORE_ENERGY_STAT 0xc001029a -#define MSR_PKG_ENERGY_STAT 0xc001029b +static struct gfx_sysfs_info gfx_info[GFX_MAX]; -#define MAX(a, b) ((a) > (b) ? (a) : (b)) +int get_msr(int cpu, off_t offset, unsigned long long *msr); +int add_counter(unsigned int msr_num, char *path, char *name, + unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int package_num); + +/* Model specific support Start */ + +/* List of features that may diverge among different platforms */ +struct platform_features { + bool has_msr_misc_feature_control; /* MSR_MISC_FEATURE_CONTROL */ + bool has_msr_misc_pwr_mgmt; /* MSR_MISC_PWR_MGMT */ + bool has_nhm_msrs; /* MSR_PLATFORM_INFO, MSR_IA32_TEMPERATURE_TARGET, MSR_SMI_COUNT, MSR_PKG_CST_CONFIG_CONTROL, MSR_IA32_POWER_CTL, TRL MSRs */ + bool has_config_tdp; /* MSR_CONFIG_TDP_NOMINAL/LEVEL_1/LEVEL_2/CONTROL, MSR_TURBO_ACTIVATION_RATIO */ + int bclk_freq; /* CPU base clock */ + int crystal_freq; /* Crystal clock to use when not available from CPUID.15 */ + int supported_cstates; /* Core cstates and Package cstates supported */ + int cst_limit; /* MSR_PKG_CST_CONFIG_CONTROL */ + bool has_cst_auto_convension; /* AUTOMATIC_CSTATE_CONVERSION bit in MSR_PKG_CST_CONFIG_CONTROL */ + bool has_irtl_msrs; /* MSR_PKGC3/PKGC6/PKGC7/PKGC8/PKGC9/PKGC10_IRTL */ + bool has_msr_core_c1_res; /* MSR_CORE_C1_RES */ + bool has_msr_module_c6_res_ms; /* MSR_MODULE_C6_RES_MS */ + bool has_msr_c6_demotion_policy_config; /* MSR_CC6_DEMOTION_POLICY_CONFIG/MSR_MC6_DEMOTION_POLICY_CONFIG */ + bool has_msr_atom_pkg_c6_residency; /* MSR_ATOM_PKG_C6_RESIDENCY */ + bool has_msr_knl_core_c6_residency; /* MSR_KNL_CORE_C6_RESIDENCY */ + bool has_ext_cst_msrs; /* MSR_PKG_WEIGHTED_CORE_C0_RES/MSR_PKG_ANY_CORE_C0_RES/MSR_PKG_ANY_GFXE_C0_RES/MSR_PKG_BOTH_CORE_GFXE_C0_RES */ + bool has_cst_prewake_bit; /* Cstate prewake bit in MSR_IA32_POWER_CTL */ + int trl_msrs; /* MSR_TURBO_RATIO_LIMIT/LIMIT1/LIMIT2/SECONDARY, Atom TRL MSRs */ + int plr_msrs; /* MSR_CORE/GFX/RING_PERF_LIMIT_REASONS */ + int plat_rapl_msrs; /* RAPL PKG/DRAM/CORE/GFX MSRs, AMD RAPL MSRs */ + bool has_per_core_rapl; /* Indicates cores energy collection is per-core, not per-package. AMD specific for now */ + bool has_rapl_divisor; /* Divisor for Energy unit raw value from MSR_RAPL_POWER_UNIT */ + bool has_fixed_rapl_unit; /* Fixed Energy Unit used for DRAM RAPL Domain */ + bool has_fixed_rapl_psys_unit; /* Fixed Energy Unit used for PSYS RAPL Domain */ + int rapl_quirk_tdp; /* Hardcoded TDP value when cannot be retrieved from hardware */ + int tcc_offset_bits; /* TCC Offset bits in MSR_IA32_TEMPERATURE_TARGET */ + bool enable_tsc_tweak; /* Use CPU Base freq instead of TSC freq for aperf/mperf counter */ + bool need_perf_multiplier; /* mperf/aperf multiplier */ +}; -int backwards_count; -char *progname; +struct platform_data { + unsigned int vfm; + const struct platform_features *features; +}; -#define CPU_SUBSET_MAXCPUS 1024 /* need to use before probe... */ -cpu_set_t *cpu_present_set, *cpu_affinity_set, *cpu_subset; -size_t cpu_present_setsize, cpu_affinity_setsize, cpu_subset_size; -#define MAX_ADDED_COUNTERS 8 -#define MAX_ADDED_THREAD_COUNTERS 24 -#define BITMASK_SIZE 32 +/* For BCLK */ +enum bclk_freq { + BCLK_100MHZ = 1, + BCLK_133MHZ, + BCLK_SLV, +}; -struct thread_data { - struct timeval tv_begin; - struct timeval tv_end; - struct timeval tv_delta; - unsigned long long tsc; - unsigned long long aperf; - unsigned long long mperf; - unsigned long long c1; - unsigned long long instr_count; - unsigned long long irq_count; - unsigned int smi_count; - unsigned int cpu_id; - unsigned int apic_id; - unsigned int x2apic_id; - unsigned int flags; - bool is_atom; -#define CPU_IS_FIRST_THREAD_IN_CORE 0x2 -#define CPU_IS_FIRST_CORE_IN_PACKAGE 0x4 - unsigned long long counter[MAX_ADDED_THREAD_COUNTERS]; -} *thread_even, *thread_odd; +#define SLM_BCLK_FREQS 5 +double slm_freq_table[SLM_BCLK_FREQS] = { 83.3, 100.0, 133.3, 116.7, 80.0 }; -struct core_data { - unsigned long long c3; - unsigned long long c6; - unsigned long long c7; - unsigned long long mc6_us; /* duplicate as per-core for now, even though per module */ - unsigned int core_temp_c; - unsigned int core_energy; /* MSR_CORE_ENERGY_STAT */ - unsigned int core_id; - unsigned long long core_throt_cnt; - unsigned long long counter[MAX_ADDED_COUNTERS]; -} *core_even, *core_odd; +double slm_bclk(void) +{ + unsigned long long msr = 3; + unsigned int i; + double freq; -struct pkg_data { - unsigned long long pc2; - unsigned long long pc3; - unsigned long long pc6; - unsigned long long pc7; - unsigned long long pc8; - unsigned long long pc9; - unsigned long long pc10; - unsigned long long cpu_lpi; - unsigned long long sys_lpi; - unsigned long long pkg_wtd_core_c0; - unsigned long long pkg_any_core_c0; - unsigned long long pkg_any_gfxe_c0; - unsigned long long pkg_both_core_gfxe_c0; - long long gfx_rc6_ms; - unsigned int gfx_mhz; - unsigned int gfx_act_mhz; - unsigned int package_id; - unsigned long long energy_pkg; /* MSR_PKG_ENERGY_STATUS */ - unsigned long long energy_dram; /* MSR_DRAM_ENERGY_STATUS */ - unsigned long long energy_cores; /* MSR_PP0_ENERGY_STATUS */ - unsigned long long energy_gfx; /* MSR_PP1_ENERGY_STATUS */ - unsigned long long rapl_pkg_perf_status; /* MSR_PKG_PERF_STATUS */ - unsigned long long rapl_dram_perf_status; /* MSR_DRAM_PERF_STATUS */ - unsigned int pkg_temp_c; - unsigned int uncore_mhz; - unsigned long long counter[MAX_ADDED_COUNTERS]; -} *package_even, *package_odd; + if (get_msr(base_cpu, MSR_FSB_FREQ, &msr)) + fprintf(outf, "SLM BCLK: unknown\n"); -#define ODD_COUNTERS thread_odd, core_odd, package_odd -#define EVEN_COUNTERS thread_even, core_even, package_even + i = msr & 0xf; + if (i >= SLM_BCLK_FREQS) { + fprintf(outf, "SLM BCLK[%d] invalid\n", i); + i = 3; + } + freq = slm_freq_table[i]; -#define GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no) \ - ((thread_base) + \ - ((pkg_no) * \ - topo.nodes_per_pkg * topo.cores_per_node * topo.threads_per_core) + \ - ((node_no) * topo.cores_per_node * topo.threads_per_core) + \ - ((core_no) * topo.threads_per_core) + \ - (thread_no)) + if (!quiet) + fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq); -#define GET_CORE(core_base, core_no, node_no, pkg_no) \ - ((core_base) + \ - ((pkg_no) * topo.nodes_per_pkg * topo.cores_per_node) + \ - ((node_no) * topo.cores_per_node) + \ - (core_no)) + return freq; +} -#define GET_PKG(pkg_base, pkg_no) (pkg_base + pkg_no) +/* For Package cstate limit */ +enum package_cstate_limit { + CST_LIMIT_NHM = 1, + CST_LIMIT_SNB, + CST_LIMIT_HSW, + CST_LIMIT_SKX, + CST_LIMIT_ICX, + CST_LIMIT_SLV, + CST_LIMIT_AMT, + CST_LIMIT_KNL, + CST_LIMIT_GMT, +}; -/* - * The accumulated sum of MSR is defined as a monotonic - * increasing MSR, it will be accumulated periodically, - * despite its register's bit width. - */ -enum { - IDX_PKG_ENERGY, - IDX_DRAM_ENERGY, - IDX_PP0_ENERGY, - IDX_PP1_ENERGY, - IDX_PKG_PERF, - IDX_DRAM_PERF, - IDX_COUNT, +/* For Turbo Ratio Limit MSRs */ +enum turbo_ratio_limit_msrs { + TRL_BASE = BIT(0), + TRL_LIMIT1 = BIT(1), + TRL_LIMIT2 = BIT(2), + TRL_ATOM = BIT(3), + TRL_KNL = BIT(4), + TRL_CORECOUNT = BIT(5), }; -int get_msr_sum(int cpu, off_t offset, unsigned long long *msr); +/* For Perf Limit Reason MSRs */ +enum perf_limit_reason_msrs { + PLR_CORE = BIT(0), + PLR_GFX = BIT(1), + PLR_RING = BIT(2), +}; -struct msr_sum_array { - /* get_msr_sum() = sum + (get_msr() - last) */ - struct { - /*The accumulated MSR value is updated by the timer */ - unsigned long long sum; - /*The MSR footprint recorded in last timer */ - unsigned long long last; - } entries[IDX_COUNT]; +/* For RAPL MSRs */ +enum rapl_msrs { + RAPL_PKG_POWER_LIMIT = BIT(0), /* 0x610 MSR_PKG_POWER_LIMIT */ + RAPL_PKG_ENERGY_STATUS = BIT(1), /* 0x611 MSR_PKG_ENERGY_STATUS */ + RAPL_PKG_PERF_STATUS = BIT(2), /* 0x613 MSR_PKG_PERF_STATUS */ + RAPL_PKG_POWER_INFO = BIT(3), /* 0x614 MSR_PKG_POWER_INFO */ + RAPL_DRAM_POWER_LIMIT = BIT(4), /* 0x618 MSR_DRAM_POWER_LIMIT */ + RAPL_DRAM_ENERGY_STATUS = BIT(5), /* 0x619 MSR_DRAM_ENERGY_STATUS */ + RAPL_DRAM_PERF_STATUS = BIT(6), /* 0x61b MSR_DRAM_PERF_STATUS */ + RAPL_DRAM_POWER_INFO = BIT(7), /* 0x61c MSR_DRAM_POWER_INFO */ + RAPL_CORE_POWER_LIMIT = BIT(8), /* 0x638 MSR_PP0_POWER_LIMIT */ + RAPL_CORE_ENERGY_STATUS = BIT(9), /* 0x639 MSR_PP0_ENERGY_STATUS */ + RAPL_CORE_POLICY = BIT(10), /* 0x63a MSR_PP0_POLICY */ + RAPL_GFX_POWER_LIMIT = BIT(11), /* 0x640 MSR_PP1_POWER_LIMIT */ + RAPL_GFX_ENERGY_STATUS = BIT(12), /* 0x641 MSR_PP1_ENERGY_STATUS */ + RAPL_GFX_POLICY = BIT(13), /* 0x642 MSR_PP1_POLICY */ + RAPL_AMD_PWR_UNIT = BIT(14), /* 0xc0010299 MSR_AMD_RAPL_POWER_UNIT */ + RAPL_AMD_CORE_ENERGY_STAT = BIT(15), /* 0xc001029a MSR_AMD_CORE_ENERGY_STATUS */ + RAPL_AMD_PKG_ENERGY_STAT = BIT(16), /* 0xc001029b MSR_AMD_PKG_ENERGY_STATUS */ + RAPL_PLATFORM_ENERGY_LIMIT = BIT(17), /* 0x64c MSR_PLATFORM_ENERGY_LIMIT */ + RAPL_PLATFORM_ENERGY_STATUS = BIT(18), /* 0x64d MSR_PLATFORM_ENERGY_STATUS */ }; -/* The percpu MSR sum array.*/ -struct msr_sum_array *per_cpu_msr_sum; +#define RAPL_PKG (RAPL_PKG_ENERGY_STATUS | RAPL_PKG_POWER_LIMIT) +#define RAPL_DRAM (RAPL_DRAM_ENERGY_STATUS | RAPL_DRAM_POWER_LIMIT) +#define RAPL_CORE (RAPL_CORE_ENERGY_STATUS | RAPL_CORE_POWER_LIMIT) +#define RAPL_GFX (RAPL_GFX_POWER_LIMIT | RAPL_GFX_ENERGY_STATUS) +#define RAPL_PSYS (RAPL_PLATFORM_ENERGY_STATUS | RAPL_PLATFORM_ENERGY_LIMIT) + +#define RAPL_PKG_ALL (RAPL_PKG | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO) +#define RAPL_DRAM_ALL (RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_DRAM_POWER_INFO) +#define RAPL_CORE_ALL (RAPL_CORE | RAPL_CORE_POLICY) +#define RAPL_GFX_ALL (RAPL_GFX | RAPL_GFX_POLICY) + +#define RAPL_AMD_F17H (RAPL_AMD_PWR_UNIT | RAPL_AMD_CORE_ENERGY_STAT | RAPL_AMD_PKG_ENERGY_STAT) + +/* For Cstates */ +enum cstates { + CC1 = BIT(0), + CC3 = BIT(1), + CC6 = BIT(2), + CC7 = BIT(3), + PC2 = BIT(4), + PC3 = BIT(5), + PC6 = BIT(6), + PC7 = BIT(7), + PC8 = BIT(8), + PC9 = BIT(9), + PC10 = BIT(10), +}; -off_t idx_to_offset(int idx) -{ - off_t offset; +static const struct platform_features nhm_features = { + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_133MHZ, + .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6, + .cst_limit = CST_LIMIT_NHM, + .trl_msrs = TRL_BASE, +}; - switch (idx) { - case IDX_PKG_ENERGY: - if (do_rapl & RAPL_AMD_F17H) - offset = MSR_PKG_ENERGY_STAT; - else - offset = MSR_PKG_ENERGY_STATUS; - break; - case IDX_DRAM_ENERGY: - offset = MSR_DRAM_ENERGY_STATUS; - break; - case IDX_PP0_ENERGY: - offset = MSR_PP0_ENERGY_STATUS; - break; - case IDX_PP1_ENERGY: - offset = MSR_PP1_ENERGY_STATUS; - break; - case IDX_PKG_PERF: - offset = MSR_PKG_PERF_STATUS; - break; - case IDX_DRAM_PERF: - offset = MSR_DRAM_PERF_STATUS; - break; - default: - offset = -1; - } - return offset; -} +static const struct platform_features nhx_features = { + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_133MHZ, + .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6, + .cst_limit = CST_LIMIT_NHM, +}; -int offset_to_idx(off_t offset) -{ - int idx; +static const struct platform_features snb_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7, + .cst_limit = CST_LIMIT_SNB, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, +}; - switch (offset) { - case MSR_PKG_ENERGY_STATUS: - case MSR_PKG_ENERGY_STAT: - idx = IDX_PKG_ENERGY; - break; - case MSR_DRAM_ENERGY_STATUS: - idx = IDX_DRAM_ENERGY; - break; - case MSR_PP0_ENERGY_STATUS: - idx = IDX_PP0_ENERGY; - break; - case MSR_PP1_ENERGY_STATUS: - idx = IDX_PP1_ENERGY; - break; - case MSR_PKG_PERF_STATUS: - idx = IDX_PKG_PERF; - break; - case MSR_DRAM_PERF_STATUS: - idx = IDX_DRAM_PERF; - break; - default: - idx = -1; - } - return idx; -} +static const struct platform_features snx_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7, + .cst_limit = CST_LIMIT_SNB, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL, +}; -int idx_valid(int idx) -{ - switch (idx) { - case IDX_PKG_ENERGY: - return do_rapl & (RAPL_PKG | RAPL_AMD_F17H); - case IDX_DRAM_ENERGY: - return do_rapl & RAPL_DRAM; - case IDX_PP0_ENERGY: - return do_rapl & RAPL_CORES_ENERGY_STATUS; - case IDX_PP1_ENERGY: - return do_rapl & RAPL_GFX; - case IDX_PKG_PERF: - return do_rapl & RAPL_PKG_PERF_STATUS; - case IDX_DRAM_PERF: - return do_rapl & RAPL_DRAM_PERF_STATUS; - default: - return 0; - } -} +static const struct platform_features ivb_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7, + .cst_limit = CST_LIMIT_SNB, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, +}; -struct sys_counters { - unsigned int added_thread_counters; - unsigned int added_core_counters; - unsigned int added_package_counters; - struct msr_counter *tp; - struct msr_counter *cp; - struct msr_counter *pp; -} sys; +static const struct platform_features ivx_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7, + .cst_limit = CST_LIMIT_SNB, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE | TRL_LIMIT1, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL, +}; -struct system_summary { - struct thread_data threads; - struct core_data cores; - struct pkg_data packages; -} average; +static const struct platform_features hsw_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7, + .cst_limit = CST_LIMIT_HSW, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE, + .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, +}; -struct cpu_topology { - int physical_package_id; - int die_id; - int logical_cpu_id; - int physical_node_id; - int logical_node_id; /* 0-based count within the package */ - int physical_core_id; - int thread_id; - cpu_set_t *put_ids; /* Processing Unit/Thread IDs */ -} *cpus; +static const struct platform_features hsx_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7, + .cst_limit = CST_LIMIT_HSW, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE | TRL_LIMIT1 | TRL_LIMIT2, + .plr_msrs = PLR_CORE | PLR_RING, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .has_fixed_rapl_unit = 1, +}; -struct topo_params { - int num_packages; - int num_die; - int num_cpus; - int num_cores; - int max_cpu_num; - int max_die_id; - int max_node_num; - int nodes_per_pkg; - int cores_per_node; - int threads_per_core; -} topo; +static const struct platform_features hswl_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10, + .cst_limit = CST_LIMIT_HSW, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE, + .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, +}; -struct timeval tv_even, tv_odd, tv_delta; +static const struct platform_features hswg_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7, + .cst_limit = CST_LIMIT_HSW, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE, + .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, +}; -int *irq_column_2_cpu; /* /proc/interrupts column numbers */ -int *irqs_per_cpu; /* indexed by cpu_num */ +static const struct platform_features bdw_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10, + .cst_limit = CST_LIMIT_HSW, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, +}; -void setup_all_buffers(void); +static const struct platform_features bdwg_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7, + .cst_limit = CST_LIMIT_HSW, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, +}; -char *sys_lpi_file; -char *sys_lpi_file_sysfs = "/sys/devices/system/cpu/cpuidle/low_power_idle_system_residency_us"; -char *sys_lpi_file_debugfs = "/sys/kernel/debug/pmc_core/slp_s0_residency_usec"; +static const struct platform_features bdx_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC3 | CC6 | PC2 | PC3 | PC6, + .cst_limit = CST_LIMIT_HSW, + .has_irtl_msrs = 1, + .has_cst_auto_convension = 1, + .trl_msrs = TRL_BASE, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .has_fixed_rapl_unit = 1, +}; -int cpu_is_not_present(int cpu) -{ - return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set); -} +static const struct platform_features skl_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .crystal_freq = 24000000, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10, + .cst_limit = CST_LIMIT_HSW, + .has_irtl_msrs = 1, + .has_ext_cst_msrs = 1, + .trl_msrs = TRL_BASE, + .tcc_offset_bits = 6, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS, + .enable_tsc_tweak = 1, +}; -/* - * run func(thread, core, package) in topology order - * skip non-present cpus - */ +static const struct platform_features cnl_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10, + .cst_limit = CST_LIMIT_HSW, + .has_irtl_msrs = 1, + .has_msr_core_c1_res = 1, + .has_ext_cst_msrs = 1, + .trl_msrs = TRL_BASE, + .tcc_offset_bits = 6, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS, + .enable_tsc_tweak = 1, +}; -int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pkg_data *), - struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base) -{ - int retval, pkg_no, core_no, thread_no, node_no; +/* Copied from cnl_features, with PC7/PC9 removed */ +static const struct platform_features adl_features = { + .has_msr_misc_feature_control = cnl_features.has_msr_misc_feature_control, + .has_msr_misc_pwr_mgmt = cnl_features.has_msr_misc_pwr_mgmt, + .has_nhm_msrs = cnl_features.has_nhm_msrs, + .has_config_tdp = cnl_features.has_config_tdp, + .bclk_freq = cnl_features.bclk_freq, + .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC8 | PC10, + .cst_limit = cnl_features.cst_limit, + .has_irtl_msrs = cnl_features.has_irtl_msrs, + .has_msr_core_c1_res = cnl_features.has_msr_core_c1_res, + .has_ext_cst_msrs = cnl_features.has_ext_cst_msrs, + .trl_msrs = cnl_features.trl_msrs, + .tcc_offset_bits = cnl_features.tcc_offset_bits, + .plat_rapl_msrs = cnl_features.plat_rapl_msrs, + .enable_tsc_tweak = cnl_features.enable_tsc_tweak, +}; - for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) { - for (node_no = 0; node_no < topo.nodes_per_pkg; node_no++) { - for (core_no = 0; core_no < topo.cores_per_node; ++core_no) { - for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) { - struct thread_data *t; - struct core_data *c; - struct pkg_data *p; +/* Copied from adl_features, with PC3/PC8 removed */ +static const struct platform_features lnl_features = { + .has_msr_misc_feature_control = adl_features.has_msr_misc_feature_control, + .has_msr_misc_pwr_mgmt = adl_features.has_msr_misc_pwr_mgmt, + .has_nhm_msrs = adl_features.has_nhm_msrs, + .has_config_tdp = adl_features.has_config_tdp, + .bclk_freq = adl_features.bclk_freq, + .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC6 | PC10, + .cst_limit = adl_features.cst_limit, + .has_irtl_msrs = adl_features.has_irtl_msrs, + .has_msr_core_c1_res = adl_features.has_msr_core_c1_res, + .has_ext_cst_msrs = adl_features.has_ext_cst_msrs, + .trl_msrs = adl_features.trl_msrs, + .tcc_offset_bits = adl_features.tcc_offset_bits, + .plat_rapl_msrs = adl_features.plat_rapl_msrs, + .enable_tsc_tweak = adl_features.enable_tsc_tweak, +}; - t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no); +static const struct platform_features skx_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC6 | PC2 | PC6, + .cst_limit = CST_LIMIT_SKX, + .has_irtl_msrs = 1, + .has_cst_auto_convension = 1, + .trl_msrs = TRL_BASE | TRL_CORECOUNT, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .has_fixed_rapl_unit = 1, +}; - if (cpu_is_not_present(t->cpu_id)) - continue; +static const struct platform_features icx_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC6 | PC2 | PC6, + .cst_limit = CST_LIMIT_ICX, + .has_msr_core_c1_res = 1, + .has_irtl_msrs = 1, + .has_cst_prewake_bit = 1, + .trl_msrs = TRL_BASE | TRL_CORECOUNT, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, + .has_fixed_rapl_unit = 1, +}; - c = GET_CORE(core_base, core_no, node_no, pkg_no); - p = GET_PKG(pkg_base, pkg_no); +static const struct platform_features spr_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC6 | PC2 | PC6, + .cst_limit = CST_LIMIT_SKX, + .has_msr_core_c1_res = 1, + .has_irtl_msrs = 1, + .has_cst_prewake_bit = 1, + .has_fixed_rapl_psys_unit = 1, + .trl_msrs = TRL_BASE | TRL_CORECOUNT, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, +}; - retval = func(t, c, p); - if (retval) - return retval; - } - } - } - } - return 0; -} +static const struct platform_features dmr_features = { + .has_msr_misc_feature_control = spr_features.has_msr_misc_feature_control, + .has_msr_misc_pwr_mgmt = spr_features.has_msr_misc_pwr_mgmt, + .has_nhm_msrs = spr_features.has_nhm_msrs, + .bclk_freq = spr_features.bclk_freq, + .supported_cstates = spr_features.supported_cstates, + .cst_limit = spr_features.cst_limit, + .has_msr_core_c1_res = spr_features.has_msr_core_c1_res, + .has_cst_prewake_bit = spr_features.has_cst_prewake_bit, + .has_fixed_rapl_psys_unit = spr_features.has_fixed_rapl_psys_unit, + .trl_msrs = spr_features.trl_msrs, + .has_msr_module_c6_res_ms = 1, /* DMR has Dual-Core-Module and MC6 MSR */ + .plat_rapl_msrs = 0, /* DMR does not have RAPL MSRs */ + .plr_msrs = 0, /* DMR does not have PLR MSRs */ + .has_irtl_msrs = 0, /* DMR does not have IRTL MSRs */ + .has_config_tdp = 0, /* DMR does not have CTDP MSRs */ +}; -int cpu_migrate(int cpu) -{ - CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set); - CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set); - if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1) - return -1; - else - return 0; -} +static const struct platform_features srf_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC6 | PC2 | PC6, + .cst_limit = CST_LIMIT_SKX, + .has_msr_core_c1_res = 1, + .has_msr_module_c6_res_ms = 1, + .has_irtl_msrs = 1, + .has_cst_prewake_bit = 1, + .trl_msrs = TRL_BASE | TRL_CORECOUNT, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, +}; -int get_msr_fd(int cpu) -{ - char pathname[32]; - int fd; +static const struct platform_features grr_features = { + .has_msr_misc_feature_control = 1, + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC6, + .cst_limit = CST_LIMIT_SKX, + .has_msr_core_c1_res = 1, + .has_msr_module_c6_res_ms = 1, + .has_irtl_msrs = 1, + .has_cst_prewake_bit = 1, + .trl_msrs = TRL_BASE | TRL_CORECOUNT, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, +}; - fd = fd_percpu[cpu]; +static const struct platform_features slv_features = { + .has_nhm_msrs = 1, + .bclk_freq = BCLK_SLV, + .supported_cstates = CC1 | CC6 | PC6, + .cst_limit = CST_LIMIT_SLV, + .has_msr_core_c1_res = 1, + .has_msr_module_c6_res_ms = 1, + .has_msr_c6_demotion_policy_config = 1, + .has_msr_atom_pkg_c6_residency = 1, + .trl_msrs = TRL_ATOM, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE, + .has_rapl_divisor = 1, + .rapl_quirk_tdp = 30, +}; - if (fd) - return fd; +static const struct platform_features slvd_features = { + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_SLV, + .supported_cstates = CC1 | CC6 | PC3 | PC6, + .cst_limit = CST_LIMIT_SLV, + .has_msr_atom_pkg_c6_residency = 1, + .trl_msrs = TRL_BASE, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE, + .rapl_quirk_tdp = 30, +}; - sprintf(pathname, "/dev/cpu/%d/msr", cpu); - fd = open(pathname, O_RDONLY); - if (fd < 0) - err(-1, "%s open failed, try chown or chmod +r /dev/cpu/*/msr, or run as root", pathname); +static const struct platform_features amt_features = { + .has_nhm_msrs = 1, + .bclk_freq = BCLK_133MHZ, + .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6, + .cst_limit = CST_LIMIT_AMT, + .trl_msrs = TRL_BASE, +}; - fd_percpu[cpu] = fd; +static const struct platform_features gmt_features = { + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_100MHZ, + .crystal_freq = 19200000, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10, + .cst_limit = CST_LIMIT_GMT, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE | TRL_CORECOUNT, + .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO, +}; - return fd; -} +static const struct platform_features gmtd_features = { + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_100MHZ, + .crystal_freq = 25000000, + .supported_cstates = CC1 | CC6 | PC2 | PC6, + .cst_limit = CST_LIMIT_GMT, + .has_irtl_msrs = 1, + .has_msr_core_c1_res = 1, + .trl_msrs = TRL_BASE | TRL_CORECOUNT, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_CORE_ENERGY_STATUS, +}; -static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags) -{ - return syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags); -} +static const struct platform_features gmtp_features = { + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_100MHZ, + .crystal_freq = 19200000, + .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10, + .cst_limit = CST_LIMIT_GMT, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE, + .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO, +}; -static int perf_instr_count_open(int cpu_num) -{ - struct perf_event_attr pea; - int fd; +static const struct platform_features tmt_features = { + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10, + .cst_limit = CST_LIMIT_GMT, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX, + .enable_tsc_tweak = 1, +}; - memset(&pea, 0, sizeof(struct perf_event_attr)); - pea.type = PERF_TYPE_HARDWARE; - pea.size = sizeof(struct perf_event_attr); - pea.config = PERF_COUNT_HW_INSTRUCTIONS; +static const struct platform_features tmtd_features = { + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC6, + .cst_limit = CST_LIMIT_GMT, + .has_irtl_msrs = 1, + .trl_msrs = TRL_BASE | TRL_CORECOUNT, + .plat_rapl_msrs = RAPL_PKG_ALL, +}; - /* counter for cpu_num, including user + kernel and all processes */ - fd = perf_event_open(&pea, -1, cpu_num, -1, 0); - if (fd == -1) { - warnx("capget(CAP_PERFMON) failed, try \"# setcap cap_sys_admin=ep %s\"", progname); - BIC_NOT_PRESENT(BIC_IPC); - } +static const struct platform_features knl_features = { + .has_msr_misc_pwr_mgmt = 1, + .has_nhm_msrs = 1, + .has_config_tdp = 1, + .bclk_freq = BCLK_100MHZ, + .supported_cstates = CC1 | CC6 | PC3 | PC6, + .cst_limit = CST_LIMIT_KNL, + .has_msr_knl_core_c6_residency = 1, + .trl_msrs = TRL_KNL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .has_fixed_rapl_unit = 1, + .need_perf_multiplier = 1, +}; - return fd; -} +static const struct platform_features default_features = { +}; -int get_instr_count_fd(int cpu) -{ - if (fd_instr_count_percpu[cpu]) - return fd_instr_count_percpu[cpu]; +static const struct platform_features amd_features_with_rapl = { + .plat_rapl_msrs = RAPL_AMD_F17H, + .has_per_core_rapl = 1, + .rapl_quirk_tdp = 280, /* This is the max stock TDP of HEDT/Server Fam17h+ chips */ +}; - fd_instr_count_percpu[cpu] = perf_instr_count_open(cpu); +static const struct platform_data turbostat_pdata[] = { + { INTEL_NEHALEM, &nhm_features }, + { INTEL_NEHALEM_G, &nhm_features }, + { INTEL_NEHALEM_EP, &nhm_features }, + { INTEL_NEHALEM_EX, &nhx_features }, + { INTEL_WESTMERE, &nhm_features }, + { INTEL_WESTMERE_EP, &nhm_features }, + { INTEL_WESTMERE_EX, &nhx_features }, + { INTEL_SANDYBRIDGE, &snb_features }, + { INTEL_SANDYBRIDGE_X, &snx_features }, + { INTEL_IVYBRIDGE, &ivb_features }, + { INTEL_IVYBRIDGE_X, &ivx_features }, + { INTEL_HASWELL, &hsw_features }, + { INTEL_HASWELL_X, &hsx_features }, + { INTEL_HASWELL_L, &hswl_features }, + { INTEL_HASWELL_G, &hswg_features }, + { INTEL_BROADWELL, &bdw_features }, + { INTEL_BROADWELL_G, &bdwg_features }, + { INTEL_BROADWELL_X, &bdx_features }, + { INTEL_BROADWELL_D, &bdx_features }, + { INTEL_SKYLAKE_L, &skl_features }, + { INTEL_SKYLAKE, &skl_features }, + { INTEL_SKYLAKE_X, &skx_features }, + { INTEL_KABYLAKE_L, &skl_features }, + { INTEL_KABYLAKE, &skl_features }, + { INTEL_COMETLAKE, &skl_features }, + { INTEL_COMETLAKE_L, &skl_features }, + { INTEL_CANNONLAKE_L, &cnl_features }, + { INTEL_ICELAKE_X, &icx_features }, + { INTEL_ICELAKE_D, &icx_features }, + { INTEL_ICELAKE_L, &cnl_features }, + { INTEL_ICELAKE_NNPI, &cnl_features }, + { INTEL_ROCKETLAKE, &cnl_features }, + { INTEL_TIGERLAKE_L, &cnl_features }, + { INTEL_TIGERLAKE, &cnl_features }, + { INTEL_SAPPHIRERAPIDS_X, &spr_features }, + { INTEL_EMERALDRAPIDS_X, &spr_features }, + { INTEL_GRANITERAPIDS_X, &spr_features }, + { INTEL_GRANITERAPIDS_D, &spr_features }, + { INTEL_DIAMONDRAPIDS_X, &dmr_features }, + { INTEL_LAKEFIELD, &cnl_features }, + { INTEL_ALDERLAKE, &adl_features }, + { INTEL_ALDERLAKE_L, &adl_features }, + { INTEL_RAPTORLAKE, &adl_features }, + { INTEL_RAPTORLAKE_P, &adl_features }, + { INTEL_RAPTORLAKE_S, &adl_features }, + { INTEL_BARTLETTLAKE, &adl_features }, + { INTEL_METEORLAKE, &adl_features }, + { INTEL_METEORLAKE_L, &adl_features }, + { INTEL_ARROWLAKE_H, &adl_features }, + { INTEL_ARROWLAKE_U, &adl_features }, + { INTEL_ARROWLAKE, &adl_features }, + { INTEL_LUNARLAKE_M, &lnl_features }, + { INTEL_PANTHERLAKE_L, &lnl_features }, + { INTEL_NOVALAKE, &lnl_features }, + { INTEL_NOVALAKE_L, &lnl_features }, + { INTEL_WILDCATLAKE_L, &lnl_features }, + { INTEL_ATOM_SILVERMONT, &slv_features }, + { INTEL_ATOM_SILVERMONT_D, &slvd_features }, + { INTEL_ATOM_AIRMONT, &amt_features }, + { INTEL_ATOM_GOLDMONT, &gmt_features }, + { INTEL_ATOM_GOLDMONT_D, &gmtd_features }, + { INTEL_ATOM_GOLDMONT_PLUS, &gmtp_features }, + { INTEL_ATOM_TREMONT_D, &tmtd_features }, + { INTEL_ATOM_TREMONT, &tmt_features }, + { INTEL_ATOM_TREMONT_L, &tmt_features }, + { INTEL_ATOM_GRACEMONT, &adl_features }, + { INTEL_ATOM_CRESTMONT_X, &srf_features }, + { INTEL_ATOM_CRESTMONT, &grr_features }, + { INTEL_ATOM_DARKMONT_X, &srf_features }, + { INTEL_XEON_PHI_KNL, &knl_features }, + { INTEL_XEON_PHI_KNM, &knl_features }, + /* + * Missing support for + * INTEL_ICELAKE + * INTEL_ATOM_SILVERMONT_MID + * INTEL_ATOM_SILVERMONT_MID2 + * INTEL_ATOM_AIRMONT_NP + */ + { 0, NULL }, +}; - return fd_instr_count_percpu[cpu]; -} +static const struct platform_features *platform; -int get_msr(int cpu, off_t offset, unsigned long long *msr) +void probe_platform_features(unsigned int family, unsigned int model) { - ssize_t retval; + int i; - retval = pread(get_msr_fd(cpu), msr, sizeof(*msr), offset); + if (authentic_amd || hygon_genuine) { + /* fallback to default features on unsupported models */ + force_load++; + if (max_extended_level >= 0x80000007) { + unsigned int eax, ebx, ecx, edx; + + __cpuid(0x80000007, eax, ebx, ecx, edx); + /* RAPL (Fam 17h+) */ + if ((edx & (1 << 14)) && family >= 0x17) + platform = &amd_features_with_rapl; + } + goto end; + } - if (retval != sizeof *msr) - err(-1, "cpu%d: msr offset 0x%llx read failed", cpu, (unsigned long long)offset); + if (!genuine_intel) + goto end; - return 0; -} + for (i = 0; turbostat_pdata[i].features; i++) { + if (VFM_FAMILY(turbostat_pdata[i].vfm) == family && VFM_MODEL(turbostat_pdata[i].vfm) == model) { + platform = turbostat_pdata[i].features; + return; + } + } -#define MAX_DEFERRED 16 -char *deferred_add_names[MAX_DEFERRED]; -char *deferred_skip_names[MAX_DEFERRED]; -int deferred_add_index; -int deferred_skip_index; +end: + if (force_load && !platform) { + fprintf(outf, "Forced to run on unsupported platform!\n"); + platform = &default_features; + } -/* - * HIDE_LIST - hide this list of counters, show the rest [default] - * SHOW_LIST - show this list of counters, hide the rest - */ -enum show_hide_mode { SHOW_LIST, HIDE_LIST } global_show_hide_mode = HIDE_LIST; + if (platform) + return; -void help(void) -{ - fprintf(outf, - "Usage: turbostat [OPTIONS][(--interval seconds) | COMMAND ...]\n" - "\n" - "Turbostat forks the specified COMMAND and prints statistics\n" - "when COMMAND completes.\n" - "If no COMMAND is specified, turbostat wakes every 5-seconds\n" - "to print statistics, until interrupted.\n" - " -a, --add add a counter\n" - " eg. --add msr0x10,u64,cpu,delta,MY_TSC\n" - " -c, --cpu cpu-set limit output to summary plus cpu-set:\n" - " {core | package | j,k,l..m,n-p }\n" - " -d, --debug displays usec, Time_Of_Day_Seconds and more debugging\n" - " -D, --Dump displays the raw counter values\n" - " -e, --enable [all | column]\n" - " shows all or the specified disabled column\n" - " -H, --hide [column|column,column,...]\n" - " hide the specified column(s)\n" - " -i, --interval sec.subsec\n" - " Override default 5-second measurement interval\n" - " -J, --Joules displays energy in Joules instead of Watts\n" - " -l, --list list column headers only\n" - " -n, --num_iterations num\n" - " number of the measurement iterations\n" - " -N, --header_iterations num\n" - " print header every num iterations\n" - " -o, --out file\n" - " create or truncate \"file\" for all output\n" - " -q, --quiet skip decoding system configuration header\n" - " -s, --show [column|column,column,...]\n" - " show only the specified column(s)\n" - " -S, --Summary\n" - " limits output to 1-line system summary per interval\n" - " -T, --TCC temperature\n" - " sets the Thermal Control Circuit temperature in\n" - " degrees Celsius\n" - " -h, --help print this help message\n" - " -v, --version print version information\n" "\n" "For more help, run \"man turbostat\"\n"); + fprintf(stderr, "Unsupported platform detected.\n\tSee RUN THE LATEST VERSION on turbostat(8)\n"); + exit(1); } -/* - * bic_lookup - * for all the strings in comma separate name_list, - * set the approprate bit in return value. - */ -unsigned long long bic_lookup(char *name_list, enum show_hide_mode mode) -{ - unsigned int i; - unsigned long long retval = 0; +/* Model specific support End */ - while (name_list) { - char *comma; +#define TJMAX_DEFAULT 100 - comma = strchr(name_list, ','); +/* MSRs that are not yet in the kernel-provided header. */ +#define MSR_RAPL_PWR_UNIT 0xc0010299 +#define MSR_CORE_ENERGY_STAT 0xc001029a +#define MSR_PKG_ENERGY_STAT 0xc001029b - if (comma) - *comma = '\0'; +#define MAX(a, b) ((a) > (b) ? (a) : (b)) - for (i = 0; i < MAX_BIC; ++i) { - if (!strcmp(name_list, bic[i].name)) { - retval |= (1ULL << i); - break; - } - if (!strcmp(name_list, "all")) { - retval |= ~0; - break; - } else if (!strcmp(name_list, "topology")) { - retval |= BIC_TOPOLOGY; - break; - } else if (!strcmp(name_list, "power")) { - retval |= BIC_THERMAL_PWR; - break; - } else if (!strcmp(name_list, "idle")) { - retval |= BIC_IDLE; - break; - } else if (!strcmp(name_list, "frequency")) { - retval |= BIC_FREQUENCY; - break; - } else if (!strcmp(name_list, "other")) { - retval |= BIC_OTHER; - break; - } +int backwards_count; +char *progname; - } - if (i == MAX_BIC) { - if (mode == SHOW_LIST) { - deferred_add_names[deferred_add_index++] = name_list; - if (deferred_add_index >= MAX_DEFERRED) { - fprintf(stderr, "More than max %d un-recognized --add options '%s'\n", - MAX_DEFERRED, name_list); - help(); - exit(1); - } - } else { - deferred_skip_names[deferred_skip_index++] = name_list; - if (debug) - fprintf(stderr, "deferred \"%s\"\n", name_list); - if (deferred_skip_index >= MAX_DEFERRED) { - fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n", - MAX_DEFERRED, name_list); - help(); - exit(1); - } - } - } +#define CPU_SUBSET_MAXCPUS 8192 /* need to use before probe... */ +cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset; +size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size; +#define MAX_ADDED_THREAD_COUNTERS 24 +#define MAX_ADDED_CORE_COUNTERS 8 +#define MAX_ADDED_PACKAGE_COUNTERS 16 +#define PMT_MAX_ADDED_THREAD_COUNTERS 24 +#define PMT_MAX_ADDED_CORE_COUNTERS 8 +#define PMT_MAX_ADDED_PACKAGE_COUNTERS 16 +#define BITMASK_SIZE 32 - name_list = comma; - if (name_list) - name_list++; +#define ZERO_ARRAY(arr) (memset(arr, 0, sizeof(arr)) + __must_be_array(arr)) + +/* Indexes used to map data read from perf and MSRs into global variables */ +enum rapl_rci_index { + RAPL_RCI_INDEX_ENERGY_PKG = 0, + RAPL_RCI_INDEX_ENERGY_CORES = 1, + RAPL_RCI_INDEX_DRAM = 2, + RAPL_RCI_INDEX_GFX = 3, + RAPL_RCI_INDEX_PKG_PERF_STATUS = 4, + RAPL_RCI_INDEX_DRAM_PERF_STATUS = 5, + RAPL_RCI_INDEX_CORE_ENERGY = 6, + RAPL_RCI_INDEX_ENERGY_PLATFORM = 7, + NUM_RAPL_COUNTERS, +}; - } - return retval; -} +enum rapl_unit { + RAPL_UNIT_INVALID, + RAPL_UNIT_JOULES, + RAPL_UNIT_WATTS, +}; -void print_header(char *delim) -{ - struct msr_counter *mp; - int printed = 0; +struct rapl_counter_info_t { + unsigned long long data[NUM_RAPL_COUNTERS]; + enum counter_source source[NUM_RAPL_COUNTERS]; + unsigned long long flags[NUM_RAPL_COUNTERS]; + double scale[NUM_RAPL_COUNTERS]; + enum rapl_unit unit[NUM_RAPL_COUNTERS]; + unsigned long long msr[NUM_RAPL_COUNTERS]; + unsigned long long msr_mask[NUM_RAPL_COUNTERS]; + int msr_shift[NUM_RAPL_COUNTERS]; + + int fd_perf; +}; - if (DO_BIC(BIC_USEC)) - outp += sprintf(outp, "%susec", (printed++ ? delim : "")); - if (DO_BIC(BIC_TOD)) - outp += sprintf(outp, "%sTime_Of_Day_Seconds", (printed++ ? delim : "")); - if (DO_BIC(BIC_Package)) - outp += sprintf(outp, "%sPackage", (printed++ ? delim : "")); - if (DO_BIC(BIC_Die)) - outp += sprintf(outp, "%sDie", (printed++ ? delim : "")); - if (DO_BIC(BIC_Node)) - outp += sprintf(outp, "%sNode", (printed++ ? delim : "")); - if (DO_BIC(BIC_Core)) - outp += sprintf(outp, "%sCore", (printed++ ? delim : "")); - if (DO_BIC(BIC_CPU)) - outp += sprintf(outp, "%sCPU", (printed++ ? delim : "")); - if (DO_BIC(BIC_APIC)) - outp += sprintf(outp, "%sAPIC", (printed++ ? delim : "")); - if (DO_BIC(BIC_X2APIC)) - outp += sprintf(outp, "%sX2APIC", (printed++ ? delim : "")); - if (DO_BIC(BIC_Avg_MHz)) - outp += sprintf(outp, "%sAvg_MHz", (printed++ ? delim : "")); - if (DO_BIC(BIC_Busy)) - outp += sprintf(outp, "%sBusy%%", (printed++ ? delim : "")); - if (DO_BIC(BIC_Bzy_MHz)) - outp += sprintf(outp, "%sBzy_MHz", (printed++ ? delim : "")); - if (DO_BIC(BIC_TSC_MHz)) - outp += sprintf(outp, "%sTSC_MHz", (printed++ ? delim : "")); +/* struct rapl_counter_info_t for each RAPL domain */ +struct rapl_counter_info_t *rapl_counter_info_perdomain; +unsigned int rapl_counter_info_perdomain_size; - if (DO_BIC(BIC_IPC)) - outp += sprintf(outp, "%sIPC", (printed++ ? delim : "")); +#define RAPL_COUNTER_FLAG_PLATFORM_COUNTER (1u << 0) +#define RAPL_COUNTER_FLAG_USE_MSR_SUM (1u << 1) - if (DO_BIC(BIC_IRQ)) { - if (sums_need_wide_columns) - outp += sprintf(outp, "%s IRQ", (printed++ ? delim : "")); - else - outp += sprintf(outp, "%sIRQ", (printed++ ? delim : "")); - } +struct rapl_counter_arch_info { + int feature_mask; /* Mask for testing if the counter is supported on host */ + const char *perf_subsys; + const char *perf_name; + unsigned long long msr; + unsigned long long msr_mask; + int msr_shift; /* Positive mean shift right, negative mean shift left */ + double *platform_rapl_msr_scale; /* Scale applied to values read by MSR (platform dependent, filled at runtime) */ + unsigned int rci_index; /* Maps data from perf counters to global variables */ + unsigned int bic_number; + double compat_scale; /* Some counters require constant scaling to be in the same range as other, similar ones */ + unsigned long long flags; +}; - if (DO_BIC(BIC_SMI)) - outp += sprintf(outp, "%sSMI", (printed++ ? delim : "")); +static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = { + { + .feature_mask = RAPL_PKG, + .perf_subsys = "power", + .perf_name = "energy-pkg", + .msr = MSR_PKG_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_PKG, + .bic_number = BIC_PkgWatt, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_PKG, + .perf_subsys = "power", + .perf_name = "energy-pkg", + .msr = MSR_PKG_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_PKG, + .bic_number = BIC_Pkg_J, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_AMD_F17H, + .perf_subsys = "power", + .perf_name = "energy-pkg", + .msr = MSR_PKG_ENERGY_STAT, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_PKG, + .bic_number = BIC_PkgWatt, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_AMD_F17H, + .perf_subsys = "power", + .perf_name = "energy-pkg", + .msr = MSR_PKG_ENERGY_STAT, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_PKG, + .bic_number = BIC_Pkg_J, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_CORE_ENERGY_STATUS, + .perf_subsys = "power", + .perf_name = "energy-cores", + .msr = MSR_PP0_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_CORES, + .bic_number = BIC_CorWatt, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_CORE_ENERGY_STATUS, + .perf_subsys = "power", + .perf_name = "energy-cores", + .msr = MSR_PP0_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_CORES, + .bic_number = BIC_Cor_J, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_DRAM, + .perf_subsys = "power", + .perf_name = "energy-ram", + .msr = MSR_DRAM_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_dram_energy_units, + .rci_index = RAPL_RCI_INDEX_DRAM, + .bic_number = BIC_RAMWatt, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_DRAM, + .perf_subsys = "power", + .perf_name = "energy-ram", + .msr = MSR_DRAM_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_dram_energy_units, + .rci_index = RAPL_RCI_INDEX_DRAM, + .bic_number = BIC_RAM_J, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_GFX, + .perf_subsys = "power", + .perf_name = "energy-gpu", + .msr = MSR_PP1_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_GFX, + .bic_number = BIC_GFXWatt, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_GFX, + .perf_subsys = "power", + .perf_name = "energy-gpu", + .msr = MSR_PP1_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_GFX, + .bic_number = BIC_GFX_J, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_PKG_PERF_STATUS, + .perf_subsys = NULL, + .perf_name = NULL, + .msr = MSR_PKG_PERF_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_time_units, + .rci_index = RAPL_RCI_INDEX_PKG_PERF_STATUS, + .bic_number = BIC_PKG__, + .compat_scale = 100.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_DRAM_PERF_STATUS, + .perf_subsys = NULL, + .perf_name = NULL, + .msr = MSR_DRAM_PERF_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_time_units, + .rci_index = RAPL_RCI_INDEX_DRAM_PERF_STATUS, + .bic_number = BIC_RAM__, + .compat_scale = 100.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_AMD_F17H, + .perf_subsys = NULL, + .perf_name = NULL, + .msr = MSR_CORE_ENERGY_STAT, + .msr_mask = 0xFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_CORE_ENERGY, + .bic_number = BIC_CorWatt, + .compat_scale = 1.0, + .flags = 0, + }, + { + .feature_mask = RAPL_AMD_F17H, + .perf_subsys = NULL, + .perf_name = NULL, + .msr = MSR_CORE_ENERGY_STAT, + .msr_mask = 0xFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_CORE_ENERGY, + .bic_number = BIC_Cor_J, + .compat_scale = 1.0, + .flags = 0, + }, + { + .feature_mask = RAPL_PSYS, + .perf_subsys = "power", + .perf_name = "energy-psys", + .msr = MSR_PLATFORM_ENERGY_STATUS, + .msr_mask = 0x00000000FFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_psys_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM, + .bic_number = BIC_SysWatt, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_PSYS, + .perf_subsys = "power", + .perf_name = "energy-psys", + .msr = MSR_PLATFORM_ENERGY_STATUS, + .msr_mask = 0x00000000FFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_psys_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM, + .bic_number = BIC_Sys_J, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, +}; - for (mp = sys.tp; mp; mp = mp->next) { +struct rapl_counter { + unsigned long long raw_value; + enum rapl_unit unit; + double scale; +}; - if (mp->format == FORMAT_RAW) { - if (mp->width == 64) - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), mp->name); - else - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), mp->name); - } else { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), mp->name); - else - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), mp->name); - } - } +/* Indexes used to map data read from perf and MSRs into global variables */ +enum ccstate_rci_index { + CCSTATE_RCI_INDEX_C1_RESIDENCY = 0, + CCSTATE_RCI_INDEX_C3_RESIDENCY = 1, + CCSTATE_RCI_INDEX_C6_RESIDENCY = 2, + CCSTATE_RCI_INDEX_C7_RESIDENCY = 3, + PCSTATE_RCI_INDEX_C2_RESIDENCY = 4, + PCSTATE_RCI_INDEX_C3_RESIDENCY = 5, + PCSTATE_RCI_INDEX_C6_RESIDENCY = 6, + PCSTATE_RCI_INDEX_C7_RESIDENCY = 7, + PCSTATE_RCI_INDEX_C8_RESIDENCY = 8, + PCSTATE_RCI_INDEX_C9_RESIDENCY = 9, + PCSTATE_RCI_INDEX_C10_RESIDENCY = 10, + NUM_CSTATE_COUNTERS, +}; - if (DO_BIC(BIC_CPU_c1)) - outp += sprintf(outp, "%sCPU%%c1", (printed++ ? delim : "")); - if (DO_BIC(BIC_CPU_c3)) - outp += sprintf(outp, "%sCPU%%c3", (printed++ ? delim : "")); - if (DO_BIC(BIC_CPU_c6)) - outp += sprintf(outp, "%sCPU%%c6", (printed++ ? delim : "")); - if (DO_BIC(BIC_CPU_c7)) - outp += sprintf(outp, "%sCPU%%c7", (printed++ ? delim : "")); +struct cstate_counter_info_t { + unsigned long long data[NUM_CSTATE_COUNTERS]; + enum counter_source source[NUM_CSTATE_COUNTERS]; + unsigned long long msr[NUM_CSTATE_COUNTERS]; + int fd_perf_core; + int fd_perf_pkg; +}; - if (DO_BIC(BIC_Mod_c6)) - outp += sprintf(outp, "%sMod%%c6", (printed++ ? delim : "")); +struct cstate_counter_info_t *ccstate_counter_info; +unsigned int ccstate_counter_info_size; - if (DO_BIC(BIC_CoreTmp)) - outp += sprintf(outp, "%sCoreTmp", (printed++ ? delim : "")); +#define CSTATE_COUNTER_FLAG_COLLECT_PER_CORE (1u << 0) +#define CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD ((1u << 1) | CSTATE_COUNTER_FLAG_COLLECT_PER_CORE) +#define CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY (1u << 2) - if (DO_BIC(BIC_CORE_THROT_CNT)) - outp += sprintf(outp, "%sCoreThr", (printed++ ? delim : "")); +struct cstate_counter_arch_info { + int feature_mask; /* Mask for testing if the counter is supported on host */ + const char *perf_subsys; + const char *perf_name; + unsigned long long msr; + unsigned int rci_index; /* Maps data from perf counters to global variables */ + unsigned int bic_number; + unsigned long long flags; + int pkg_cstate_limit; +}; - if (do_rapl && !rapl_joules) { - if (DO_BIC(BIC_CorWatt) && (do_rapl & RAPL_PER_CORE_ENERGY)) - outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : "")); - } else if (do_rapl && rapl_joules) { - if (DO_BIC(BIC_Cor_J) && (do_rapl & RAPL_PER_CORE_ENERGY)) - outp += sprintf(outp, "%sCor_J", (printed++ ? delim : "")); - } +static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { + { + .feature_mask = CC1, + .perf_subsys = "cstate_core", + .perf_name = "c1-residency", + .msr = MSR_CORE_C1_RES, + .rci_index = CCSTATE_RCI_INDEX_C1_RESIDENCY, + .bic_number = BIC_CPU_c1, + .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD, + .pkg_cstate_limit = 0, + }, + { + .feature_mask = CC3, + .perf_subsys = "cstate_core", + .perf_name = "c3-residency", + .msr = MSR_CORE_C3_RESIDENCY, + .rci_index = CCSTATE_RCI_INDEX_C3_RESIDENCY, + .bic_number = BIC_CPU_c3, + .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + .pkg_cstate_limit = 0, + }, + { + .feature_mask = CC6, + .perf_subsys = "cstate_core", + .perf_name = "c6-residency", + .msr = MSR_CORE_C6_RESIDENCY, + .rci_index = CCSTATE_RCI_INDEX_C6_RESIDENCY, + .bic_number = BIC_CPU_c6, + .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + .pkg_cstate_limit = 0, + }, + { + .feature_mask = CC7, + .perf_subsys = "cstate_core", + .perf_name = "c7-residency", + .msr = MSR_CORE_C7_RESIDENCY, + .rci_index = CCSTATE_RCI_INDEX_C7_RESIDENCY, + .bic_number = BIC_CPU_c7, + .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, + .pkg_cstate_limit = 0, + }, + { + .feature_mask = PC2, + .perf_subsys = "cstate_pkg", + .perf_name = "c2-residency", + .msr = MSR_PKG_C2_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C2_RESIDENCY, + .bic_number = BIC_Pkgpc2, + .flags = 0, + .pkg_cstate_limit = PCL__2, + }, + { + .feature_mask = PC3, + .perf_subsys = "cstate_pkg", + .perf_name = "c3-residency", + .msr = MSR_PKG_C3_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C3_RESIDENCY, + .bic_number = BIC_Pkgpc3, + .flags = 0, + .pkg_cstate_limit = PCL__3, + }, + { + .feature_mask = PC6, + .perf_subsys = "cstate_pkg", + .perf_name = "c6-residency", + .msr = MSR_PKG_C6_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C6_RESIDENCY, + .bic_number = BIC_Pkgpc6, + .flags = 0, + .pkg_cstate_limit = PCL__6, + }, + { + .feature_mask = PC7, + .perf_subsys = "cstate_pkg", + .perf_name = "c7-residency", + .msr = MSR_PKG_C7_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C7_RESIDENCY, + .bic_number = BIC_Pkgpc7, + .flags = 0, + .pkg_cstate_limit = PCL__7, + }, + { + .feature_mask = PC8, + .perf_subsys = "cstate_pkg", + .perf_name = "c8-residency", + .msr = MSR_PKG_C8_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C8_RESIDENCY, + .bic_number = BIC_Pkgpc8, + .flags = 0, + .pkg_cstate_limit = PCL__8, + }, + { + .feature_mask = PC9, + .perf_subsys = "cstate_pkg", + .perf_name = "c9-residency", + .msr = MSR_PKG_C9_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C9_RESIDENCY, + .bic_number = BIC_Pkgpc9, + .flags = 0, + .pkg_cstate_limit = PCL__9, + }, + { + .feature_mask = PC10, + .perf_subsys = "cstate_pkg", + .perf_name = "c10-residency", + .msr = MSR_PKG_C10_RESIDENCY, + .rci_index = PCSTATE_RCI_INDEX_C10_RESIDENCY, + .bic_number = BIC_Pkgpc10, + .flags = 0, + .pkg_cstate_limit = PCL_10, + }, +}; - for (mp = sys.cp; mp; mp = mp->next) { - if (mp->format == FORMAT_RAW) { - if (mp->width == 64) - outp += sprintf(outp, "%s%18.18s", delim, mp->name); - else - outp += sprintf(outp, "%s%10.10s", delim, mp->name); - } else { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", delim, mp->name); - else - outp += sprintf(outp, "%s%s", delim, mp->name); - } - } +/* Indexes used to map data read from perf and MSRs into global variables */ +enum msr_rci_index { + MSR_RCI_INDEX_APERF = 0, + MSR_RCI_INDEX_MPERF = 1, + MSR_RCI_INDEX_SMI = 2, + NUM_MSR_COUNTERS, +}; - if (DO_BIC(BIC_PkgTmp)) - outp += sprintf(outp, "%sPkgTmp", (printed++ ? delim : "")); +struct msr_counter_info_t { + unsigned long long data[NUM_MSR_COUNTERS]; + enum counter_source source[NUM_MSR_COUNTERS]; + unsigned long long msr[NUM_MSR_COUNTERS]; + unsigned long long msr_mask[NUM_MSR_COUNTERS]; + int fd_perf; +}; - if (DO_BIC(BIC_GFX_rc6)) - outp += sprintf(outp, "%sGFX%%rc6", (printed++ ? delim : "")); +struct msr_counter_info_t *msr_counter_info; +unsigned int msr_counter_info_size; - if (DO_BIC(BIC_GFXMHz)) - outp += sprintf(outp, "%sGFXMHz", (printed++ ? delim : "")); +struct msr_counter_arch_info { + const char *perf_subsys; + const char *perf_name; + unsigned long long msr; + unsigned long long msr_mask; + unsigned int rci_index; /* Maps data from perf counters to global variables */ + bool needed; + bool present; +}; - if (DO_BIC(BIC_GFXACTMHz)) - outp += sprintf(outp, "%sGFXAMHz", (printed++ ? delim : "")); +enum msr_arch_info_index { + MSR_ARCH_INFO_APERF_INDEX = 0, + MSR_ARCH_INFO_MPERF_INDEX = 1, + MSR_ARCH_INFO_SMI_INDEX = 2, +}; - if (DO_BIC(BIC_Totl_c0)) - outp += sprintf(outp, "%sTotl%%C0", (printed++ ? delim : "")); - if (DO_BIC(BIC_Any_c0)) - outp += sprintf(outp, "%sAny%%C0", (printed++ ? delim : "")); - if (DO_BIC(BIC_GFX_c0)) - outp += sprintf(outp, "%sGFX%%C0", (printed++ ? delim : "")); - if (DO_BIC(BIC_CPUGFX)) - outp += sprintf(outp, "%sCPUGFX%%", (printed++ ? delim : "")); +static struct msr_counter_arch_info msr_counter_arch_infos[] = { + [MSR_ARCH_INFO_APERF_INDEX] = { + .perf_subsys = "msr", + .perf_name = "aperf", + .msr = MSR_IA32_APERF, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .rci_index = MSR_RCI_INDEX_APERF, + }, + + [MSR_ARCH_INFO_MPERF_INDEX] = { + .perf_subsys = "msr", + .perf_name = "mperf", + .msr = MSR_IA32_MPERF, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .rci_index = MSR_RCI_INDEX_MPERF, + }, + + [MSR_ARCH_INFO_SMI_INDEX] = { + .perf_subsys = "msr", + .perf_name = "smi", + .msr = MSR_SMI_COUNT, + .msr_mask = 0xFFFFFFFF, + .rci_index = MSR_RCI_INDEX_SMI, + }, +}; - if (DO_BIC(BIC_Pkgpc2)) - outp += sprintf(outp, "%sPkg%%pc2", (printed++ ? delim : "")); - if (DO_BIC(BIC_Pkgpc3)) - outp += sprintf(outp, "%sPkg%%pc3", (printed++ ? delim : "")); - if (DO_BIC(BIC_Pkgpc6)) - outp += sprintf(outp, "%sPkg%%pc6", (printed++ ? delim : "")); - if (DO_BIC(BIC_Pkgpc7)) - outp += sprintf(outp, "%sPkg%%pc7", (printed++ ? delim : "")); - if (DO_BIC(BIC_Pkgpc8)) - outp += sprintf(outp, "%sPkg%%pc8", (printed++ ? delim : "")); - if (DO_BIC(BIC_Pkgpc9)) - outp += sprintf(outp, "%sPkg%%pc9", (printed++ ? delim : "")); - if (DO_BIC(BIC_Pkgpc10)) - outp += sprintf(outp, "%sPk%%pc10", (printed++ ? delim : "")); - if (DO_BIC(BIC_CPU_LPI)) - outp += sprintf(outp, "%sCPU%%LPI", (printed++ ? delim : "")); - if (DO_BIC(BIC_SYS_LPI)) - outp += sprintf(outp, "%sSYS%%LPI", (printed++ ? delim : "")); +/* Can be redefined when compiling, useful for testing. */ +#ifndef SYSFS_TELEM_PATH +#define SYSFS_TELEM_PATH "/sys/class/intel_pmt" +#endif - if (do_rapl && !rapl_joules) { - if (DO_BIC(BIC_PkgWatt)) - outp += sprintf(outp, "%sPkgWatt", (printed++ ? delim : "")); - if (DO_BIC(BIC_CorWatt) && !(do_rapl & RAPL_PER_CORE_ENERGY)) - outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : "")); - if (DO_BIC(BIC_GFXWatt)) - outp += sprintf(outp, "%sGFXWatt", (printed++ ? delim : "")); - if (DO_BIC(BIC_RAMWatt)) - outp += sprintf(outp, "%sRAMWatt", (printed++ ? delim : "")); - if (DO_BIC(BIC_PKG__)) - outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : "")); - if (DO_BIC(BIC_RAM__)) - outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : "")); - } else if (do_rapl && rapl_joules) { - if (DO_BIC(BIC_Pkg_J)) - outp += sprintf(outp, "%sPkg_J", (printed++ ? delim : "")); - if (DO_BIC(BIC_Cor_J) && !(do_rapl & RAPL_PER_CORE_ENERGY)) - outp += sprintf(outp, "%sCor_J", (printed++ ? delim : "")); - if (DO_BIC(BIC_GFX_J)) - outp += sprintf(outp, "%sGFX_J", (printed++ ? delim : "")); - if (DO_BIC(BIC_RAM_J)) - outp += sprintf(outp, "%sRAM_J", (printed++ ? delim : "")); - if (DO_BIC(BIC_PKG__)) - outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : "")); - if (DO_BIC(BIC_RAM__)) - outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : "")); - } - if (DO_BIC(BIC_UNCORE_MHZ)) - outp += sprintf(outp, "%sUncMHz", (printed++ ? delim : "")); +#define PMT_COUNTER_MTL_DC6_OFFSET 120 +#define PMT_COUNTER_MTL_DC6_LSB 0 +#define PMT_COUNTER_MTL_DC6_MSB 63 +#define PMT_MTL_DC6_GUID 0x1a067102 +#define PMT_MTL_DC6_SEQ 0 - for (mp = sys.pp; mp; mp = mp->next) { - if (mp->format == FORMAT_RAW) { - if (mp->width == 64) - outp += sprintf(outp, "%s%18.18s", delim, mp->name); - else - outp += sprintf(outp, "%s%10.10s", delim, mp->name); - } else { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", delim, mp->name); - else - outp += sprintf(outp, "%s%s", delim, mp->name); - } - } +#define PMT_COUNTER_CWF_MC1E_OFFSET_BASE 20936 +#define PMT_COUNTER_CWF_MC1E_OFFSET_INCREMENT 24 +#define PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE 12 +#define PMT_COUNTER_CWF_CPUS_PER_MODULE 4 +#define PMT_COUNTER_CWF_MC1E_LSB 0 +#define PMT_COUNTER_CWF_MC1E_MSB 63 +#define PMT_CWF_MC1E_GUID 0x14421519 - outp += sprintf(outp, "\n"); -} +unsigned long long tcore_clock_freq_hz = 800000000; -int dump_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) -{ - int i; - struct msr_counter *mp; +#define PMT_COUNTER_NAME_SIZE_BYTES 16 +#define PMT_COUNTER_TYPE_NAME_SIZE_BYTES 32 - outp += sprintf(outp, "t %p, c %p, p %p\n", t, c, p); +struct pmt_mmio { + struct pmt_mmio *next; - if (t) { - outp += sprintf(outp, "CPU: %d flags 0x%x\n", t->cpu_id, t->flags); - outp += sprintf(outp, "TSC: %016llX\n", t->tsc); - outp += sprintf(outp, "aperf: %016llX\n", t->aperf); - outp += sprintf(outp, "mperf: %016llX\n", t->mperf); - outp += sprintf(outp, "c1: %016llX\n", t->c1); + unsigned int guid; + unsigned int size; - if (DO_BIC(BIC_IPC)) - outp += sprintf(outp, "IPC: %lld\n", t->instr_count); + /* Base pointer to the mmaped memory. */ + void *mmio_base; - if (DO_BIC(BIC_IRQ)) - outp += sprintf(outp, "IRQ: %lld\n", t->irq_count); - if (DO_BIC(BIC_SMI)) - outp += sprintf(outp, "SMI: %d\n", t->smi_count); + /* + * Offset to be applied to the mmio_base + * to get the beginning of the PMT counters for given GUID. + */ + unsigned long pmt_offset; +} *pmt_mmios; - for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - outp += sprintf(outp, "tADDED [%d] msr0x%x: %08llX\n", i, mp->msr_num, t->counter[i]); - } - } +enum pmt_datatype { + PMT_TYPE_RAW, + PMT_TYPE_XTAL_TIME, + PMT_TYPE_TCORE_CLOCK, +}; - if (c) { - outp += sprintf(outp, "core: %d\n", c->core_id); - outp += sprintf(outp, "c3: %016llX\n", c->c3); - outp += sprintf(outp, "c6: %016llX\n", c->c6); - outp += sprintf(outp, "c7: %016llX\n", c->c7); - outp += sprintf(outp, "DTS: %dC\n", c->core_temp_c); - outp += sprintf(outp, "cpu_throt_count: %016llX\n", c->core_throt_cnt); - outp += sprintf(outp, "Joules: %0X\n", c->core_energy); +struct pmt_domain_info { + /* + * Pointer to the MMIO obtained by applying a counter offset + * to the mmio_base of the mmaped region for the given GUID. + * + * This is where to read the raw value of the counter from. + */ + unsigned long *pcounter; +}; - for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - outp += sprintf(outp, "cADDED [%d] msr0x%x: %08llX\n", i, mp->msr_num, c->counter[i]); - } - outp += sprintf(outp, "mc6_us: %016llX\n", c->mc6_us); - } +struct pmt_counter { + struct pmt_counter *next; - if (p) { - outp += sprintf(outp, "package: %d\n", p->package_id); + /* PMT metadata */ + char name[PMT_COUNTER_NAME_SIZE_BYTES]; + enum pmt_datatype type; + enum counter_scope scope; + unsigned int lsb; + unsigned int msb; - outp += sprintf(outp, "Weighted cores: %016llX\n", p->pkg_wtd_core_c0); - outp += sprintf(outp, "Any cores: %016llX\n", p->pkg_any_core_c0); - outp += sprintf(outp, "Any GFX: %016llX\n", p->pkg_any_gfxe_c0); - outp += sprintf(outp, "CPU + GFX: %016llX\n", p->pkg_both_core_gfxe_c0); + /* BIC-like metadata */ + enum counter_format format; - outp += sprintf(outp, "pc2: %016llX\n", p->pc2); - if (DO_BIC(BIC_Pkgpc3)) - outp += sprintf(outp, "pc3: %016llX\n", p->pc3); - if (DO_BIC(BIC_Pkgpc6)) - outp += sprintf(outp, "pc6: %016llX\n", p->pc6); - if (DO_BIC(BIC_Pkgpc7)) - outp += sprintf(outp, "pc7: %016llX\n", p->pc7); - outp += sprintf(outp, "pc8: %016llX\n", p->pc8); - outp += sprintf(outp, "pc9: %016llX\n", p->pc9); - outp += sprintf(outp, "pc10: %016llX\n", p->pc10); - outp += sprintf(outp, "cpu_lpi: %016llX\n", p->cpu_lpi); - outp += sprintf(outp, "sys_lpi: %016llX\n", p->sys_lpi); - outp += sprintf(outp, "Joules PKG: %0llX\n", p->energy_pkg); - outp += sprintf(outp, "Joules COR: %0llX\n", p->energy_cores); - outp += sprintf(outp, "Joules GFX: %0llX\n", p->energy_gfx); - outp += sprintf(outp, "Joules RAM: %0llX\n", p->energy_dram); - outp += sprintf(outp, "Throttle PKG: %0llX\n", p->rapl_pkg_perf_status); - outp += sprintf(outp, "Throttle RAM: %0llX\n", p->rapl_dram_perf_status); - outp += sprintf(outp, "PTM: %dC\n", p->pkg_temp_c); + unsigned int num_domains; + struct pmt_domain_info *domains; +}; - for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - outp += sprintf(outp, "pADDED [%d] msr0x%x: %08llX\n", i, mp->msr_num, p->counter[i]); - } - } +/* + * PMT telemetry directory iterator. + * Used to iterate telemetry files in sysfs in correct order. + */ +struct pmt_diriter_t { + DIR *dir; + struct dirent **namelist; + unsigned int num_names; + unsigned int current_name_idx; +}; - outp += sprintf(outp, "\n"); +int pmt_telemdir_filter(const struct dirent *e) +{ + unsigned int dummy; - return 0; + return sscanf(e->d_name, "telem%u", &dummy); } -/* - * column formatting convention & formats - */ -int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int pmt_telemdir_sort(const struct dirent **a, const struct dirent **b) { - double interval_float, tsc; - char *fmt8; - int i; - struct msr_counter *mp; - char *delim = "\t"; - int printed = 0; + unsigned int aidx = 0, bidx = 0; - /* if showing only 1st thread in core and this isn't one, bail out */ - if (show_core_only && !(t->flags & CPU_IS_FIRST_THREAD_IN_CORE)) - return 0; + sscanf((*a)->d_name, "telem%u", &aidx); + sscanf((*b)->d_name, "telem%u", &bidx); - /* if showing only 1st thread in pkg and this isn't one, bail out */ - if (show_pkg_only && !(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) - return 0; + return (aidx > bidx) ? 1 : (aidx < bidx) ? -1 : 0; +} - /*if not summary line and --cpu is used */ - if ((t != &average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset))) - return 0; +const struct dirent *pmt_diriter_next(struct pmt_diriter_t *iter) +{ + const struct dirent *ret = NULL; - if (DO_BIC(BIC_USEC)) { - /* on each row, print how many usec each timestamp took to gather */ - struct timeval tv; + if (!iter->dir) + return NULL; - timersub(&t->tv_end, &t->tv_begin, &tv); - outp += sprintf(outp, "%5ld\t", tv.tv_sec * 1000000 + tv.tv_usec); - } + if (iter->current_name_idx >= iter->num_names) + return NULL; - /* Time_Of_Day_Seconds: on each row, print sec.usec last timestamp taken */ - if (DO_BIC(BIC_TOD)) - outp += sprintf(outp, "%10ld.%06ld\t", t->tv_end.tv_sec, t->tv_end.tv_usec); + ret = iter->namelist[iter->current_name_idx]; + ++iter->current_name_idx; - interval_float = t->tv_delta.tv_sec + t->tv_delta.tv_usec / 1000000.0; + return ret; +} + +const struct dirent *pmt_diriter_begin(struct pmt_diriter_t *iter, const char *pmt_root_path) +{ + int num_names = iter->num_names; + + if (!iter->dir) { + iter->dir = opendir(pmt_root_path); + if (iter->dir == NULL) + return NULL; + + num_names = scandir(pmt_root_path, &iter->namelist, pmt_telemdir_filter, pmt_telemdir_sort); + if (num_names == -1) + return NULL; + } + + iter->current_name_idx = 0; + iter->num_names = num_names; + + return pmt_diriter_next(iter); +} + +void pmt_diriter_init(struct pmt_diriter_t *iter) +{ + memset(iter, 0, sizeof(*iter)); +} + +void pmt_diriter_remove(struct pmt_diriter_t *iter) +{ + if (iter->namelist) { + for (unsigned int i = 0; i < iter->num_names; i++) { + free(iter->namelist[i]); + iter->namelist[i] = NULL; + } + } + + free(iter->namelist); + iter->namelist = NULL; + iter->num_names = 0; + iter->current_name_idx = 0; + + closedir(iter->dir); + iter->dir = NULL; +} + +unsigned int pmt_counter_get_width(const struct pmt_counter *p) +{ + return (p->msb - p->lsb) + 1; +} + +void pmt_counter_resize_(struct pmt_counter *pcounter, unsigned int new_size) +{ + struct pmt_domain_info *new_mem; + + new_mem = (struct pmt_domain_info *)reallocarray(pcounter->domains, new_size, sizeof(*pcounter->domains)); + if (!new_mem) { + fprintf(stderr, "%s: failed to allocate memory for PMT counters\n", __func__); + exit(1); + } + + /* Zero initialize just allocated memory. */ + const size_t num_new_domains = new_size - pcounter->num_domains; + + memset(&new_mem[pcounter->num_domains], 0, num_new_domains * sizeof(*pcounter->domains)); + + pcounter->num_domains = new_size; + pcounter->domains = new_mem; +} + +void pmt_counter_resize(struct pmt_counter *pcounter, unsigned int new_size) +{ + /* + * Allocate more memory ahead of time. + * + * Always allocate space for at least 8 elements + * and double the size when growing. + */ + if (new_size < 8) + new_size = 8; + new_size = MAX(new_size, pcounter->num_domains * 2); + + pmt_counter_resize_(pcounter, new_size); +} + +struct llc_stats { + unsigned long long references; + unsigned long long misses; +}; +struct thread_data { + struct timeval tv_begin; + struct timeval tv_end; + struct timeval tv_delta; + unsigned long long tsc; + unsigned long long aperf; + unsigned long long mperf; + unsigned long long c1; + unsigned long long instr_count; + unsigned long long irq_count; + unsigned long long nmi_count; + unsigned int smi_count; + struct llc_stats llc; + unsigned int cpu_id; + unsigned int apic_id; + unsigned int x2apic_id; + unsigned int flags; + bool is_atom; + unsigned long long counter[MAX_ADDED_THREAD_COUNTERS]; + unsigned long long perf_counter[MAX_ADDED_THREAD_COUNTERS]; + unsigned long long pmt_counter[PMT_MAX_ADDED_THREAD_COUNTERS]; +} *thread_even, *thread_odd; + +struct core_data { + int base_cpu; + unsigned long long c3; + unsigned long long c6; + unsigned long long c7; + unsigned long long mc6_us; /* duplicate as per-core for now, even though per module */ + unsigned int core_temp_c; + struct rapl_counter core_energy; /* MSR_CORE_ENERGY_STAT */ + unsigned int core_id; + unsigned long long core_throt_cnt; + unsigned long long counter[MAX_ADDED_CORE_COUNTERS]; + unsigned long long perf_counter[MAX_ADDED_CORE_COUNTERS]; + unsigned long long pmt_counter[PMT_MAX_ADDED_CORE_COUNTERS]; +} *core_even, *core_odd; + +struct pkg_data { + int base_cpu; + unsigned long long pc2; + unsigned long long pc3; + unsigned long long pc6; + unsigned long long pc7; + unsigned long long pc8; + unsigned long long pc9; + unsigned long long pc10; + long long cpu_lpi; + long long sys_lpi; + unsigned long long pkg_wtd_core_c0; + unsigned long long pkg_any_core_c0; + unsigned long long pkg_any_gfxe_c0; + unsigned long long pkg_both_core_gfxe_c0; + long long gfx_rc6_ms; + unsigned int gfx_mhz; + unsigned int gfx_act_mhz; + long long sam_mc6_ms; + unsigned int sam_mhz; + unsigned int sam_act_mhz; + unsigned int package_id; + struct rapl_counter energy_pkg; /* MSR_PKG_ENERGY_STATUS */ + struct rapl_counter energy_dram; /* MSR_DRAM_ENERGY_STATUS */ + struct rapl_counter energy_cores; /* MSR_PP0_ENERGY_STATUS */ + struct rapl_counter energy_gfx; /* MSR_PP1_ENERGY_STATUS */ + struct rapl_counter rapl_pkg_perf_status; /* MSR_PKG_PERF_STATUS */ + struct rapl_counter rapl_dram_perf_status; /* MSR_DRAM_PERF_STATUS */ + unsigned int pkg_temp_c; + unsigned int uncore_mhz; + unsigned long long die_c6; + unsigned long long counter[MAX_ADDED_PACKAGE_COUNTERS]; + unsigned long long perf_counter[MAX_ADDED_PACKAGE_COUNTERS]; + unsigned long long pmt_counter[PMT_MAX_ADDED_PACKAGE_COUNTERS]; +} *package_even, *package_odd; + +#define ODD_COUNTERS thread_odd, core_odd, package_odd +#define EVEN_COUNTERS thread_even, core_even, package_even + +#define GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no) \ + ((thread_base) + \ + ((pkg_no) * \ + topo.nodes_per_pkg * topo.cores_per_node * topo.threads_per_core) + \ + ((node_no) * topo.cores_per_node * topo.threads_per_core) + \ + ((core_no) * topo.threads_per_core) + \ + (thread_no)) + +#define GET_CORE(core_base, core_no, node_no, pkg_no) \ + ((core_base) + \ + ((pkg_no) * topo.nodes_per_pkg * topo.cores_per_node) + \ + ((node_no) * topo.cores_per_node) + \ + (core_no)) + +/* + * The accumulated sum of MSR is defined as a monotonic + * increasing MSR, it will be accumulated periodically, + * despite its register's bit width. + */ +enum { + IDX_PKG_ENERGY, + IDX_DRAM_ENERGY, + IDX_PP0_ENERGY, + IDX_PP1_ENERGY, + IDX_PKG_PERF, + IDX_DRAM_PERF, + IDX_PSYS_ENERGY, + IDX_COUNT, +}; + +int get_msr_sum(int cpu, off_t offset, unsigned long long *msr); + +struct msr_sum_array { + /* get_msr_sum() = sum + (get_msr() - last) */ + struct { + /*The accumulated MSR value is updated by the timer */ + unsigned long long sum; + /*The MSR footprint recorded in last timer */ + unsigned long long last; + } entries[IDX_COUNT]; +}; + +/* The percpu MSR sum array.*/ +struct msr_sum_array *per_cpu_msr_sum; + +off_t idx_to_offset(int idx) +{ + off_t offset; + + switch (idx) { + case IDX_PKG_ENERGY: + if (platform->plat_rapl_msrs & RAPL_AMD_F17H) + offset = MSR_PKG_ENERGY_STAT; + else + offset = MSR_PKG_ENERGY_STATUS; + break; + case IDX_DRAM_ENERGY: + offset = MSR_DRAM_ENERGY_STATUS; + break; + case IDX_PP0_ENERGY: + offset = MSR_PP0_ENERGY_STATUS; + break; + case IDX_PP1_ENERGY: + offset = MSR_PP1_ENERGY_STATUS; + break; + case IDX_PKG_PERF: + offset = MSR_PKG_PERF_STATUS; + break; + case IDX_DRAM_PERF: + offset = MSR_DRAM_PERF_STATUS; + break; + case IDX_PSYS_ENERGY: + offset = MSR_PLATFORM_ENERGY_STATUS; + break; + default: + offset = -1; + } + return offset; +} + +int offset_to_idx(off_t offset) +{ + int idx; + + switch (offset) { + case MSR_PKG_ENERGY_STATUS: + case MSR_PKG_ENERGY_STAT: + idx = IDX_PKG_ENERGY; + break; + case MSR_DRAM_ENERGY_STATUS: + idx = IDX_DRAM_ENERGY; + break; + case MSR_PP0_ENERGY_STATUS: + idx = IDX_PP0_ENERGY; + break; + case MSR_PP1_ENERGY_STATUS: + idx = IDX_PP1_ENERGY; + break; + case MSR_PKG_PERF_STATUS: + idx = IDX_PKG_PERF; + break; + case MSR_DRAM_PERF_STATUS: + idx = IDX_DRAM_PERF; + break; + case MSR_PLATFORM_ENERGY_STATUS: + idx = IDX_PSYS_ENERGY; + break; + default: + idx = -1; + } + return idx; +} + +int idx_valid(int idx) +{ + switch (idx) { + case IDX_PKG_ENERGY: + return valid_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H); + case IDX_DRAM_ENERGY: + return valid_rapl_msrs & RAPL_DRAM; + case IDX_PP0_ENERGY: + return valid_rapl_msrs & RAPL_CORE_ENERGY_STATUS; + case IDX_PP1_ENERGY: + return valid_rapl_msrs & RAPL_GFX; + case IDX_PKG_PERF: + return valid_rapl_msrs & RAPL_PKG_PERF_STATUS; + case IDX_DRAM_PERF: + return valid_rapl_msrs & RAPL_DRAM_PERF_STATUS; + case IDX_PSYS_ENERGY: + return valid_rapl_msrs & RAPL_PSYS; + default: + return 0; + } +} + +struct sys_counters { + /* MSR added counters */ + unsigned int added_thread_counters; + unsigned int added_core_counters; + unsigned int added_package_counters; + struct msr_counter *tp; + struct msr_counter *cp; + struct msr_counter *pp; + + /* perf added counters */ + unsigned int added_thread_perf_counters; + unsigned int added_core_perf_counters; + unsigned int added_package_perf_counters; + struct perf_counter_info *perf_tp; + struct perf_counter_info *perf_cp; + struct perf_counter_info *perf_pp; + + struct pmt_counter *pmt_tp; + struct pmt_counter *pmt_cp; + struct pmt_counter *pmt_pp; +} sys; + +static size_t free_msr_counters_(struct msr_counter **pp) +{ + struct msr_counter *p = NULL; + size_t num_freed = 0; + + while (*pp) { + p = *pp; + + if (p->msr_num != 0) { + *pp = p->next; + + free(p); + ++num_freed; + + continue; + } + + pp = &p->next; + } + + return num_freed; +} + +/* + * Free all added counters accessed via msr. + */ +static void free_sys_msr_counters(void) +{ + /* Thread counters */ + sys.added_thread_counters -= free_msr_counters_(&sys.tp); + + /* Core counters */ + sys.added_core_counters -= free_msr_counters_(&sys.cp); + + /* Package counters */ + sys.added_package_counters -= free_msr_counters_(&sys.pp); +} + +struct system_summary { + struct thread_data threads; + struct core_data cores; + struct pkg_data packages; +} average; + +struct platform_counters { + struct rapl_counter energy_psys; /* MSR_PLATFORM_ENERGY_STATUS */ +} platform_counters_odd, platform_counters_even; + +struct cpu_topology { + int physical_package_id; + int die_id; + int l3_id; + int logical_cpu_id; + int physical_node_id; + int logical_node_id; /* 0-based count within the package */ + int physical_core_id; + int thread_id; + int type; + cpu_set_t *put_ids; /* Processing Unit/Thread IDs */ +} *cpus; + +struct topo_params { + int num_packages; + int num_die; + int num_cpus; + int num_cores; + int allowed_packages; + int allowed_cpus; + int allowed_cores; + int max_cpu_num; + int max_core_id; + int max_package_id; + int max_die_id; + int max_l3_id; + int max_node_num; + int nodes_per_pkg; + int cores_per_node; + int threads_per_core; +} topo; + +struct timeval tv_even, tv_odd, tv_delta; + +int *irq_column_2_cpu; /* /proc/interrupts column numbers */ +int *irqs_per_cpu; /* indexed by cpu_num */ +int *nmi_per_cpu; /* indexed by cpu_num */ + +void setup_all_buffers(bool startup); + +char *sys_lpi_file; +char *sys_lpi_file_sysfs = "/sys/devices/system/cpu/cpuidle/low_power_idle_system_residency_us"; +char *sys_lpi_file_debugfs = "/sys/kernel/debug/pmc_core/slp_s0_residency_usec"; + +int cpu_is_not_present(int cpu) +{ + return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set); +} + +int cpu_is_not_allowed(int cpu) +{ + return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set); +} + +/* + * run func(thread, core, package) in topology order + * skip non-present cpus + */ + +#define PER_THREAD_PARAMS struct thread_data *t, struct core_data *c, struct pkg_data *p + +int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pkg_data *), + struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base) +{ + int retval, pkg_no, core_no, thread_no, node_no; + + retval = 0; + + for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) { + for (node_no = 0; node_no < topo.nodes_per_pkg; node_no++) { + for (core_no = 0; core_no < topo.cores_per_node; ++core_no) { + for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) { + struct thread_data *t; + struct core_data *c; + + t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no); + + if (cpu_is_not_allowed(t->cpu_id)) + continue; + + c = GET_CORE(core_base, core_no, node_no, pkg_no); + + retval |= func(t, c, &pkg_base[pkg_no]); + } + } + } + } + return retval; +} + +int is_cpu_first_thread_in_core(struct thread_data *t, struct core_data *c) +{ + return ((int)t->cpu_id == c->base_cpu || c->base_cpu < 0); +} + +int is_cpu_first_core_in_package(struct thread_data *t, struct pkg_data *p) +{ + return ((int)t->cpu_id == p->base_cpu || p->base_cpu < 0); +} + +int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p) +{ + return is_cpu_first_thread_in_core(t, c) && is_cpu_first_core_in_package(t, p); +} + +int cpu_migrate(int cpu) +{ + CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set); + CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set); + if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1) + return -1; + else + return 0; +} + +int get_msr_fd(int cpu) +{ + char pathname[32]; + int fd; + + fd = fd_percpu[cpu]; + + if (fd) + return fd; + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu); + fd = open(pathname, O_RDONLY); + if (fd < 0) + err(-1, "%s open failed, try chown or chmod +r %s, " + "or run with --no-msr, or run as root", pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr"); + fd_percpu[cpu] = fd; + + return fd; +} + +static void bic_disable_msr_access(void) +{ + CLR_BIC(BIC_Mod_c6, &bic_enabled); + CLR_BIC(BIC_CoreTmp, &bic_enabled); + CLR_BIC(BIC_Totl_c0, &bic_enabled); + CLR_BIC(BIC_Any_c0, &bic_enabled); + CLR_BIC(BIC_GFX_c0, &bic_enabled); + CLR_BIC(BIC_CPUGFX, &bic_enabled); + CLR_BIC(BIC_PkgTmp, &bic_enabled); + + free_sys_msr_counters(); +} + +static void bic_disable_perf_access(void) +{ + CLR_BIC(BIC_IPC, &bic_enabled); + CLR_BIC(BIC_LLC_RPS, &bic_enabled); + CLR_BIC(BIC_LLC_HIT, &bic_enabled); +} + +static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags) +{ + assert(!no_perf); + + return syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags); +} + +static long open_perf_counter(int cpu, unsigned int type, unsigned int config, int group_fd, __u64 read_format) +{ + struct perf_event_attr attr; + const pid_t pid = -1; + const unsigned long flags = 0; + + assert(!no_perf); + + memset(&attr, 0, sizeof(struct perf_event_attr)); + + attr.type = type; + attr.size = sizeof(struct perf_event_attr); + attr.config = config; + attr.disabled = 0; + attr.sample_type = PERF_SAMPLE_IDENTIFIER; + attr.read_format = read_format; + + const int fd = perf_event_open(&attr, pid, cpu, group_fd, flags); + + return fd; +} + +int get_instr_count_fd(int cpu) +{ + if (fd_instr_count_percpu[cpu]) + return fd_instr_count_percpu[cpu]; + + fd_instr_count_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0); + + return fd_instr_count_percpu[cpu]; +} + +int get_msr(int cpu, off_t offset, unsigned long long *msr) +{ + ssize_t retval; + + assert(!no_msr); + + retval = pread(get_msr_fd(cpu), msr, sizeof(*msr), offset); + + if (retval != sizeof *msr) + err(-1, "cpu%d: msr offset 0x%llx read failed", cpu, (unsigned long long)offset); + + return 0; +} + +int add_msr_counter(int cpu, off_t offset) +{ + ssize_t retval; + unsigned long long value; + + if (no_msr) + return -1; + + if (!offset) + return -1; + + retval = pread(get_msr_fd(cpu), &value, sizeof(value), offset); + + /* if the read failed, the probe fails */ + if (retval != sizeof(value)) + return -1; + + if (value == 0) + return 0; + + return 1; +} + +int add_rapl_msr_counter(int cpu, const struct rapl_counter_arch_info *cai) +{ + int ret; + + if (!(valid_rapl_msrs & cai->feature_mask)) + return -1; + + ret = add_msr_counter(cpu, cai->msr); + if (ret < 0) + return -1; + + switch (cai->rci_index) { + case RAPL_RCI_INDEX_ENERGY_PKG: + case RAPL_RCI_INDEX_ENERGY_CORES: + case RAPL_RCI_INDEX_DRAM: + case RAPL_RCI_INDEX_GFX: + case RAPL_RCI_INDEX_ENERGY_PLATFORM: + if (ret == 0) + return 1; + } + + /* PKG,DRAM_PERF_STATUS MSRs, can return any value */ + return 1; +} + +/* Convert CPU ID to domain ID for given added perf counter. */ +unsigned int cpu_to_domain(const struct perf_counter_info *pc, int cpu) +{ + switch (pc->scope) { + case SCOPE_CPU: + return cpu; + + case SCOPE_CORE: + return cpus[cpu].physical_core_id; + + case SCOPE_PACKAGE: + return cpus[cpu].physical_package_id; + } + + __builtin_unreachable(); +} + +#define MAX_DEFERRED 16 +char *deferred_add_names[MAX_DEFERRED]; +char *deferred_skip_names[MAX_DEFERRED]; +int deferred_add_index; +int deferred_skip_index; +unsigned int deferred_add_consumed; +unsigned int deferred_skip_consumed; + +/* + * HIDE_LIST - hide this list of counters, show the rest [default] + * SHOW_LIST - show this list of counters, hide the rest + */ +enum show_hide_mode { SHOW_LIST, HIDE_LIST } global_show_hide_mode = HIDE_LIST; + +void help(void) +{ + fprintf(outf, + "Usage: turbostat [OPTIONS][(--interval seconds) | COMMAND ...]\n" + "\n" + "Turbostat forks the specified COMMAND and prints statistics\n" + "when COMMAND completes.\n" + "If no COMMAND is specified, turbostat wakes every 5-seconds\n" + "to print statistics, until interrupted.\n" + " -a, --add counter\n" + " add a counter\n" + " eg. --add msr0x10,u64,cpu,delta,MY_TSC\n" + " eg. --add perf/cstate_pkg/c2-residency,package,delta,percent,perfPC2\n" + " eg. --add pmt,name=XTAL,type=raw,domain=package0,offset=0,lsb=0,msb=63,guid=0x1a067102\n" + " -c, --cpu cpu-set\n" + " limit output to summary plus cpu-set:\n" + " {core | package | j,k,l..m,n-p }\n" + " -d, --debug\n" + " displays usec, Time_Of_Day_Seconds and more debugging\n" + " debug messages are printed to stderr\n" + " -D, --Dump\n" + " displays the raw counter values\n" + " -e, --enable [all | column]\n" + " shows all or the specified disabled column\n" + " -f, --force\n" + " force load turbostat with minimum default features on unsupported platforms.\n" + " -H, --hide [column | column,column,...]\n" + " hide the specified column(s)\n" + " -i, --interval sec.subsec\n" + " override default 5-second measurement interval\n" + " -J, --Joules\n" + " displays energy in Joules instead of Watts\n" + " -l, --list\n" + " list column headers only\n" + " -M, --no-msr\n" + " disable all uses of the MSR driver\n" + " -P, --no-perf\n" + " disable all uses of the perf API\n" + " -n, --num_iterations num\n" + " number of the measurement iterations\n" + " -N, --header_iterations num\n" + " print header every num iterations\n" + " -o, --out file\n" + " create or truncate \"file\" for all output\n" + " -q, --quiet\n" + " skip decoding system configuration header\n" + " -s, --show [column | column,column,...]\n" + " show only the specified column(s)\n" + " -S, --Summary\n" + " limits output to 1-line system summary per interval\n" + " -T, --TCC temperature\n" + " sets the Thermal Control Circuit temperature in\n" + " degrees Celsius\n" + " -h, --help\n" + " print this help message\n" + " -v, --version\n\t\tprint version information\n\nFor more help, run \"man turbostat\"\n"); +} + +/* + * bic_lookup + * for all the strings in comma separate name_list, + * set the approprate bit in return value. + */ +void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode) +{ + unsigned int i; + + while (name_list) { + char *comma; + + comma = strchr(name_list, ','); + + if (comma) + *comma = '\0'; + + for (i = 0; i < MAX_BIC; ++i) { + if (!strcmp(name_list, bic[i].name)) { + SET_BIC(i, ret_set); + break; + } + if (!strcmp(name_list, "all")) { + bic_set_all(ret_set); + break; + } else if (!strcmp(name_list, "topology")) { + CPU_OR(ret_set, ret_set, &bic_group_topology); + break; + } else if (!strcmp(name_list, "power")) { + CPU_OR(ret_set, ret_set, &bic_group_thermal_pwr); + break; + } else if (!strcmp(name_list, "idle")) { + CPU_OR(ret_set, ret_set, &bic_group_idle); + break; + } else if (!strcmp(name_list, "cache")) { + CPU_OR(ret_set, ret_set, &bic_group_cache); + break; + } else if (!strcmp(name_list, "llc")) { + CPU_OR(ret_set, ret_set, &bic_group_cache); + break; + } else if (!strcmp(name_list, "swidle")) { + CPU_OR(ret_set, ret_set, &bic_group_sw_idle); + break; + } else if (!strcmp(name_list, "sysfs")) { /* legacy compatibility */ + CPU_OR(ret_set, ret_set, &bic_group_sw_idle); + break; + } else if (!strcmp(name_list, "hwidle")) { + CPU_OR(ret_set, ret_set, &bic_group_hw_idle); + break; + } else if (!strcmp(name_list, "frequency")) { + CPU_OR(ret_set, ret_set, &bic_group_frequency); + break; + } else if (!strcmp(name_list, "other")) { + CPU_OR(ret_set, ret_set, &bic_group_other); + break; + } + } + if (i == MAX_BIC) { + if (mode == SHOW_LIST) { + deferred_add_names[deferred_add_index++] = name_list; + if (deferred_add_index >= MAX_DEFERRED) { + fprintf(stderr, "More than max %d un-recognized --add options '%s'\n", MAX_DEFERRED, name_list); + help(); + exit(1); + } + } else { + deferred_skip_names[deferred_skip_index++] = name_list; + if (debug) + fprintf(stderr, "deferred \"%s\"\n", name_list); + if (deferred_skip_index >= MAX_DEFERRED) { + fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n", MAX_DEFERRED, name_list); + help(); + exit(1); + } + } + } + + name_list = comma; + if (name_list) + name_list++; + + } +} + +/* + * print_name() + * Print column header name for raw 64-bit counter in 16 columns (at least 8-char plus a tab) + * Otherwise, allow the name + tab to fit within 8-coumn tab-stop. + * In both cases, left justififed, just like other turbostat columns, + * to allow the column values to consume the tab. + * + * Yes, 32-bit counters can overflow 8-columns, and + * 64-bit counters can overflow 16-columns, but that is uncommon. + */ +static inline int print_name(int width, int *printed, char *delim, char *name, enum counter_type type, enum counter_format format) +{ + UNUSED(type); + + if (format == FORMAT_RAW && width >= 64) + return (sprintf(outp, "%s%-8s", ((*printed)++ ? delim : ""), name)); + else + return (sprintf(outp, "%s%s", ((*printed)++ ? delim : ""), name)); +} + +static inline int print_hex_value(int width, int *printed, char *delim, unsigned long long value) +{ + if (width <= 32) + return (sprintf(outp, "%s%08x", ((*printed)++ ? delim : ""), (unsigned int)value)); + else + return (sprintf(outp, "%s%016llx", ((*printed)++ ? delim : ""), value)); +} + +static inline int print_decimal_value(int width, int *printed, char *delim, unsigned long long value) +{ + UNUSED(width); + + return (sprintf(outp, "%s%lld", ((*printed)++ ? delim : ""), value)); +} + +static inline int print_float_value(int *printed, char *delim, double value) +{ + return (sprintf(outp, "%s%0.2f", ((*printed)++ ? delim : ""), value)); +} + +void print_header(char *delim) +{ + struct msr_counter *mp; + struct perf_counter_info *pp; + struct pmt_counter *ppmt; + int printed = 0; + + if (DO_BIC(BIC_USEC)) + outp += sprintf(outp, "%susec", (printed++ ? delim : "")); + if (DO_BIC(BIC_TOD)) + outp += sprintf(outp, "%sTime_Of_Day_Seconds", (printed++ ? delim : "")); + if (DO_BIC(BIC_Package)) + outp += sprintf(outp, "%sPackage", (printed++ ? delim : "")); + if (DO_BIC(BIC_Die)) + outp += sprintf(outp, "%sDie", (printed++ ? delim : "")); + if (DO_BIC(BIC_L3)) + outp += sprintf(outp, "%sL3", (printed++ ? delim : "")); + if (DO_BIC(BIC_Node)) + outp += sprintf(outp, "%sNode", (printed++ ? delim : "")); + if (DO_BIC(BIC_Core)) + outp += sprintf(outp, "%sCore", (printed++ ? delim : "")); + if (DO_BIC(BIC_CPU)) + outp += sprintf(outp, "%sCPU", (printed++ ? delim : "")); + if (DO_BIC(BIC_APIC)) + outp += sprintf(outp, "%sAPIC", (printed++ ? delim : "")); + if (DO_BIC(BIC_X2APIC)) + outp += sprintf(outp, "%sX2APIC", (printed++ ? delim : "")); + if (DO_BIC(BIC_Avg_MHz)) + outp += sprintf(outp, "%sAvg_MHz", (printed++ ? delim : "")); + if (DO_BIC(BIC_Busy)) + outp += sprintf(outp, "%sBusy%%", (printed++ ? delim : "")); + if (DO_BIC(BIC_Bzy_MHz)) + outp += sprintf(outp, "%sBzy_MHz", (printed++ ? delim : "")); + if (DO_BIC(BIC_TSC_MHz)) + outp += sprintf(outp, "%sTSC_MHz", (printed++ ? delim : "")); + + if (DO_BIC(BIC_IPC)) + outp += sprintf(outp, "%sIPC", (printed++ ? delim : "")); + + if (DO_BIC(BIC_IRQ)) { + if (sums_need_wide_columns) + outp += sprintf(outp, "%s IRQ", (printed++ ? delim : "")); + else + outp += sprintf(outp, "%sIRQ", (printed++ ? delim : "")); + } + if (DO_BIC(BIC_NMI)) { + if (sums_need_wide_columns) + outp += sprintf(outp, "%s NMI", (printed++ ? delim : "")); + else + outp += sprintf(outp, "%sNMI", (printed++ ? delim : "")); + } + + if (DO_BIC(BIC_SMI)) + outp += sprintf(outp, "%sSMI", (printed++ ? delim : "")); + + if (DO_BIC(BIC_LLC_RPS)) + outp += sprintf(outp, "%sLLCkRPS", (printed++ ? delim : "")); + + if (DO_BIC(BIC_LLC_HIT)) + outp += sprintf(outp, "%sLLC%%hit", (printed++ ? delim : "")); + + for (mp = sys.tp; mp; mp = mp->next) + outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format); + + for (pp = sys.perf_tp; pp; pp = pp->next) + outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format); + + ppmt = sys.pmt_tp; + while (ppmt) { + switch (ppmt->type) { + case PMT_TYPE_RAW: + outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); + break; + + case PMT_TYPE_XTAL_TIME: + case PMT_TYPE_TCORE_CLOCK: + outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); + break; + } + + ppmt = ppmt->next; + } + + if (DO_BIC(BIC_CPU_c1)) + outp += sprintf(outp, "%sCPU%%c1", (printed++ ? delim : "")); + if (DO_BIC(BIC_CPU_c3)) + outp += sprintf(outp, "%sCPU%%c3", (printed++ ? delim : "")); + if (DO_BIC(BIC_CPU_c6)) + outp += sprintf(outp, "%sCPU%%c6", (printed++ ? delim : "")); + if (DO_BIC(BIC_CPU_c7)) + outp += sprintf(outp, "%sCPU%%c7", (printed++ ? delim : "")); + + if (DO_BIC(BIC_Mod_c6)) + outp += sprintf(outp, "%sMod%%c6", (printed++ ? delim : "")); + + if (DO_BIC(BIC_CoreTmp)) + outp += sprintf(outp, "%sCoreTmp", (printed++ ? delim : "")); + + if (DO_BIC(BIC_CORE_THROT_CNT)) + outp += sprintf(outp, "%sCoreThr", (printed++ ? delim : "")); + + if (valid_rapl_msrs && !rapl_joules) { + if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl) + outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : "")); + } else if (valid_rapl_msrs && rapl_joules) { + if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl) + outp += sprintf(outp, "%sCor_J", (printed++ ? delim : "")); + } + + for (mp = sys.cp; mp; mp = mp->next) + outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format); + + for (pp = sys.perf_cp; pp; pp = pp->next) + outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format); + + ppmt = sys.pmt_cp; + while (ppmt) { + switch (ppmt->type) { + case PMT_TYPE_RAW: + outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); + + break; + + case PMT_TYPE_XTAL_TIME: + case PMT_TYPE_TCORE_CLOCK: + outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); + break; + } + + ppmt = ppmt->next; + } + if (DO_BIC(BIC_PkgTmp)) + outp += sprintf(outp, "%sPkgTmp", (printed++ ? delim : "")); + + if (DO_BIC(BIC_GFX_rc6)) + outp += sprintf(outp, "%sGFX%%rc6", (printed++ ? delim : "")); + + if (DO_BIC(BIC_GFXMHz)) + outp += sprintf(outp, "%sGFXMHz", (printed++ ? delim : "")); + + if (DO_BIC(BIC_GFXACTMHz)) + outp += sprintf(outp, "%sGFXAMHz", (printed++ ? delim : "")); + + if (DO_BIC(BIC_SAM_mc6)) + outp += sprintf(outp, "%sSAM%%mc6", (printed++ ? delim : "")); + + if (DO_BIC(BIC_SAMMHz)) + outp += sprintf(outp, "%sSAMMHz", (printed++ ? delim : "")); + + if (DO_BIC(BIC_SAMACTMHz)) + outp += sprintf(outp, "%sSAMAMHz", (printed++ ? delim : "")); + + if (DO_BIC(BIC_Totl_c0)) + outp += sprintf(outp, "%sTotl%%C0", (printed++ ? delim : "")); + if (DO_BIC(BIC_Any_c0)) + outp += sprintf(outp, "%sAny%%C0", (printed++ ? delim : "")); + if (DO_BIC(BIC_GFX_c0)) + outp += sprintf(outp, "%sGFX%%C0", (printed++ ? delim : "")); + if (DO_BIC(BIC_CPUGFX)) + outp += sprintf(outp, "%sCPUGFX%%", (printed++ ? delim : "")); + + if (DO_BIC(BIC_Pkgpc2)) + outp += sprintf(outp, "%sPkg%%pc2", (printed++ ? delim : "")); + if (DO_BIC(BIC_Pkgpc3)) + outp += sprintf(outp, "%sPkg%%pc3", (printed++ ? delim : "")); + if (DO_BIC(BIC_Pkgpc6)) + outp += sprintf(outp, "%sPkg%%pc6", (printed++ ? delim : "")); + if (DO_BIC(BIC_Pkgpc7)) + outp += sprintf(outp, "%sPkg%%pc7", (printed++ ? delim : "")); + if (DO_BIC(BIC_Pkgpc8)) + outp += sprintf(outp, "%sPkg%%pc8", (printed++ ? delim : "")); + if (DO_BIC(BIC_Pkgpc9)) + outp += sprintf(outp, "%sPkg%%pc9", (printed++ ? delim : "")); + if (DO_BIC(BIC_Pkgpc10)) + outp += sprintf(outp, "%sPk%%pc10", (printed++ ? delim : "")); + if (DO_BIC(BIC_Diec6)) + outp += sprintf(outp, "%sDie%%c6", (printed++ ? delim : "")); + if (DO_BIC(BIC_CPU_LPI)) + outp += sprintf(outp, "%sCPU%%LPI", (printed++ ? delim : "")); + if (DO_BIC(BIC_SYS_LPI)) + outp += sprintf(outp, "%sSYS%%LPI", (printed++ ? delim : "")); + + if (!rapl_joules) { + if (DO_BIC(BIC_PkgWatt)) + outp += sprintf(outp, "%sPkgWatt", (printed++ ? delim : "")); + if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl) + outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : "")); + if (DO_BIC(BIC_GFXWatt)) + outp += sprintf(outp, "%sGFXWatt", (printed++ ? delim : "")); + if (DO_BIC(BIC_RAMWatt)) + outp += sprintf(outp, "%sRAMWatt", (printed++ ? delim : "")); + if (DO_BIC(BIC_PKG__)) + outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : "")); + if (DO_BIC(BIC_RAM__)) + outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : "")); + } else { + if (DO_BIC(BIC_Pkg_J)) + outp += sprintf(outp, "%sPkg_J", (printed++ ? delim : "")); + if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl) + outp += sprintf(outp, "%sCor_J", (printed++ ? delim : "")); + if (DO_BIC(BIC_GFX_J)) + outp += sprintf(outp, "%sGFX_J", (printed++ ? delim : "")); + if (DO_BIC(BIC_RAM_J)) + outp += sprintf(outp, "%sRAM_J", (printed++ ? delim : "")); + if (DO_BIC(BIC_PKG__)) + outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : "")); + if (DO_BIC(BIC_RAM__)) + outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : "")); + } + if (DO_BIC(BIC_UNCORE_MHZ)) + outp += sprintf(outp, "%sUncMHz", (printed++ ? delim : "")); + + for (mp = sys.pp; mp; mp = mp->next) + outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format); + + for (pp = sys.perf_pp; pp; pp = pp->next) + outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format); + + ppmt = sys.pmt_pp; + while (ppmt) { + switch (ppmt->type) { + case PMT_TYPE_RAW: + outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); + break; + + case PMT_TYPE_XTAL_TIME: + case PMT_TYPE_TCORE_CLOCK: + outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); + break; + } + + ppmt = ppmt->next; + } + + if (DO_BIC(BIC_SysWatt)) + outp += sprintf(outp, "%sSysWatt", (printed++ ? delim : "")); + if (DO_BIC(BIC_Sys_J)) + outp += sprintf(outp, "%sSys_J", (printed++ ? delim : "")); + + outp += sprintf(outp, "\n"); +} + +/* + * pct(numerator, denominator) + * + * Return sanity checked percentage (100.0 * numerator/denominotor) + * + * n < 0: nan + * d <= 0: nan + * n/d > 1.1: nan + */ +double pct(double numerator, double denominator) +{ + double retval; + + if (numerator < 0) + return nan(""); + + if (denominator <= 0) + return nan(""); + + retval = 100.0 * numerator / denominator; + + if (retval > 110.0) + return nan(""); + + return retval; +} + +int dump_counters(PER_THREAD_PARAMS) +{ + int i; + struct msr_counter *mp; + struct platform_counters *pplat_cnt = p == package_odd ? &platform_counters_odd : &platform_counters_even; + + outp += sprintf(outp, "t %p, c %p, p %p\n", t, c, p); + + if (t) { + outp += sprintf(outp, "CPU: %d flags 0x%x\n", t->cpu_id, t->flags); + outp += sprintf(outp, "TSC: %016llX\n", t->tsc); + outp += sprintf(outp, "aperf: %016llX\n", t->aperf); + outp += sprintf(outp, "mperf: %016llX\n", t->mperf); + outp += sprintf(outp, "c1: %016llX\n", t->c1); + + if (DO_BIC(BIC_IPC)) + outp += sprintf(outp, "IPC: %lld\n", t->instr_count); + + if (DO_BIC(BIC_IRQ)) + outp += sprintf(outp, "IRQ: %lld\n", t->irq_count); + if (DO_BIC(BIC_NMI)) + outp += sprintf(outp, "IRQ: %lld\n", t->nmi_count); + if (DO_BIC(BIC_SMI)) + outp += sprintf(outp, "SMI: %d\n", t->smi_count); + + outp += sprintf(outp, "LLC refs: %lld", t->llc.references); + outp += sprintf(outp, "LLC miss: %lld", t->llc.misses); + outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses), t->llc.references)); + + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { + outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path); + } + } + + if (c && is_cpu_first_thread_in_core(t, c)) { + outp += sprintf(outp, "core: %d\n", c->core_id); + outp += sprintf(outp, "c3: %016llX\n", c->c3); + outp += sprintf(outp, "c6: %016llX\n", c->c6); + outp += sprintf(outp, "c7: %016llX\n", c->c7); + outp += sprintf(outp, "DTS: %dC\n", c->core_temp_c); + outp += sprintf(outp, "cpu_throt_count: %016llX\n", c->core_throt_cnt); + + const unsigned long long energy_value = c->core_energy.raw_value * c->core_energy.scale; + const double energy_scale = c->core_energy.scale; + + if (c->core_energy.unit == RAPL_UNIT_JOULES) + outp += sprintf(outp, "Joules: %0llX (scale: %lf)\n", energy_value, energy_scale); + + for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { + outp += sprintf(outp, "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, c->counter[i], mp->sp->path); + } + outp += sprintf(outp, "mc6_us: %016llX\n", c->mc6_us); + } + + if (p && is_cpu_first_core_in_package(t, p)) { + outp += sprintf(outp, "package: %d\n", p->package_id); + + outp += sprintf(outp, "Weighted cores: %016llX\n", p->pkg_wtd_core_c0); + outp += sprintf(outp, "Any cores: %016llX\n", p->pkg_any_core_c0); + outp += sprintf(outp, "Any GFX: %016llX\n", p->pkg_any_gfxe_c0); + outp += sprintf(outp, "CPU + GFX: %016llX\n", p->pkg_both_core_gfxe_c0); + + outp += sprintf(outp, "pc2: %016llX\n", p->pc2); + if (DO_BIC(BIC_Pkgpc3)) + outp += sprintf(outp, "pc3: %016llX\n", p->pc3); + if (DO_BIC(BIC_Pkgpc6)) + outp += sprintf(outp, "pc6: %016llX\n", p->pc6); + if (DO_BIC(BIC_Pkgpc7)) + outp += sprintf(outp, "pc7: %016llX\n", p->pc7); + outp += sprintf(outp, "pc8: %016llX\n", p->pc8); + outp += sprintf(outp, "pc9: %016llX\n", p->pc9); + outp += sprintf(outp, "pc10: %016llX\n", p->pc10); + outp += sprintf(outp, "cpu_lpi: %016llX\n", p->cpu_lpi); + outp += sprintf(outp, "sys_lpi: %016llX\n", p->sys_lpi); + outp += sprintf(outp, "Joules PKG: %0llX\n", p->energy_pkg.raw_value); + outp += sprintf(outp, "Joules COR: %0llX\n", p->energy_cores.raw_value); + outp += sprintf(outp, "Joules GFX: %0llX\n", p->energy_gfx.raw_value); + outp += sprintf(outp, "Joules RAM: %0llX\n", p->energy_dram.raw_value); + outp += sprintf(outp, "Joules PSYS: %0llX\n", pplat_cnt->energy_psys.raw_value); + outp += sprintf(outp, "Throttle PKG: %0llX\n", p->rapl_pkg_perf_status.raw_value); + outp += sprintf(outp, "Throttle RAM: %0llX\n", p->rapl_dram_perf_status.raw_value); + outp += sprintf(outp, "PTM: %dC\n", p->pkg_temp_c); + + for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { + outp += sprintf(outp, "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, p->counter[i], mp->sp->path); + } + } + + outp += sprintf(outp, "\n"); + + return 0; +} + +double rapl_counter_get_value(const struct rapl_counter *c, enum rapl_unit desired_unit, double interval) +{ + assert(desired_unit != RAPL_UNIT_INVALID); + + /* + * For now we don't expect anything other than joules, + * so just simplify the logic. + */ + assert(c->unit == RAPL_UNIT_JOULES); + + const double scaled = c->raw_value * c->scale; + + if (desired_unit == RAPL_UNIT_WATTS) + return scaled / interval; + return scaled; +} + +void get_perf_llc_stats(int cpu, struct llc_stats *llc) +{ + struct read_format { + unsigned long long num_read; + struct llc_stats llc; + } r; + const ssize_t expected_read_size = sizeof(r); + ssize_t actual_read_size; + + actual_read_size = read(fd_llc_percpu[cpu], &r, expected_read_size); + + if (actual_read_size == -1) + err(-1, "%s(cpu%d,) %d,,%ld\n", __func__, cpu, fd_llc_percpu[cpu], expected_read_size); + + llc->references = r.llc.references; + llc->misses = r.llc.misses; + if (actual_read_size != expected_read_size) + warn("%s: failed to read perf_data (req %zu act %zu)", __func__, expected_read_size, actual_read_size); +} + +/* + * column formatting convention & formats + */ +int format_counters(PER_THREAD_PARAMS) +{ + static int count; + + struct platform_counters *pplat_cnt = NULL; + double interval_float, tsc; + char *fmt8 = "%s%.2f"; + + int i; + struct msr_counter *mp; + struct perf_counter_info *pp; + struct pmt_counter *ppmt; + char *delim = "\t"; + int printed = 0; + + if (t == &average.threads) { + pplat_cnt = count & 1 ? &platform_counters_odd : &platform_counters_even; + ++count; + } + + /* if showing only 1st thread in core and this isn't one, bail out */ + if (show_core_only && !is_cpu_first_thread_in_core(t, c)) + return 0; + + /* if showing only 1st thread in pkg and this isn't one, bail out */ + if (show_pkg_only && !is_cpu_first_core_in_package(t, p)) + return 0; + + /*if not summary line and --cpu is used */ + if ((t != &average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset))) + return 0; + + if (DO_BIC(BIC_USEC)) { + /* on each row, print how many usec each timestamp took to gather */ + struct timeval tv; + + timersub(&t->tv_end, &t->tv_begin, &tv); + outp += sprintf(outp, "%5ld\t", tv.tv_sec * 1000000 + tv.tv_usec); + } + + /* Time_Of_Day_Seconds: on each row, print sec.usec last timestamp taken */ + if (DO_BIC(BIC_TOD)) + outp += sprintf(outp, "%10ld.%06ld\t", t->tv_end.tv_sec, t->tv_end.tv_usec); + + interval_float = t->tv_delta.tv_sec + t->tv_delta.tv_usec / 1000000.0; + + tsc = t->tsc * tsc_tweak; + + /* topo columns, print blanks on 1st (average) line */ + if (t == &average.threads) { + if (DO_BIC(BIC_Package)) + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + if (DO_BIC(BIC_Die)) + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + if (DO_BIC(BIC_L3)) + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + if (DO_BIC(BIC_Node)) + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + if (DO_BIC(BIC_Core)) + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + if (DO_BIC(BIC_CPU)) + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + if (DO_BIC(BIC_APIC)) + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + if (DO_BIC(BIC_X2APIC)) + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + } else { + if (DO_BIC(BIC_Package)) { + if (p) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->package_id); + else + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + } + if (DO_BIC(BIC_Die)) { + if (c) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].die_id); + else + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + } + if (DO_BIC(BIC_L3)) { + if (c) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].l3_id); + else + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + } + if (DO_BIC(BIC_Node)) { + if (t) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id); + else + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + } + if (DO_BIC(BIC_Core)) { + if (c) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_id); + else + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + } + if (DO_BIC(BIC_CPU)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->cpu_id); + if (DO_BIC(BIC_APIC)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->apic_id); + if (DO_BIC(BIC_X2APIC)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->x2apic_id); + } + + if (DO_BIC(BIC_Avg_MHz)) + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float); + + if (DO_BIC(BIC_Busy)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf, tsc)); + + if (DO_BIC(BIC_Bzy_MHz)) { + if (has_base_hz) + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf); + else + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), tsc / units * t->aperf / t->mperf / interval_float); + } + + if (DO_BIC(BIC_TSC_MHz)) + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 * t->tsc / units / interval_float); + + if (DO_BIC(BIC_IPC)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 1.0 * t->instr_count / t->aperf); + + /* IRQ */ + if (DO_BIC(BIC_IRQ)) { + if (sums_need_wide_columns) + outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->irq_count); + else + outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->irq_count); + } + + /* NMI */ + if (DO_BIC(BIC_NMI)) { + if (sums_need_wide_columns) + outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->nmi_count); + else + outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->nmi_count); + } + + /* SMI */ + if (DO_BIC(BIC_SMI)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->smi_count); + + /* LLC Stats */ + if (DO_BIC(BIC_LLC_RPS) || DO_BIC(BIC_LLC_HIT)) { + if (DO_BIC(BIC_LLC_RPS)) + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000); + + if (DO_BIC(BIC_LLC_HIT)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses), t->llc.references)); + } + + /* Added Thread Counters */ + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW) + outp += print_hex_value(mp->width, &printed, delim, t->counter[i]); + else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(mp->width, &printed, delim, t->counter[i]); + else if (mp->format == FORMAT_PERCENT) { + if (mp->type == COUNTER_USEC) + outp += print_float_value(&printed, delim, t->counter[i] / interval_float / 10000); + else + outp += print_float_value(&printed, delim, pct(t->counter[i], tsc)); + } + } + + /* Added perf Thread Counters */ + for (i = 0, pp = sys.perf_tp; pp; ++i, pp = pp->next) { + if (pp->format == FORMAT_RAW) + outp += print_hex_value(pp->width, &printed, delim, t->perf_counter[i]); + else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE) + outp += print_decimal_value(pp->width, &printed, delim, t->perf_counter[i]); + else if (pp->format == FORMAT_PERCENT) { + if (pp->type == COUNTER_USEC) + outp += print_float_value(&printed, delim, t->perf_counter[i] / interval_float / 10000); + else + outp += print_float_value(&printed, delim, pct(t->perf_counter[i], tsc)); + } + } + + /* Added PMT Thread Counters */ + for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) { + const unsigned long value_raw = t->pmt_counter[i]; + double value_converted; + switch (ppmt->type) { + case PMT_TYPE_RAW: + outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, t->pmt_counter[i]); + break; + + case PMT_TYPE_XTAL_TIME: + value_converted = pct(value_raw / crystal_hz, interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + break; + + case PMT_TYPE_TCORE_CLOCK: + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + } + } + + /* C1 */ + if (DO_BIC(BIC_CPU_c1)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1, tsc)); + + /* print per-core data only for 1st thread in core */ + if (!is_cpu_first_thread_in_core(t, c)) + goto done; + + if (DO_BIC(BIC_CPU_c3)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3, tsc)); + if (DO_BIC(BIC_CPU_c6)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6, tsc)); + if (DO_BIC(BIC_CPU_c7)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7, tsc)); + + /* Mod%c6 */ + if (DO_BIC(BIC_Mod_c6)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us, tsc)); + + if (DO_BIC(BIC_CoreTmp)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c); + + /* Core throttle count */ + if (DO_BIC(BIC_CORE_THROT_CNT)) + outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->core_throt_cnt); + + /* Added Core Counters */ + for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW) + outp += print_hex_value(mp->width, &printed, delim, c->counter[i]); + else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(mp->width, &printed, delim, c->counter[i]); + else if (mp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(c->counter[i], tsc)); + } + + /* Added perf Core counters */ + for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { + if (pp->format == FORMAT_RAW) + outp += print_hex_value(pp->width, &printed, delim, c->perf_counter[i]); + else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE) + outp += print_decimal_value(pp->width, &printed, delim, c->perf_counter[i]); + else if (pp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(c->perf_counter[i], tsc)); + } + + /* Added PMT Core counters */ + for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) { + const unsigned long value_raw = c->pmt_counter[i]; + double value_converted; + switch (ppmt->type) { + case PMT_TYPE_RAW: + outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, c->pmt_counter[i]); + break; + + case PMT_TYPE_XTAL_TIME: + value_converted = pct(value_raw / crystal_hz, interval_float); + outp += print_float_value(&printed, delim, value_converted); + break; + + case PMT_TYPE_TCORE_CLOCK: + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); + outp += print_float_value(&printed, delim, value_converted); + } + } + + if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_WATTS, interval_float)); + if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_JOULES, interval_float)); + + /* print per-package data only for 1st core in package */ + if (!is_cpu_first_core_in_package(t, p)) + goto done; + + /* PkgTmp */ + if (DO_BIC(BIC_PkgTmp)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->pkg_temp_c); + + /* GFXrc6 */ + if (DO_BIC(BIC_GFX_rc6)) { + if (p->gfx_rc6_ms == -1) { /* detect GFX counter reset */ + outp += sprintf(outp, "%s**.**", (printed++ ? delim : "")); + } else { + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->gfx_rc6_ms / 10.0 / interval_float); + } + } + + /* GFXMHz */ + if (DO_BIC(BIC_GFXMHz)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_mhz); + + /* GFXACTMHz */ + if (DO_BIC(BIC_GFXACTMHz)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_act_mhz); + + /* SAMmc6 */ + if (DO_BIC(BIC_SAM_mc6)) { + if (p->sam_mc6_ms == -1) { /* detect GFX counter reset */ + outp += sprintf(outp, "%s**.**", (printed++ ? delim : "")); + } else { + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->sam_mc6_ms / 10.0 / interval_float); + } + } + + /* SAMMHz */ + if (DO_BIC(BIC_SAMMHz)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->sam_mhz); + + /* SAMACTMHz */ + if (DO_BIC(BIC_SAMACTMHz)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->sam_act_mhz); + + /* Totl%C0, Any%C0 GFX%C0 CPUGFX% */ + if (DO_BIC(BIC_Totl_c0)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc); /* can exceed 100% */ + if (DO_BIC(BIC_Any_c0)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0, tsc)); + if (DO_BIC(BIC_GFX_c0)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0, tsc)); + if (DO_BIC(BIC_CPUGFX)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0, tsc)); + + if (DO_BIC(BIC_Pkgpc2)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2, tsc)); + if (DO_BIC(BIC_Pkgpc3)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3, tsc)); + if (DO_BIC(BIC_Pkgpc6)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6, tsc)); + if (DO_BIC(BIC_Pkgpc7)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7, tsc)); + if (DO_BIC(BIC_Pkgpc8)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8, tsc)); + if (DO_BIC(BIC_Pkgpc9)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9, tsc)); + if (DO_BIC(BIC_Pkgpc10)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10, tsc)); + + if (DO_BIC(BIC_Diec6)) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz, interval_float)); + + if (DO_BIC(BIC_CPU_LPI)) { + if (p->cpu_lpi >= 0) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0, interval_float)); + else + outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); + } + if (DO_BIC(BIC_SYS_LPI)) { + if (p->sys_lpi >= 0) + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0, interval_float)); + else + outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); + } + + if (DO_BIC(BIC_PkgWatt)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_WATTS, interval_float)); + if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_WATTS, interval_float)); + if (DO_BIC(BIC_GFXWatt)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_WATTS, interval_float)); + if (DO_BIC(BIC_RAMWatt)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_WATTS, interval_float)); + if (DO_BIC(BIC_Pkg_J)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_JOULES, interval_float)); + if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_JOULES, interval_float)); + if (DO_BIC(BIC_GFX_J)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_JOULES, interval_float)); + if (DO_BIC(BIC_RAM_J)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_JOULES, interval_float)); + if (DO_BIC(BIC_PKG__)) + outp += + sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float)); + if (DO_BIC(BIC_RAM__)) + outp += + sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float)); + /* UncMHz */ + if (DO_BIC(BIC_UNCORE_MHZ)) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->uncore_mhz); + + /* Added Package Counters */ + for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW) + outp += print_hex_value(mp->width, &printed, delim, p->counter[i]); + else if (mp->type == COUNTER_K2M) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->counter[i] / 1000); + else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(mp->width, &printed, delim, p->counter[i]); + else if (mp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(p->counter[i], tsc)); + } + + /* Added perf Package Counters */ + for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) { + if (pp->format == FORMAT_RAW) + outp += print_hex_value(pp->width, &printed, delim, p->perf_counter[i]); + else if (pp->type == COUNTER_K2M) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000); + else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE) + outp += print_decimal_value(pp->width, &printed, delim, p->perf_counter[i]); + else if (pp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(p->perf_counter[i], tsc)); + } + + /* Added PMT Package Counters */ + for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) { + const unsigned long value_raw = p->pmt_counter[i]; + double value_converted; + switch (ppmt->type) { + case PMT_TYPE_RAW: + outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, p->pmt_counter[i]); + break; + + case PMT_TYPE_XTAL_TIME: + value_converted = pct(value_raw / crystal_hz, interval_float); + outp += print_float_value(&printed, delim, value_converted); + break; + + case PMT_TYPE_TCORE_CLOCK: + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); + outp += print_float_value(&printed, delim, value_converted); + } + } + + if (DO_BIC(BIC_SysWatt) && (t == &average.threads)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), + rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_WATTS, interval_float)); + if (DO_BIC(BIC_Sys_J) && (t == &average.threads)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), + rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_JOULES, interval_float)); + +done: + if (*(outp - 1) != '\n') + outp += sprintf(outp, "\n"); + + return 0; +} + +void flush_output_stdout(void) +{ + FILE *filep; + + if (outf == stderr) + filep = stdout; + else + filep = outf; + + fputs(output_buffer, filep); + fflush(filep); + + outp = output_buffer; +} + +void flush_output_stderr(void) +{ + fputs(output_buffer, outf); + fflush(outf); + outp = output_buffer; +} + +void format_all_counters(PER_THREAD_PARAMS) +{ + static int count; + + if ((!count || (header_iterations && !(count % header_iterations))) || !summary_only) + print_header("\t"); + + format_counters(&average.threads, &average.cores, &average.packages); + + count++; + + if (summary_only) + return; + + for_all_cpus(format_counters, t, c, p); +} + +#define DELTA_WRAP32(new, old) \ + old = ((((unsigned long long)new << 32) - ((unsigned long long)old << 32)) >> 32); + +int delta_package(struct pkg_data *new, struct pkg_data *old) +{ + int i; + struct msr_counter *mp; + struct perf_counter_info *pp; + struct pmt_counter *ppmt; + + if (DO_BIC(BIC_Totl_c0)) + old->pkg_wtd_core_c0 = new->pkg_wtd_core_c0 - old->pkg_wtd_core_c0; + if (DO_BIC(BIC_Any_c0)) + old->pkg_any_core_c0 = new->pkg_any_core_c0 - old->pkg_any_core_c0; + if (DO_BIC(BIC_GFX_c0)) + old->pkg_any_gfxe_c0 = new->pkg_any_gfxe_c0 - old->pkg_any_gfxe_c0; + if (DO_BIC(BIC_CPUGFX)) + old->pkg_both_core_gfxe_c0 = new->pkg_both_core_gfxe_c0 - old->pkg_both_core_gfxe_c0; + + old->pc2 = new->pc2 - old->pc2; + if (DO_BIC(BIC_Pkgpc3)) + old->pc3 = new->pc3 - old->pc3; + if (DO_BIC(BIC_Pkgpc6)) + old->pc6 = new->pc6 - old->pc6; + if (DO_BIC(BIC_Pkgpc7)) + old->pc7 = new->pc7 - old->pc7; + old->pc8 = new->pc8 - old->pc8; + old->pc9 = new->pc9 - old->pc9; + old->pc10 = new->pc10 - old->pc10; + old->die_c6 = new->die_c6 - old->die_c6; + old->cpu_lpi = new->cpu_lpi - old->cpu_lpi; + old->sys_lpi = new->sys_lpi - old->sys_lpi; + old->pkg_temp_c = new->pkg_temp_c; + + /* flag an error when rc6 counter resets/wraps */ + if (old->gfx_rc6_ms > new->gfx_rc6_ms) + old->gfx_rc6_ms = -1; + else + old->gfx_rc6_ms = new->gfx_rc6_ms - old->gfx_rc6_ms; + + old->uncore_mhz = new->uncore_mhz; + old->gfx_mhz = new->gfx_mhz; + old->gfx_act_mhz = new->gfx_act_mhz; + + /* flag an error when mc6 counter resets/wraps */ + if (old->sam_mc6_ms > new->sam_mc6_ms) + old->sam_mc6_ms = -1; + else + old->sam_mc6_ms = new->sam_mc6_ms - old->sam_mc6_ms; + + old->sam_mhz = new->sam_mhz; + old->sam_act_mhz = new->sam_act_mhz; + + old->energy_pkg.raw_value = new->energy_pkg.raw_value - old->energy_pkg.raw_value; + old->energy_cores.raw_value = new->energy_cores.raw_value - old->energy_cores.raw_value; + old->energy_gfx.raw_value = new->energy_gfx.raw_value - old->energy_gfx.raw_value; + old->energy_dram.raw_value = new->energy_dram.raw_value - old->energy_dram.raw_value; + old->rapl_pkg_perf_status.raw_value = new->rapl_pkg_perf_status.raw_value - old->rapl_pkg_perf_status.raw_value; + old->rapl_dram_perf_status.raw_value = new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value; + + for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW) + old->counter[i] = new->counter[i]; + else if (mp->format == FORMAT_AVERAGE) + old->counter[i] = new->counter[i]; + else + old->counter[i] = new->counter[i] - old->counter[i]; + } + + for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) { + if (pp->format == FORMAT_RAW) + old->perf_counter[i] = new->perf_counter[i]; + else if (pp->format == FORMAT_AVERAGE) + old->perf_counter[i] = new->perf_counter[i]; + else + old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i]; + } + + for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) { + if (ppmt->format == FORMAT_RAW) + old->pmt_counter[i] = new->pmt_counter[i]; + else + old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i]; + } + + return 0; +} + +void delta_core(struct core_data *new, struct core_data *old) +{ + int i; + struct msr_counter *mp; + struct perf_counter_info *pp; + struct pmt_counter *ppmt; + + old->c3 = new->c3 - old->c3; + old->c6 = new->c6 - old->c6; + old->c7 = new->c7 - old->c7; + old->core_temp_c = new->core_temp_c; + old->core_throt_cnt = new->core_throt_cnt - old->core_throt_cnt; + old->mc6_us = new->mc6_us - old->mc6_us; + + DELTA_WRAP32(new->core_energy.raw_value, old->core_energy.raw_value); + + for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) + old->counter[i] = new->counter[i]; + else + old->counter[i] = new->counter[i] - old->counter[i]; + } + + for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { + if (pp->format == FORMAT_RAW) + old->perf_counter[i] = new->perf_counter[i]; + else + old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i]; + } + + for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) { + if (ppmt->format == FORMAT_RAW) + old->pmt_counter[i] = new->pmt_counter[i]; + else + old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i]; + } +} + +int soft_c1_residency_display(int bic) +{ + if (!DO_BIC(BIC_CPU_c1) || platform->has_msr_core_c1_res) + return 0; + + return DO_BIC_READ(bic); +} + +/* + * old = new - old + */ +int delta_thread(struct thread_data *new, struct thread_data *old, struct core_data *core_delta) +{ + int i; + struct msr_counter *mp; + struct perf_counter_info *pp; + struct pmt_counter *ppmt; + + /* we run cpuid just the 1st time, copy the results */ + if (DO_BIC(BIC_APIC)) + new->apic_id = old->apic_id; + if (DO_BIC(BIC_X2APIC)) + new->x2apic_id = old->x2apic_id; + + /* + * the timestamps from start of measurement interval are in "old" + * the timestamp from end of measurement interval are in "new" + * over-write old w/ new so we can print end of interval values + */ + + timersub(&new->tv_begin, &old->tv_begin, &old->tv_delta); + old->tv_begin = new->tv_begin; + old->tv_end = new->tv_end; + + old->tsc = new->tsc - old->tsc; + + /* check for TSC < 1 Mcycles over interval */ + if (old->tsc < (1000 * 1000)) + errx(-3, "Insanely slow TSC rate, TSC stops in idle?\n" + "You can disable all c-states by booting with \"idle=poll\"\n" "or just the deep ones with \"processor.max_cstate=1\""); + + old->c1 = new->c1 - old->c1; + + if (DO_BIC(BIC_Avg_MHz) || DO_BIC(BIC_Busy) || DO_BIC(BIC_Bzy_MHz) || DO_BIC(BIC_IPC) + || soft_c1_residency_display(BIC_Avg_MHz)) { + if ((new->aperf > old->aperf) && (new->mperf > old->mperf)) { + old->aperf = new->aperf - old->aperf; + old->mperf = new->mperf - old->mperf; + } else { + return -1; + } + } + + if (platform->has_msr_core_c1_res) { + /* + * Some models have a dedicated C1 residency MSR, + * which should be more accurate than the derivation below. + */ + } else { + /* + * As counter collection is not atomic, + * it is possible for mperf's non-halted cycles + idle states + * to exceed TSC's all cycles: show c1 = 0% in that case. + */ + if ((old->mperf + core_delta->c3 + core_delta->c6 + core_delta->c7) > (old->tsc * tsc_tweak)) + old->c1 = 0; + else { + /* normal case, derive c1 */ + old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3 - core_delta->c6 - core_delta->c7; + } + } + + if (old->mperf == 0) { + if (debug > 1) + fprintf(outf, "cpu%d MPERF 0!\n", old->cpu_id); + old->mperf = 1; /* divide by 0 protection */ + } + + if (DO_BIC(BIC_IPC)) + old->instr_count = new->instr_count - old->instr_count; + + if (DO_BIC(BIC_IRQ)) + old->irq_count = new->irq_count - old->irq_count; + + if (DO_BIC(BIC_NMI)) + old->nmi_count = new->nmi_count - old->nmi_count; + + if (DO_BIC(BIC_SMI)) + old->smi_count = new->smi_count - old->smi_count; + + if (DO_BIC(BIC_LLC_RPS)) + old->llc.references = new->llc.references - old->llc.references; + + if (DO_BIC(BIC_LLC_HIT)) + old->llc.misses = new->llc.misses - old->llc.misses; + + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) + old->counter[i] = new->counter[i]; + else + old->counter[i] = new->counter[i] - old->counter[i]; + } + + for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) { + if (pp->format == FORMAT_RAW) + old->perf_counter[i] = new->perf_counter[i]; + else + old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i]; + } + + for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) { + if (ppmt->format == FORMAT_RAW) + old->pmt_counter[i] = new->pmt_counter[i]; + else + old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i]; + } + + return 0; +} + +int delta_cpu(struct thread_data *t, struct core_data *c, struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2) +{ + int retval = 0; + + /* calculate core delta only for 1st thread in core */ + if (is_cpu_first_thread_in_core(t, c)) + delta_core(c, c2); + + /* always calculate thread delta */ + retval = delta_thread(t, t2, c2); /* c2 is core delta */ + + /* calculate package delta only for 1st core in package */ + if (is_cpu_first_core_in_package(t, p)) + retval |= delta_package(p, p2); + + return retval; +} + +void delta_platform(struct platform_counters *new, struct platform_counters *old) +{ + old->energy_psys.raw_value = new->energy_psys.raw_value - old->energy_psys.raw_value; +} + +void rapl_counter_clear(struct rapl_counter *c) +{ + c->raw_value = 0; + c->scale = 0.0; + c->unit = RAPL_UNIT_INVALID; +} + +void clear_counters(PER_THREAD_PARAMS) +{ + int i; + struct msr_counter *mp; + + t->tv_begin.tv_sec = 0; + t->tv_begin.tv_usec = 0; + t->tv_end.tv_sec = 0; + t->tv_end.tv_usec = 0; + t->tv_delta.tv_sec = 0; + t->tv_delta.tv_usec = 0; + + t->tsc = 0; + t->aperf = 0; + t->mperf = 0; + t->c1 = 0; + + t->instr_count = 0; + + t->irq_count = 0; + t->nmi_count = 0; + t->smi_count = 0; + + t->llc.references = 0; + t->llc.misses = 0; + + c->c3 = 0; + c->c6 = 0; + c->c7 = 0; + c->mc6_us = 0; + c->core_temp_c = 0; + rapl_counter_clear(&c->core_energy); + c->core_throt_cnt = 0; + + t->llc.references = 0; + t->llc.misses = 0; + + p->pkg_wtd_core_c0 = 0; + p->pkg_any_core_c0 = 0; + p->pkg_any_gfxe_c0 = 0; + p->pkg_both_core_gfxe_c0 = 0; + + p->pc2 = 0; + if (DO_BIC(BIC_Pkgpc3)) + p->pc3 = 0; + if (DO_BIC(BIC_Pkgpc6)) + p->pc6 = 0; + if (DO_BIC(BIC_Pkgpc7)) + p->pc7 = 0; + p->pc8 = 0; + p->pc9 = 0; + p->pc10 = 0; + p->die_c6 = 0; + p->cpu_lpi = 0; + p->sys_lpi = 0; + + rapl_counter_clear(&p->energy_pkg); + rapl_counter_clear(&p->energy_dram); + rapl_counter_clear(&p->energy_cores); + rapl_counter_clear(&p->energy_gfx); + rapl_counter_clear(&p->rapl_pkg_perf_status); + rapl_counter_clear(&p->rapl_dram_perf_status); + p->pkg_temp_c = 0; + + p->gfx_rc6_ms = 0; + p->uncore_mhz = 0; + p->gfx_mhz = 0; + p->gfx_act_mhz = 0; + p->sam_mc6_ms = 0; + p->sam_mhz = 0; + p->sam_act_mhz = 0; + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) + t->counter[i] = 0; + + for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) + c->counter[i] = 0; + + for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) + p->counter[i] = 0; + + memset(&t->perf_counter[0], 0, sizeof(t->perf_counter)); + memset(&c->perf_counter[0], 0, sizeof(c->perf_counter)); + memset(&p->perf_counter[0], 0, sizeof(p->perf_counter)); + + memset(&t->pmt_counter[0], 0, ARRAY_SIZE(t->pmt_counter)); + memset(&c->pmt_counter[0], 0, ARRAY_SIZE(c->pmt_counter)); + memset(&p->pmt_counter[0], 0, ARRAY_SIZE(p->pmt_counter)); +} + +void rapl_counter_accumulate(struct rapl_counter *dst, const struct rapl_counter *src) +{ + /* Copy unit and scale from src if dst is not initialized */ + if (dst->unit == RAPL_UNIT_INVALID) { + dst->unit = src->unit; + dst->scale = src->scale; + } + + assert(dst->unit == src->unit); + assert(dst->scale == src->scale); + + dst->raw_value += src->raw_value; +} + +int sum_counters(PER_THREAD_PARAMS) +{ + int i; + struct msr_counter *mp; + struct perf_counter_info *pp; + struct pmt_counter *ppmt; + + /* copy un-changing apic_id's */ + if (DO_BIC(BIC_APIC)) + average.threads.apic_id = t->apic_id; + if (DO_BIC(BIC_X2APIC)) + average.threads.x2apic_id = t->x2apic_id; + + /* remember first tv_begin */ + if (average.threads.tv_begin.tv_sec == 0) + average.threads.tv_begin = procsysfs_tv_begin; + + /* remember last tv_end */ + average.threads.tv_end = t->tv_end; + + average.threads.tsc += t->tsc; + average.threads.aperf += t->aperf; + average.threads.mperf += t->mperf; + average.threads.c1 += t->c1; + + average.threads.instr_count += t->instr_count; + + average.threads.irq_count += t->irq_count; + average.threads.nmi_count += t->nmi_count; + average.threads.smi_count += t->smi_count; + + average.threads.llc.references += t->llc.references; + average.threads.llc.misses += t->llc.misses; + + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW) + continue; + average.threads.counter[i] += t->counter[i]; + } + + for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) { + if (pp->format == FORMAT_RAW) + continue; + average.threads.perf_counter[i] += t->perf_counter[i]; + } + + for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) { + average.threads.pmt_counter[i] += t->pmt_counter[i]; + } + + /* sum per-core values only for 1st thread in core */ + if (!is_cpu_first_thread_in_core(t, c)) + return 0; + + average.cores.c3 += c->c3; + average.cores.c6 += c->c6; + average.cores.c7 += c->c7; + average.cores.mc6_us += c->mc6_us; + + average.cores.core_temp_c = MAX(average.cores.core_temp_c, c->core_temp_c); + average.cores.core_throt_cnt = MAX(average.cores.core_throt_cnt, c->core_throt_cnt); + + rapl_counter_accumulate(&average.cores.core_energy, &c->core_energy); + + for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW) + continue; + average.cores.counter[i] += c->counter[i]; + } + + for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { + if (pp->format == FORMAT_RAW) + continue; + average.cores.perf_counter[i] += c->perf_counter[i]; + } + + for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) { + average.cores.pmt_counter[i] += c->pmt_counter[i]; + } + + /* sum per-pkg values only for 1st core in pkg */ + if (!is_cpu_first_core_in_package(t, p)) + return 0; + + if (DO_BIC(BIC_Totl_c0)) + average.packages.pkg_wtd_core_c0 += p->pkg_wtd_core_c0; + if (DO_BIC(BIC_Any_c0)) + average.packages.pkg_any_core_c0 += p->pkg_any_core_c0; + if (DO_BIC(BIC_GFX_c0)) + average.packages.pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0; + if (DO_BIC(BIC_CPUGFX)) + average.packages.pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0; + + average.packages.pc2 += p->pc2; + if (DO_BIC(BIC_Pkgpc3)) + average.packages.pc3 += p->pc3; + if (DO_BIC(BIC_Pkgpc6)) + average.packages.pc6 += p->pc6; + if (DO_BIC(BIC_Pkgpc7)) + average.packages.pc7 += p->pc7; + average.packages.pc8 += p->pc8; + average.packages.pc9 += p->pc9; + average.packages.pc10 += p->pc10; + average.packages.die_c6 += p->die_c6; + + average.packages.cpu_lpi = p->cpu_lpi; + average.packages.sys_lpi = p->sys_lpi; + + rapl_counter_accumulate(&average.packages.energy_pkg, &p->energy_pkg); + rapl_counter_accumulate(&average.packages.energy_dram, &p->energy_dram); + rapl_counter_accumulate(&average.packages.energy_cores, &p->energy_cores); + rapl_counter_accumulate(&average.packages.energy_gfx, &p->energy_gfx); + + average.packages.gfx_rc6_ms = p->gfx_rc6_ms; + average.packages.uncore_mhz = p->uncore_mhz; + average.packages.gfx_mhz = p->gfx_mhz; + average.packages.gfx_act_mhz = p->gfx_act_mhz; + average.packages.sam_mc6_ms = p->sam_mc6_ms; + average.packages.sam_mhz = p->sam_mhz; + average.packages.sam_act_mhz = p->sam_act_mhz; + + average.packages.pkg_temp_c = MAX(average.packages.pkg_temp_c, p->pkg_temp_c); + + rapl_counter_accumulate(&average.packages.rapl_pkg_perf_status, &p->rapl_pkg_perf_status); + rapl_counter_accumulate(&average.packages.rapl_dram_perf_status, &p->rapl_dram_perf_status); + + for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { + if ((mp->format == FORMAT_RAW) && (topo.num_packages == 0)) + average.packages.counter[i] = p->counter[i]; + else + average.packages.counter[i] += p->counter[i]; + } + + for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) { + if ((pp->format == FORMAT_RAW) && (topo.num_packages == 0)) + average.packages.perf_counter[i] = p->perf_counter[i]; + else + average.packages.perf_counter[i] += p->perf_counter[i]; + } + + for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) { + average.packages.pmt_counter[i] += p->pmt_counter[i]; + } + + return 0; +} + +/* + * sum the counters for all cpus in the system + * compute the weighted average + */ +void compute_average(PER_THREAD_PARAMS) +{ + int i; + struct msr_counter *mp; + struct perf_counter_info *pp; + struct pmt_counter *ppmt; + + clear_counters(&average.threads, &average.cores, &average.packages); + + for_all_cpus(sum_counters, t, c, p); + + /* Use the global time delta for the average. */ + average.threads.tv_delta = tv_delta; + + average.threads.tsc /= topo.allowed_cpus; + average.threads.aperf /= topo.allowed_cpus; + average.threads.mperf /= topo.allowed_cpus; + average.threads.instr_count /= topo.allowed_cpus; + average.threads.c1 /= topo.allowed_cpus; + + if (average.threads.irq_count > 9999999) + sums_need_wide_columns = 1; + if (average.threads.nmi_count > 9999999) + sums_need_wide_columns = 1; + + average.cores.c3 /= topo.allowed_cores; + average.cores.c6 /= topo.allowed_cores; + average.cores.c7 /= topo.allowed_cores; + average.cores.mc6_us /= topo.allowed_cores; + + if (DO_BIC(BIC_Totl_c0)) + average.packages.pkg_wtd_core_c0 /= topo.allowed_packages; + if (DO_BIC(BIC_Any_c0)) + average.packages.pkg_any_core_c0 /= topo.allowed_packages; + if (DO_BIC(BIC_GFX_c0)) + average.packages.pkg_any_gfxe_c0 /= topo.allowed_packages; + if (DO_BIC(BIC_CPUGFX)) + average.packages.pkg_both_core_gfxe_c0 /= topo.allowed_packages; + + average.packages.pc2 /= topo.allowed_packages; + if (DO_BIC(BIC_Pkgpc3)) + average.packages.pc3 /= topo.allowed_packages; + if (DO_BIC(BIC_Pkgpc6)) + average.packages.pc6 /= topo.allowed_packages; + if (DO_BIC(BIC_Pkgpc7)) + average.packages.pc7 /= topo.allowed_packages; + + average.packages.pc8 /= topo.allowed_packages; + average.packages.pc9 /= topo.allowed_packages; + average.packages.pc10 /= topo.allowed_packages; + average.packages.die_c6 /= topo.allowed_packages; + + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW) + continue; + if (mp->type == COUNTER_ITEMS) { + if (average.threads.counter[i] > 9999999) + sums_need_wide_columns = 1; + continue; + } + average.threads.counter[i] /= topo.allowed_cpus; + } + for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW) + continue; + if (mp->type == COUNTER_ITEMS) { + if (average.cores.counter[i] > 9999999) + sums_need_wide_columns = 1; + } + average.cores.counter[i] /= topo.allowed_cores; + } + for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { + if (mp->format == FORMAT_RAW) + continue; + if (mp->type == COUNTER_ITEMS) { + if (average.packages.counter[i] > 9999999) + sums_need_wide_columns = 1; + } + average.packages.counter[i] /= topo.allowed_packages; + } + + for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) { + if (pp->format == FORMAT_RAW) + continue; + if (pp->type == COUNTER_ITEMS) { + if (average.threads.perf_counter[i] > 9999999) + sums_need_wide_columns = 1; + continue; + } + average.threads.perf_counter[i] /= topo.allowed_cpus; + } + for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { + if (pp->format == FORMAT_RAW) + continue; + if (pp->type == COUNTER_ITEMS) { + if (average.cores.perf_counter[i] > 9999999) + sums_need_wide_columns = 1; + } + average.cores.perf_counter[i] /= topo.allowed_cores; + } + for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) { + if (pp->format == FORMAT_RAW) + continue; + if (pp->type == COUNTER_ITEMS) { + if (average.packages.perf_counter[i] > 9999999) + sums_need_wide_columns = 1; + } + average.packages.perf_counter[i] /= topo.allowed_packages; + } + + for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) { + average.threads.pmt_counter[i] /= topo.allowed_cpus; + } + for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) { + average.cores.pmt_counter[i] /= topo.allowed_cores; + } + for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) { + average.packages.pmt_counter[i] /= topo.allowed_packages; + } +} + +static unsigned long long rdtsc(void) +{ + unsigned int low, high; + + asm volatile ("rdtsc":"=a" (low), "=d"(high)); + + return low | ((unsigned long long)high) << 32; +} + +/* + * Open a file, and exit on failure + */ +FILE *fopen_or_die(const char *path, const char *mode) +{ + FILE *filep = fopen(path, mode); + + if (!filep) + err(1, "%s: open failed", path); + return filep; +} + +/* + * snapshot_sysfs_counter() + * + * return snapshot of given counter + */ +unsigned long long snapshot_sysfs_counter(char *path) +{ + FILE *fp; + int retval; + unsigned long long counter; + + fp = fopen_or_die(path, "r"); + + retval = fscanf(fp, "%lld", &counter); + if (retval != 1) + err(1, "snapshot_sysfs_counter(%s)", path); + + fclose(fp); + + return counter; +} + +int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp, char *counter_path) +{ + if (mp->msr_num != 0) { + assert(!no_msr); + if (get_msr(cpu, mp->msr_num, counterp)) + return -1; + } else { + char path[128 + PATH_BYTES]; + + if (mp->flags & SYSFS_PERCPU) { + sprintf(path, "/sys/devices/system/cpu/cpu%d/%s", cpu, mp->sp->path); + + *counterp = snapshot_sysfs_counter(path); + } else { + *counterp = snapshot_sysfs_counter(counter_path); + } + } + + return 0; +} + +unsigned long long get_legacy_uncore_mhz(int package) +{ + char path[128]; + int die; + static int warn_once; + + /* + * for this package, use the first die_id that exists + */ + for (die = 0; die <= topo.max_die_id; ++die) { + + sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", package, die); + + if (access(path, R_OK) == 0) + return (snapshot_sysfs_counter(path) / 1000); + } + if (!warn_once) { + warnx("BUG: %s: No %s", __func__, path); + warn_once = 1; + } + + return 0; +} + +int get_epb(int cpu) +{ + char path[128 + PATH_BYTES]; + unsigned long long msr; + int ret, epb = -1; + FILE *fp; + + sprintf(path, "/sys/devices/system/cpu/cpu%d/power/energy_perf_bias", cpu); + + fp = fopen(path, "r"); + if (!fp) + goto msr_fallback; + + ret = fscanf(fp, "%d", &epb); + if (ret != 1) + err(1, "%s(%s)", __func__, path); + + fclose(fp); + + return epb; + +msr_fallback: + if (no_msr) + return -1; + + get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr); + + return msr & 0xf; +} + +void get_apic_id(struct thread_data *t) +{ + unsigned int eax, ebx, ecx, edx; + + if (DO_BIC(BIC_APIC)) { + eax = ebx = ecx = edx = 0; + __cpuid(1, eax, ebx, ecx, edx); + + t->apic_id = (ebx >> 24) & 0xff; + } + + if (!DO_BIC(BIC_X2APIC)) + return; + + if (authentic_amd || hygon_genuine) { + unsigned int topology_extensions; + + if (max_extended_level < 0x8000001e) + return; + + eax = ebx = ecx = edx = 0; + __cpuid(0x80000001, eax, ebx, ecx, edx); + topology_extensions = ecx & (1 << 22); + + if (topology_extensions == 0) + return; + + eax = ebx = ecx = edx = 0; + __cpuid(0x8000001e, eax, ebx, ecx, edx); + + t->x2apic_id = eax; + return; + } + + if (!genuine_intel) + return; + + if (max_level < 0xb) + return; + + ecx = 0; + __cpuid(0xb, eax, ebx, ecx, edx); + t->x2apic_id = edx; + + if (debug && (t->apic_id != (t->x2apic_id & 0xff))) + fprintf(outf, "cpu%d: BIOS BUG: apic 0x%x x2apic 0x%x\n", t->cpu_id, t->apic_id, t->x2apic_id); +} + +int get_core_throt_cnt(int cpu, unsigned long long *cnt) +{ + char path[128 + PATH_BYTES]; + unsigned long long tmp; + FILE *fp; + int ret; + + sprintf(path, "/sys/devices/system/cpu/cpu%d/thermal_throttle/core_throttle_count", cpu); + fp = fopen(path, "r"); + if (!fp) + return -1; + ret = fscanf(fp, "%lld", &tmp); + fclose(fp); + if (ret != 1) + return -1; + *cnt = tmp; + + return 0; +} + +static int read_perf_counter_info(const char *const path, const char *const parse_format, void *value_ptr) +{ + int fdmt; + int bytes_read; + char buf[64]; + int ret = -1; + + fdmt = open(path, O_RDONLY, 0); + if (fdmt == -1) { + if (debug) + fprintf(stderr, "Failed to parse perf counter info %s\n", path); + ret = -1; + goto cleanup_and_exit; + } + + bytes_read = read(fdmt, buf, sizeof(buf) - 1); + if (bytes_read <= 0 || bytes_read >= (int)sizeof(buf)) { + if (debug) + fprintf(stderr, "Failed to parse perf counter info %s\n", path); + ret = -1; + goto cleanup_and_exit; + } + + buf[bytes_read] = '\0'; + + if (sscanf(buf, parse_format, value_ptr) != 1) { + if (debug) + fprintf(stderr, "Failed to parse perf counter info %s\n", path); + ret = -1; + goto cleanup_and_exit; + } + + ret = 0; + +cleanup_and_exit: + close(fdmt); + return ret; +} + +static unsigned int read_perf_counter_info_n(const char *const path, const char *const parse_format) +{ + unsigned int v; + int status; + + status = read_perf_counter_info(path, parse_format, &v); + if (status) + v = -1; + + return v; +} + +static unsigned int read_perf_type(const char *subsys) +{ + const char *const path_format = "/sys/bus/event_source/devices/%s/type"; + const char *const format = "%u"; + char path[128]; + + snprintf(path, sizeof(path), path_format, subsys); + + return read_perf_counter_info_n(path, format); +} + +static unsigned int read_perf_config(const char *subsys, const char *event_name) +{ + const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s"; + FILE *fconfig = NULL; + char path[128]; + char config_str[64]; + unsigned int config; + unsigned int umask; + bool has_config = false; + bool has_umask = false; + unsigned int ret = -1; + + snprintf(path, sizeof(path), path_format, subsys, event_name); + + fconfig = fopen(path, "r"); + if (!fconfig) + return -1; + + if (fgets(config_str, ARRAY_SIZE(config_str), fconfig) != config_str) + goto cleanup_and_exit; + + for (char *pconfig_str = &config_str[0]; pconfig_str;) { + if (sscanf(pconfig_str, "event=%x", &config) == 1) { + has_config = true; + goto next; + } + + if (sscanf(pconfig_str, "umask=%x", &umask) == 1) { + has_umask = true; + goto next; + } + +next: + pconfig_str = strchr(pconfig_str, ','); + if (pconfig_str) { + *pconfig_str = '\0'; + ++pconfig_str; + } + } + + if (!has_umask) + umask = 0; + + if (has_config) + ret = (umask << 8) | config; + +cleanup_and_exit: + fclose(fconfig); + return ret; +} + +static unsigned int read_perf_rapl_unit(const char *subsys, const char *event_name) +{ + const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.unit"; + const char *const format = "%s"; + char path[128]; + char unit_buffer[16]; + + snprintf(path, sizeof(path), path_format, subsys, event_name); + + read_perf_counter_info(path, format, &unit_buffer); + if (strcmp("Joules", unit_buffer) == 0) + return RAPL_UNIT_JOULES; + + return RAPL_UNIT_INVALID; +} + +static double read_perf_scale(const char *subsys, const char *event_name) +{ + const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.scale"; + const char *const format = "%lf"; + char path[128]; + double scale; + + snprintf(path, sizeof(path), path_format, subsys, event_name); + + if (read_perf_counter_info(path, format, &scale)) + return 0.0; + + return scale; +} + +size_t rapl_counter_info_count_perf(const struct rapl_counter_info_t *rci) +{ + size_t ret = 0; + + for (int i = 0; i < NUM_RAPL_COUNTERS; ++i) + if (rci->source[i] == COUNTER_SOURCE_PERF) + ++ret; + + return ret; +} + +static size_t cstate_counter_info_count_perf(const struct cstate_counter_info_t *cci) +{ + size_t ret = 0; + + for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i) + if (cci->source[i] == COUNTER_SOURCE_PERF) + ++ret; + + return ret; +} + +void write_rapl_counter(struct rapl_counter *rc, struct rapl_counter_info_t *rci, unsigned int idx) +{ + if (rci->source[idx] == COUNTER_SOURCE_NONE) + return; + + rc->raw_value = rci->data[idx]; + rc->unit = rci->unit[idx]; + rc->scale = rci->scale[idx]; +} + +int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct pkg_data *p) +{ + struct platform_counters *pplat_cnt = p == package_odd ? &platform_counters_odd : &platform_counters_even; + unsigned long long perf_data[NUM_RAPL_COUNTERS + 1]; + struct rapl_counter_info_t *rci; + + if (debug >= 2) + fprintf(stderr, "%s: cpu%d domain%d\n", __func__, cpu, domain); + + assert(rapl_counter_info_perdomain); + assert(domain < rapl_counter_info_perdomain_size); + + rci = &rapl_counter_info_perdomain[domain]; + + /* + * If we have any perf counters to read, read them all now, in bulk + */ + if (rci->fd_perf != -1) { + size_t num_perf_counters = rapl_counter_info_count_perf(rci); + const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long); + const ssize_t actual_read_size = read(rci->fd_perf, &perf_data[0], sizeof(perf_data)); + + if (actual_read_size != expected_read_size) + err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size); + } + + for (unsigned int i = 0, pi = 1; i < NUM_RAPL_COUNTERS; ++i) { + switch (rci->source[i]) { + case COUNTER_SOURCE_NONE: + rci->data[i] = 0; + break; + + case COUNTER_SOURCE_PERF: + assert(pi < ARRAY_SIZE(perf_data)); + assert(rci->fd_perf != -1); + + if (debug >= 2) + fprintf(stderr, "Reading rapl counter via perf at %u (%llu %e %lf)\n", + i, perf_data[pi], rci->scale[i], perf_data[pi] * rci->scale[i]); + + rci->data[i] = perf_data[pi]; + + ++pi; + break; + + case COUNTER_SOURCE_MSR: + if (debug >= 2) + fprintf(stderr, "Reading rapl counter via msr at %u\n", i); + + assert(!no_msr); + if (rci->flags[i] & RAPL_COUNTER_FLAG_USE_MSR_SUM) { + if (get_msr_sum(cpu, rci->msr[i], &rci->data[i])) + return -13 - i; + } else { + if (get_msr(cpu, rci->msr[i], &rci->data[i])) + return -13 - i; + } + + rci->data[i] &= rci->msr_mask[i]; + if (rci->msr_shift[i] >= 0) + rci->data[i] >>= abs(rci->msr_shift[i]); + else + rci->data[i] <<= abs(rci->msr_shift[i]); + + break; + } + } + + BUILD_BUG_ON(NUM_RAPL_COUNTERS != 8); + write_rapl_counter(&p->energy_pkg, rci, RAPL_RCI_INDEX_ENERGY_PKG); + write_rapl_counter(&p->energy_cores, rci, RAPL_RCI_INDEX_ENERGY_CORES); + write_rapl_counter(&p->energy_dram, rci, RAPL_RCI_INDEX_DRAM); + write_rapl_counter(&p->energy_gfx, rci, RAPL_RCI_INDEX_GFX); + write_rapl_counter(&p->rapl_pkg_perf_status, rci, RAPL_RCI_INDEX_PKG_PERF_STATUS); + write_rapl_counter(&p->rapl_dram_perf_status, rci, RAPL_RCI_INDEX_DRAM_PERF_STATUS); + write_rapl_counter(&c->core_energy, rci, RAPL_RCI_INDEX_CORE_ENERGY); + write_rapl_counter(&pplat_cnt->energy_psys, rci, RAPL_RCI_INDEX_ENERGY_PLATFORM); + + return 0; +} + +char *find_sysfs_path_by_id(struct sysfs_path *sp, int id) +{ + while (sp) { + if (sp->id == id) + return (sp->path); + sp = sp->next; + } + if (debug) + warnx("%s: id%d not found", __func__, id); + return NULL; +} + +int get_cstate_counters(unsigned int cpu, PER_THREAD_PARAMS) +{ + /* + * Overcommit memory a little bit here, + * but skip calculating exact sizes for the buffers. + */ + unsigned long long perf_data[NUM_CSTATE_COUNTERS]; + unsigned long long perf_data_core[NUM_CSTATE_COUNTERS + 1]; + unsigned long long perf_data_pkg[NUM_CSTATE_COUNTERS + 1]; + + struct cstate_counter_info_t *cci; + + if (debug >= 2) + fprintf(stderr, "%s: cpu%d\n", __func__, cpu); + + assert(ccstate_counter_info); + assert(cpu <= ccstate_counter_info_size); + + ZERO_ARRAY(perf_data); + ZERO_ARRAY(perf_data_core); + ZERO_ARRAY(perf_data_pkg); + + cci = &ccstate_counter_info[cpu]; + + /* + * If we have any perf counters to read, read them all now, in bulk + */ + const size_t num_perf_counters = cstate_counter_info_count_perf(cci); + ssize_t expected_read_size = num_perf_counters * sizeof(unsigned long long); + ssize_t actual_read_size_core = 0, actual_read_size_pkg = 0; + + if (cci->fd_perf_core != -1) { + /* Each descriptor read begins with number of counters read. */ + expected_read_size += sizeof(unsigned long long); + + actual_read_size_core = read(cci->fd_perf_core, &perf_data_core[0], sizeof(perf_data_core)); + + if (actual_read_size_core <= 0) + err(-1, "%s: read perf %s: %ld", __func__, "core", actual_read_size_core); + } + + if (cci->fd_perf_pkg != -1) { + /* Each descriptor read begins with number of counters read. */ + expected_read_size += sizeof(unsigned long long); + + actual_read_size_pkg = read(cci->fd_perf_pkg, &perf_data_pkg[0], sizeof(perf_data_pkg)); + + if (actual_read_size_pkg <= 0) + err(-1, "%s: read perf %s: %ld", __func__, "pkg", actual_read_size_pkg); + } + + const ssize_t actual_read_size_total = actual_read_size_core + actual_read_size_pkg; + + if (actual_read_size_total != expected_read_size) + err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size_total); + + /* + * Copy ccstate and pcstate data into unified buffer. + * + * Skip first element from core and pkg buffers. + * Kernel puts there how many counters were read. + */ + const size_t num_core_counters = perf_data_core[0]; + const size_t num_pkg_counters = perf_data_pkg[0]; - tsc = t->tsc * tsc_tweak; + assert(num_perf_counters == num_core_counters + num_pkg_counters); - /* topo columns, print blanks on 1st (average) line */ - if (t == &average.threads) { - if (DO_BIC(BIC_Package)) - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); - if (DO_BIC(BIC_Die)) - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); - if (DO_BIC(BIC_Node)) - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); - if (DO_BIC(BIC_Core)) - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); - if (DO_BIC(BIC_CPU)) - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); - if (DO_BIC(BIC_APIC)) - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); - if (DO_BIC(BIC_X2APIC)) - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); - } else { - if (DO_BIC(BIC_Package)) { - if (p) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->package_id); - else - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); - } - if (DO_BIC(BIC_Die)) { - if (c) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].die_id); - else - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + /* Copy ccstate perf data */ + memcpy(&perf_data[0], &perf_data_core[1], num_core_counters * sizeof(unsigned long long)); + + /* Copy pcstate perf data */ + memcpy(&perf_data[num_core_counters], &perf_data_pkg[1], num_pkg_counters * sizeof(unsigned long long)); + + for (unsigned int i = 0, pi = 0; i < NUM_CSTATE_COUNTERS; ++i) { + switch (cci->source[i]) { + case COUNTER_SOURCE_NONE: + break; + + case COUNTER_SOURCE_PERF: + assert(pi < ARRAY_SIZE(perf_data)); + assert(cci->fd_perf_core != -1 || cci->fd_perf_pkg != -1); + + if (debug >= 2) + fprintf(stderr, "cstate via %s %u: %llu\n", "perf", i, perf_data[pi]); + + cci->data[i] = perf_data[pi]; + + ++pi; + break; + + case COUNTER_SOURCE_MSR: + assert(!no_msr); + if (get_msr(cpu, cci->msr[i], &cci->data[i])) + return -13 - i; + + if (debug >= 2) + fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", "msr", cci->msr[i], i, cci->data[i]); + + break; } - if (DO_BIC(BIC_Node)) { - if (t) - outp += sprintf(outp, "%s%d", - (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id); - else - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + } + + /* + * Helper to write the data only if the source of + * the counter for the current cpu is not none. + * + * Otherwise we would overwrite core data with 0 (default value), + * when invoked for the thread sibling. + */ +#define PERF_COUNTER_WRITE_DATA(out_counter, index) do { \ + if (cci->source[index] != COUNTER_SOURCE_NONE) \ + out_counter = cci->data[index]; \ +} while (0) + + BUILD_BUG_ON(NUM_CSTATE_COUNTERS != 11); + + PERF_COUNTER_WRITE_DATA(t->c1, CCSTATE_RCI_INDEX_C1_RESIDENCY); + PERF_COUNTER_WRITE_DATA(c->c3, CCSTATE_RCI_INDEX_C3_RESIDENCY); + PERF_COUNTER_WRITE_DATA(c->c6, CCSTATE_RCI_INDEX_C6_RESIDENCY); + PERF_COUNTER_WRITE_DATA(c->c7, CCSTATE_RCI_INDEX_C7_RESIDENCY); + + PERF_COUNTER_WRITE_DATA(p->pc2, PCSTATE_RCI_INDEX_C2_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc3, PCSTATE_RCI_INDEX_C3_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc6, PCSTATE_RCI_INDEX_C6_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc7, PCSTATE_RCI_INDEX_C7_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc8, PCSTATE_RCI_INDEX_C8_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc9, PCSTATE_RCI_INDEX_C9_RESIDENCY); + PERF_COUNTER_WRITE_DATA(p->pc10, PCSTATE_RCI_INDEX_C10_RESIDENCY); + +#undef PERF_COUNTER_WRITE_DATA + + return 0; +} + +size_t msr_counter_info_count_perf(const struct msr_counter_info_t *mci) +{ + size_t ret = 0; + + for (int i = 0; i < NUM_MSR_COUNTERS; ++i) + if (mci->source[i] == COUNTER_SOURCE_PERF) + ++ret; + + return ret; +} + +int get_smi_aperf_mperf(unsigned int cpu, struct thread_data *t) +{ + unsigned long long perf_data[NUM_MSR_COUNTERS + 1]; + + struct msr_counter_info_t *mci; + + if (debug >= 2) + fprintf(stderr, "%s: cpu%d\n", __func__, cpu); + + assert(msr_counter_info); + assert(cpu <= msr_counter_info_size); + + mci = &msr_counter_info[cpu]; + + ZERO_ARRAY(perf_data); + ZERO_ARRAY(mci->data); + + if (mci->fd_perf != -1) { + const size_t num_perf_counters = msr_counter_info_count_perf(mci); + const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long); + const ssize_t actual_read_size = read(mci->fd_perf, &perf_data[0], sizeof(perf_data)); + + if (actual_read_size != expected_read_size) + err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size); + } + + for (unsigned int i = 0, pi = 1; i < NUM_MSR_COUNTERS; ++i) { + switch (mci->source[i]) { + case COUNTER_SOURCE_NONE: + break; + + case COUNTER_SOURCE_PERF: + assert(pi < ARRAY_SIZE(perf_data)); + assert(mci->fd_perf != -1); + + if (debug >= 2) + fprintf(stderr, "Reading msr counter via perf at %u: %llu\n", i, perf_data[pi]); + + mci->data[i] = perf_data[pi]; + + ++pi; + break; + + case COUNTER_SOURCE_MSR: + assert(!no_msr); + + if (get_msr(cpu, mci->msr[i], &mci->data[i])) + return -2 - i; + + mci->data[i] &= mci->msr_mask[i]; + + if (debug >= 2) + fprintf(stderr, "Reading msr counter via msr at %u: %llu\n", i, mci->data[i]); + + break; } - if (DO_BIC(BIC_Core)) { - if (c) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_id); - else - outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + } + + BUILD_BUG_ON(NUM_MSR_COUNTERS != 3); + t->aperf = mci->data[MSR_RCI_INDEX_APERF]; + t->mperf = mci->data[MSR_RCI_INDEX_MPERF]; + t->smi_count = mci->data[MSR_RCI_INDEX_SMI]; + + return 0; +} + +int perf_counter_info_read_values(struct perf_counter_info *pp, int cpu, unsigned long long *out, size_t out_size) +{ + unsigned int domain; + unsigned long long value; + int fd_counter; + + for (size_t i = 0; pp; ++i, pp = pp->next) { + domain = cpu_to_domain(pp, cpu); + assert(domain < pp->num_domains); + + fd_counter = pp->fd_perf_per_domain[domain]; + + if (fd_counter == -1) + continue; + + if (read(fd_counter, &value, sizeof(value)) != sizeof(value)) + return 1; + + assert(i < out_size); + out[i] = value * pp->scale; + } + + return 0; +} + +unsigned long pmt_gen_value_mask(unsigned int lsb, unsigned int msb) +{ + unsigned long mask; + + if (msb == 63) + mask = 0xffffffffffffffff; + else + mask = ((1 << (msb + 1)) - 1); + + mask -= (1 << lsb) - 1; + + return mask; +} + +unsigned long pmt_read_counter(struct pmt_counter *ppmt, unsigned int domain_id) +{ + if (domain_id >= ppmt->num_domains) + return 0; + + const unsigned long *pmmio = ppmt->domains[domain_id].pcounter; + const unsigned long value = pmmio ? *pmmio : 0; + const unsigned long value_mask = pmt_gen_value_mask(ppmt->lsb, ppmt->msb); + const unsigned long value_shift = ppmt->lsb; + + return (value & value_mask) >> value_shift; +} + +/* Rapl domain enumeration helpers */ +static inline int get_rapl_num_domains(void) +{ + int num_packages = topo.max_package_id + 1; + int num_cores_per_package; + int num_cores; + + if (!platform->has_per_core_rapl) + return num_packages; + + num_cores_per_package = topo.max_core_id + 1; + num_cores = num_cores_per_package * num_packages; + + return num_cores; +} + +static inline int get_rapl_domain_id(int cpu) +{ + int nr_cores_per_package = topo.max_core_id + 1; + int rapl_core_id; + + if (!platform->has_per_core_rapl) + return cpus[cpu].physical_package_id; + + /* Compute the system-wide unique core-id for @cpu */ + rapl_core_id = cpus[cpu].physical_core_id; + rapl_core_id += cpus[cpu].physical_package_id * nr_cores_per_package; + + return rapl_core_id; +} + +/* + * get_counters(...) + * migrate to cpu + * acquire and record local counters for that cpu + */ +int get_counters(PER_THREAD_PARAMS) +{ + int cpu = t->cpu_id; + unsigned long long msr; + struct msr_counter *mp; + struct pmt_counter *pp; + int i; + int status; + + if (cpu_migrate(cpu)) { + fprintf(outf, "%s: Could not migrate to CPU %d\n", __func__, cpu); + return -1; + } + + gettimeofday(&t->tv_begin, (struct timezone *)NULL); + + if (first_counter_read) + get_apic_id(t); + + t->tsc = rdtsc(); /* we are running on local CPU of interest */ + + get_smi_aperf_mperf(cpu, t); + + if (DO_BIC(BIC_LLC_RPS) || DO_BIC(BIC_LLC_HIT)) + get_perf_llc_stats(cpu, &t->llc); + + if (DO_BIC(BIC_IPC)) + if (read(get_instr_count_fd(cpu), &t->instr_count, sizeof(long long)) != sizeof(long long)) + return -4; + + if (DO_BIC(BIC_IRQ)) + t->irq_count = irqs_per_cpu[cpu]; + if (DO_BIC(BIC_NMI)) + t->nmi_count = nmi_per_cpu[cpu]; + + get_cstate_counters(cpu, t, c, p); + + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { + if (get_mp(cpu, mp, &t->counter[i], mp->sp->path)) + return -10; + } + + if (perf_counter_info_read_values(sys.perf_tp, cpu, t->perf_counter, MAX_ADDED_THREAD_COUNTERS)) + return -10; + + for (i = 0, pp = sys.pmt_tp; pp; i++, pp = pp->next) + t->pmt_counter[i] = pmt_read_counter(pp, t->cpu_id); + + /* collect core counters only for 1st thread in core */ + if (!is_cpu_first_thread_in_core(t, c)) + goto done; + + if (platform->has_per_core_rapl) { + status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p); + if (status != 0) + return status; + } + + if (DO_BIC(BIC_CPU_c7) && t->is_atom) { + /* + * For Atom CPUs that has core cstate deeper than c6, + * MSR_CORE_C6_RESIDENCY returns residency of cc6 and deeper. + * Minus CC7 (and deeper cstates) residency to get + * accturate cc6 residency. + */ + c->c6 -= c->c7; + } + + if (DO_BIC(BIC_Mod_c6)) + if (get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us)) + return -8; + + if (DO_BIC(BIC_CoreTmp)) { + if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr)) + return -9; + c->core_temp_c = tj_max - ((msr >> 16) & 0x7F); + } + + if (DO_BIC(BIC_CORE_THROT_CNT)) + get_core_throt_cnt(cpu, &c->core_throt_cnt); + + for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { + if (get_mp(cpu, mp, &c->counter[i], mp->sp->path)) + return -10; + } + + if (perf_counter_info_read_values(sys.perf_cp, cpu, c->perf_counter, MAX_ADDED_CORE_COUNTERS)) + return -10; + + for (i = 0, pp = sys.pmt_cp; pp; i++, pp = pp->next) + c->pmt_counter[i] = pmt_read_counter(pp, c->core_id); + + /* collect package counters only for 1st core in package */ + if (!is_cpu_first_core_in_package(t, p)) + goto done; + + if (DO_BIC(BIC_Totl_c0)) { + if (get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, &p->pkg_wtd_core_c0)) + return -10; + } + if (DO_BIC(BIC_Any_c0)) { + if (get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, &p->pkg_any_core_c0)) + return -11; + } + if (DO_BIC(BIC_GFX_c0)) { + if (get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, &p->pkg_any_gfxe_c0)) + return -12; + } + if (DO_BIC(BIC_CPUGFX)) { + if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0)) + return -13; + } + + if (DO_BIC(BIC_CPU_LPI)) + p->cpu_lpi = cpuidle_cur_cpu_lpi_us; + if (DO_BIC(BIC_SYS_LPI)) + p->sys_lpi = cpuidle_cur_sys_lpi_us; + + if (!platform->has_per_core_rapl) { + status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p); + if (status != 0) + return status; + } + + if (DO_BIC(BIC_PkgTmp)) { + if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr)) + return -17; + p->pkg_temp_c = tj_max - ((msr >> 16) & 0x7F); + } + + if (DO_BIC(BIC_UNCORE_MHZ)) + p->uncore_mhz = get_legacy_uncore_mhz(p->package_id); + + if (DO_BIC(BIC_GFX_rc6)) + p->gfx_rc6_ms = gfx_info[GFX_rc6].val_ull; + + if (DO_BIC(BIC_GFXMHz)) + p->gfx_mhz = gfx_info[GFX_MHz].val; + + if (DO_BIC(BIC_GFXACTMHz)) + p->gfx_act_mhz = gfx_info[GFX_ACTMHz].val; + + if (DO_BIC(BIC_SAM_mc6)) + p->sam_mc6_ms = gfx_info[SAM_mc6].val_ull; + + if (DO_BIC(BIC_SAMMHz)) + p->sam_mhz = gfx_info[SAM_MHz].val; + + if (DO_BIC(BIC_SAMACTMHz)) + p->sam_act_mhz = gfx_info[SAM_ACTMHz].val; + + for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { + char *path = NULL; + + if (mp->msr_num == 0) { + path = find_sysfs_path_by_id(mp->sp, p->package_id); + if (path == NULL) { + warnx("%s: package_id %d not found", __func__, p->package_id); + return -10; + } } - if (DO_BIC(BIC_CPU)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->cpu_id); - if (DO_BIC(BIC_APIC)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->apic_id); - if (DO_BIC(BIC_X2APIC)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->x2apic_id); + if (get_mp(cpu, mp, &p->counter[i], path)) + return -10; } - if (DO_BIC(BIC_Avg_MHz)) - outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float); + if (perf_counter_info_read_values(sys.perf_pp, cpu, p->perf_counter, MAX_ADDED_PACKAGE_COUNTERS)) + return -10; - if (DO_BIC(BIC_Busy)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->mperf / tsc); + for (i = 0, pp = sys.pmt_pp; pp; i++, pp = pp->next) + p->pmt_counter[i] = pmt_read_counter(pp, p->package_id); - if (DO_BIC(BIC_Bzy_MHz)) { - if (has_base_hz) - outp += - sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf); - else - outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), - tsc / units * t->aperf / t->mperf / interval_float); - } +done: + gettimeofday(&t->tv_end, (struct timezone *)NULL); - if (DO_BIC(BIC_TSC_MHz)) - outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 * t->tsc / units / interval_float); + return 0; +} - if (DO_BIC(BIC_IPC)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 1.0 * t->instr_count / t->aperf); +int pkg_cstate_limit = PCLUKN; +char *pkg_cstate_limit_strings[] = { "unknown", "reserved", "pc0", "pc1", "pc2", + "pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited" +}; - /* IRQ */ - if (DO_BIC(BIC_IRQ)) { - if (sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->irq_count); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->irq_count); - } +int nhm_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, + PCLRSV, PCLRSV +}; - /* SMI */ - if (DO_BIC(BIC_SMI)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->smi_count); +int snb_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, + PCLRSV, PCLRSV +}; - /* Added counters */ - for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) { - if (mp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)t->counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->counter[i]); - } else if (mp->format == FORMAT_DELTA) { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->counter[i]); - } else if (mp->format == FORMAT_PERCENT) { - if (mp->type == COUNTER_USEC) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - t->counter[i] / interval_float / 10000); - else - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->counter[i] / tsc); - } - } +int hsw_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, + PCLRSV, PCLRSV +}; - /* C1 */ - if (DO_BIC(BIC_CPU_c1)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->c1 / tsc); +int slv_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, + PCL__6, PCL__7 +}; - /* print per-core data only for 1st thread in core */ - if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE)) - goto done; +int amt_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, + PCLRSV, PCLRSV +}; - if (DO_BIC(BIC_CPU_c3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c3 / tsc); - if (DO_BIC(BIC_CPU_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c6 / tsc); - if (DO_BIC(BIC_CPU_c7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c7 / tsc); +int phi_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, + PCLRSV, PCLRSV +}; - /* Mod%c6 */ - if (DO_BIC(BIC_Mod_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->mc6_us / tsc); +int glm_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, + PCLRSV, PCLRSV +}; - if (DO_BIC(BIC_CoreTmp)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c); +int skx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, + PCLRSV, PCLRSV +}; - /* Core throttle count */ - if (DO_BIC(BIC_CORE_THROT_CNT)) - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->core_throt_cnt); +int icx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, + PCLRSV, PCLRSV +}; - for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) { - if (mp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)c->counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->counter[i]); - } else if (mp->format == FORMAT_DELTA) { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), c->counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->counter[i]); - } else if (mp->format == FORMAT_PERCENT) { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->counter[i] / tsc); - } +void probe_cst_limit(void) +{ + unsigned long long msr; + int *pkg_cstate_limits; + + if (!platform->has_nhm_msrs || no_msr) + return; + + switch (platform->cst_limit) { + case CST_LIMIT_NHM: + pkg_cstate_limits = nhm_pkg_cstate_limits; + break; + case CST_LIMIT_SNB: + pkg_cstate_limits = snb_pkg_cstate_limits; + break; + case CST_LIMIT_HSW: + pkg_cstate_limits = hsw_pkg_cstate_limits; + break; + case CST_LIMIT_SKX: + pkg_cstate_limits = skx_pkg_cstate_limits; + break; + case CST_LIMIT_ICX: + pkg_cstate_limits = icx_pkg_cstate_limits; + break; + case CST_LIMIT_SLV: + pkg_cstate_limits = slv_pkg_cstate_limits; + break; + case CST_LIMIT_AMT: + pkg_cstate_limits = amt_pkg_cstate_limits; + break; + case CST_LIMIT_KNL: + pkg_cstate_limits = phi_pkg_cstate_limits; + break; + case CST_LIMIT_GMT: + pkg_cstate_limits = glm_pkg_cstate_limits; + break; + default: + return; } - fmt8 = "%s%.2f"; + get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); + pkg_cstate_limit = pkg_cstate_limits[msr & 0xF]; +} - if (DO_BIC(BIC_CorWatt) && (do_rapl & RAPL_PER_CORE_ENERGY)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), c->core_energy * rapl_energy_units / interval_float); - if (DO_BIC(BIC_Cor_J) && (do_rapl & RAPL_PER_CORE_ENERGY)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), c->core_energy * rapl_energy_units); +static void dump_platform_info(void) +{ + unsigned long long msr; + unsigned int ratio; - /* print per-package data only for 1st core in package */ - if (!(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) - goto done; + if (!platform->has_nhm_msrs || no_msr) + return; - /* PkgTmp */ - if (DO_BIC(BIC_PkgTmp)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->pkg_temp_c); + get_msr(base_cpu, MSR_PLATFORM_INFO, &msr); - /* GFXrc6 */ - if (DO_BIC(BIC_GFX_rc6)) { - if (p->gfx_rc6_ms == -1) { /* detect GFX counter reset */ - outp += sprintf(outp, "%s**.**", (printed++ ? delim : "")); - } else { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - p->gfx_rc6_ms / 10.0 / interval_float); - } - } + fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", base_cpu, msr); - /* GFXMHz */ - if (DO_BIC(BIC_GFXMHz)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_mhz); + ratio = (msr >> 40) & 0xFF; + fprintf(outf, "%d * %.1f = %.1f MHz max efficiency frequency\n", ratio, bclk, ratio * bclk); - /* GFXACTMHz */ - if (DO_BIC(BIC_GFXACTMHz)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_act_mhz); + ratio = (msr >> 8) & 0xFF; + fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk); +} - /* Totl%C0, Any%C0 GFX%C0 CPUGFX% */ - if (DO_BIC(BIC_Totl_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_wtd_core_c0 / tsc); - if (DO_BIC(BIC_Any_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_any_core_c0 / tsc); - if (DO_BIC(BIC_GFX_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_any_gfxe_c0 / tsc); - if (DO_BIC(BIC_CPUGFX)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_both_core_gfxe_c0 / tsc); +static void dump_power_ctl(void) +{ + unsigned long long msr; - if (DO_BIC(BIC_Pkgpc2)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc2 / tsc); - if (DO_BIC(BIC_Pkgpc3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc3 / tsc); - if (DO_BIC(BIC_Pkgpc6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc6 / tsc); - if (DO_BIC(BIC_Pkgpc7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc7 / tsc); - if (DO_BIC(BIC_Pkgpc8)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc8 / tsc); - if (DO_BIC(BIC_Pkgpc9)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc9 / tsc); - if (DO_BIC(BIC_Pkgpc10)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc10 / tsc); + if (!platform->has_nhm_msrs || no_msr) + return; - if (DO_BIC(BIC_CPU_LPI)) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->cpu_lpi / 1000000.0 / interval_float); - if (DO_BIC(BIC_SYS_LPI)) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->sys_lpi / 1000000.0 / interval_float); + get_msr(base_cpu, MSR_IA32_POWER_CTL, &msr); + fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", base_cpu, msr, msr & 0x2 ? "EN" : "DIS"); - if (DO_BIC(BIC_PkgWatt)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_pkg * rapl_energy_units / interval_float); + /* C-state Pre-wake Disable (CSTATE_PREWAKE_DISABLE) */ + if (platform->has_cst_prewake_bit) + fprintf(outf, "C-state Pre-wake: %sabled\n", msr & 0x40000000 ? "DIS" : "EN"); - if (DO_BIC(BIC_CorWatt) && !(do_rapl & RAPL_PER_CORE_ENERGY)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_cores * rapl_energy_units / interval_float); - if (DO_BIC(BIC_GFXWatt)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_gfx * rapl_energy_units / interval_float); - if (DO_BIC(BIC_RAMWatt)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - p->energy_dram * rapl_dram_energy_units / interval_float); - if (DO_BIC(BIC_Pkg_J)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_pkg * rapl_energy_units); - if (DO_BIC(BIC_Cor_J) && !(do_rapl & RAPL_PER_CORE_ENERGY)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_cores * rapl_energy_units); - if (DO_BIC(BIC_GFX_J)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_gfx * rapl_energy_units); - if (DO_BIC(BIC_RAM_J)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), p->energy_dram * rapl_dram_energy_units); - if (DO_BIC(BIC_PKG__)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - 100.0 * p->rapl_pkg_perf_status * rapl_time_units / interval_float); - if (DO_BIC(BIC_RAM__)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - 100.0 * p->rapl_dram_perf_status * rapl_time_units / interval_float); - /* UncMHz */ - if (DO_BIC(BIC_UNCORE_MHZ)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->uncore_mhz); + return; +} - for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) { - if (mp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)p->counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->counter[i]); - } else if (mp->format == FORMAT_DELTA) { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), p->counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), p->counter[i]); - } else if (mp->format == FORMAT_PERCENT) { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->counter[i] / tsc); - } +static void dump_turbo_ratio_limit2(void) +{ + unsigned long long msr; + unsigned int ratio; + + get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT2, &msr); + + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", base_cpu, msr); + + ratio = (msr >> 8) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 18 active cores\n", ratio, bclk, ratio * bclk); + + ratio = (msr >> 0) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 17 active cores\n", ratio, bclk, ratio * bclk); + return; +} + +static void dump_turbo_ratio_limit1(void) +{ + unsigned long long msr; + unsigned int ratio; + + get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &msr); + + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, msr); + + ratio = (msr >> 56) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 16 active cores\n", ratio, bclk, ratio * bclk); + + ratio = (msr >> 48) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 15 active cores\n", ratio, bclk, ratio * bclk); + + ratio = (msr >> 40) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 14 active cores\n", ratio, bclk, ratio * bclk); + + ratio = (msr >> 32) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 13 active cores\n", ratio, bclk, ratio * bclk); + + ratio = (msr >> 24) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 12 active cores\n", ratio, bclk, ratio * bclk); + + ratio = (msr >> 16) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 11 active cores\n", ratio, bclk, ratio * bclk); + + ratio = (msr >> 8) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 10 active cores\n", ratio, bclk, ratio * bclk); + + ratio = (msr >> 0) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 9 active cores\n", ratio, bclk, ratio * bclk); + return; +} + +static void dump_turbo_ratio_limits(int trl_msr_offset) +{ + unsigned long long msr, core_counts; + int shift; + + get_msr(base_cpu, trl_msr_offset, &msr); + fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n", + base_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr); + + if (platform->trl_msrs & TRL_CORECOUNT) { + get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts); + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, core_counts); + } else { + core_counts = 0x0807060504030201; + } + + for (shift = 56; shift >= 0; shift -= 8) { + unsigned int ratio, group_size; + + ratio = (msr >> shift) & 0xFF; + group_size = (core_counts >> shift) & 0xFF; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio, bclk, ratio * bclk, group_size); } -done: - if (*(outp - 1) != '\n') - outp += sprintf(outp, "\n"); - - return 0; + return; } -void flush_output_stdout(void) +static void dump_atom_turbo_ratio_limits(void) { - FILE *filep; - - if (outf == stderr) - filep = stdout; - else - filep = outf; + unsigned long long msr; + unsigned int ratio; - fputs(output_buffer, filep); - fflush(filep); + get_msr(base_cpu, MSR_ATOM_CORE_RATIOS, &msr); + fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF); - outp = output_buffer; -} + ratio = (msr >> 0) & 0x3F; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz minimum operating frequency\n", ratio, bclk, ratio * bclk); -void flush_output_stderr(void) -{ - fputs(output_buffer, outf); - fflush(outf); - outp = output_buffer; -} + ratio = (msr >> 8) & 0x3F; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz low frequency mode (LFM)\n", ratio, bclk, ratio * bclk); -void format_all_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) -{ - static int count; + ratio = (msr >> 16) & 0x3F; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk); - if ((!count || (header_iterations && !(count % header_iterations))) || !summary_only) - print_header("\t"); + get_msr(base_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr); + fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF); - format_counters(&average.threads, &average.cores, &average.packages); + ratio = (msr >> 24) & 0x3F; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 4 active cores\n", ratio, bclk, ratio * bclk); - count++; + ratio = (msr >> 16) & 0x3F; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 3 active cores\n", ratio, bclk, ratio * bclk); - if (summary_only) - return; + ratio = (msr >> 8) & 0x3F; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 2 active cores\n", ratio, bclk, ratio * bclk); - for_all_cpus(format_counters, t, c, p); + ratio = (msr >> 0) & 0x3F; + if (ratio) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo 1 active core\n", ratio, bclk, ratio * bclk); } -#define DELTA_WRAP32(new, old) \ - old = ((((unsigned long long)new << 32) - ((unsigned long long)old << 32)) >> 32); - -int delta_package(struct pkg_data *new, struct pkg_data *old) +static void dump_knl_turbo_ratio_limits(void) { - int i; - struct msr_counter *mp; + const unsigned int buckets_no = 7; - if (DO_BIC(BIC_Totl_c0)) - old->pkg_wtd_core_c0 = new->pkg_wtd_core_c0 - old->pkg_wtd_core_c0; - if (DO_BIC(BIC_Any_c0)) - old->pkg_any_core_c0 = new->pkg_any_core_c0 - old->pkg_any_core_c0; - if (DO_BIC(BIC_GFX_c0)) - old->pkg_any_gfxe_c0 = new->pkg_any_gfxe_c0 - old->pkg_any_gfxe_c0; - if (DO_BIC(BIC_CPUGFX)) - old->pkg_both_core_gfxe_c0 = new->pkg_both_core_gfxe_c0 - old->pkg_both_core_gfxe_c0; + unsigned long long msr; + int delta_cores, delta_ratio; + int i, b_nr; + unsigned int cores[buckets_no]; + unsigned int ratio[buckets_no]; - old->pc2 = new->pc2 - old->pc2; - if (DO_BIC(BIC_Pkgpc3)) - old->pc3 = new->pc3 - old->pc3; - if (DO_BIC(BIC_Pkgpc6)) - old->pc6 = new->pc6 - old->pc6; - if (DO_BIC(BIC_Pkgpc7)) - old->pc7 = new->pc7 - old->pc7; - old->pc8 = new->pc8 - old->pc8; - old->pc9 = new->pc9 - old->pc9; - old->pc10 = new->pc10 - old->pc10; - old->cpu_lpi = new->cpu_lpi - old->cpu_lpi; - old->sys_lpi = new->sys_lpi - old->sys_lpi; - old->pkg_temp_c = new->pkg_temp_c; + get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT, &msr); - /* flag an error when rc6 counter resets/wraps */ - if (old->gfx_rc6_ms > new->gfx_rc6_ms) - old->gfx_rc6_ms = -1; - else - old->gfx_rc6_ms = new->gfx_rc6_ms - old->gfx_rc6_ms; + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", base_cpu, msr); - old->uncore_mhz = new->uncore_mhz; - old->gfx_mhz = new->gfx_mhz; - old->gfx_act_mhz = new->gfx_act_mhz; + /* + * Turbo encoding in KNL is as follows: + * [0] -- Reserved + * [7:1] -- Base value of number of active cores of bucket 1. + * [15:8] -- Base value of freq ratio of bucket 1. + * [20:16] -- +ve delta of number of active cores of bucket 2. + * i.e. active cores of bucket 2 = + * active cores of bucket 1 + delta + * [23:21] -- Negative delta of freq ratio of bucket 2. + * i.e. freq ratio of bucket 2 = + * freq ratio of bucket 1 - delta + * [28:24]-- +ve delta of number of active cores of bucket 3. + * [31:29]-- -ve delta of freq ratio of bucket 3. + * [36:32]-- +ve delta of number of active cores of bucket 4. + * [39:37]-- -ve delta of freq ratio of bucket 4. + * [44:40]-- +ve delta of number of active cores of bucket 5. + * [47:45]-- -ve delta of freq ratio of bucket 5. + * [52:48]-- +ve delta of number of active cores of bucket 6. + * [55:53]-- -ve delta of freq ratio of bucket 6. + * [60:56]-- +ve delta of number of active cores of bucket 7. + * [63:61]-- -ve delta of freq ratio of bucket 7. + */ + + b_nr = 0; + cores[b_nr] = (msr & 0xFF) >> 1; + ratio[b_nr] = (msr >> 8) & 0xFF; - old->energy_pkg = new->energy_pkg - old->energy_pkg; - old->energy_cores = new->energy_cores - old->energy_cores; - old->energy_gfx = new->energy_gfx - old->energy_gfx; - old->energy_dram = new->energy_dram - old->energy_dram; - old->rapl_pkg_perf_status = new->rapl_pkg_perf_status - old->rapl_pkg_perf_status; - old->rapl_dram_perf_status = new->rapl_dram_perf_status - old->rapl_dram_perf_status; + for (i = 16; i < 64; i += 8) { + delta_cores = (msr >> i) & 0x1F; + delta_ratio = (msr >> (i + 5)) & 0x7; - for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) - old->counter[i] = new->counter[i]; - else - old->counter[i] = new->counter[i] - old->counter[i]; + cores[b_nr + 1] = cores[b_nr] + delta_cores; + ratio[b_nr + 1] = ratio[b_nr] - delta_ratio; + b_nr++; } - return 0; + for (i = buckets_no - 1; i >= 0; i--) + if (i > 0 ? ratio[i] != ratio[i - 1] : 1) + fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio[i], bclk, ratio[i] * bclk, cores[i]); } -void delta_core(struct core_data *new, struct core_data *old) +static void dump_cst_cfg(void) { - int i; - struct msr_counter *mp; + unsigned long long msr; - old->c3 = new->c3 - old->c3; - old->c6 = new->c6 - old->c6; - old->c7 = new->c7 - old->c7; - old->core_temp_c = new->core_temp_c; - old->core_throt_cnt = new->core_throt_cnt; - old->mc6_us = new->mc6_us - old->mc6_us; + if (!platform->has_nhm_msrs || no_msr) + return; - DELTA_WRAP32(new->core_energy, old->core_energy); + get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); - for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) - old->counter[i] = new->counter[i]; - else - old->counter[i] = new->counter[i] - old->counter[i]; + fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", base_cpu, msr); + + fprintf(outf, " (%s%s%s%s%slocked, pkg-cstate-limit=%d (%s)", + (msr & SNB_C3_AUTO_UNDEMOTE) ? "UNdemote-C3, " : "", + (msr & SNB_C1_AUTO_UNDEMOTE) ? "UNdemote-C1, " : "", + (msr & NHM_C3_AUTO_DEMOTE) ? "demote-C3, " : "", + (msr & NHM_C1_AUTO_DEMOTE) ? "demote-C1, " : "", + (msr & (1 << 15)) ? "" : "UN", (unsigned int)msr & 0xF, pkg_cstate_limit_strings[pkg_cstate_limit]); + +#define AUTOMATIC_CSTATE_CONVERSION (1UL << 16) + if (platform->has_cst_auto_convension) { + fprintf(outf, ", automatic c-state conversion=%s", (msr & AUTOMATIC_CSTATE_CONVERSION) ? "on" : "off"); } -} -int soft_c1_residency_display(int bic) -{ - if (!DO_BIC(BIC_CPU_c1) || use_c1_residency_msr) - return 0; + fprintf(outf, ")\n"); - return DO_BIC_READ(bic); + return; } -/* - * old = new - old - */ -int delta_thread(struct thread_data *new, struct thread_data *old, struct core_data *core_delta) +static void dump_config_tdp(void) { - int i; - struct msr_counter *mp; + unsigned long long msr; - /* we run cpuid just the 1st time, copy the results */ - if (DO_BIC(BIC_APIC)) - new->apic_id = old->apic_id; - if (DO_BIC(BIC_X2APIC)) - new->x2apic_id = old->x2apic_id; + get_msr(base_cpu, MSR_CONFIG_TDP_NOMINAL, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", base_cpu, msr); + fprintf(outf, " (base_ratio=%d)\n", (unsigned int)msr & 0xFF); - /* - * the timestamps from start of measurement interval are in "old" - * the timestamp from end of measurement interval are in "new" - * over-write old w/ new so we can print end of interval values - */ + get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_1, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", base_cpu, msr); + if (msr) { + fprintf(outf, "PKG_MIN_PWR_LVL1=%d ", (unsigned int)(msr >> 48) & 0x7FFF); + fprintf(outf, "PKG_MAX_PWR_LVL1=%d ", (unsigned int)(msr >> 32) & 0x7FFF); + fprintf(outf, "LVL1_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF); + fprintf(outf, "PKG_TDP_LVL1=%d", (unsigned int)(msr) & 0x7FFF); + } + fprintf(outf, ")\n"); - timersub(&new->tv_begin, &old->tv_begin, &old->tv_delta); - old->tv_begin = new->tv_begin; - old->tv_end = new->tv_end; + get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_2, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", base_cpu, msr); + if (msr) { + fprintf(outf, "PKG_MIN_PWR_LVL2=%d ", (unsigned int)(msr >> 48) & 0x7FFF); + fprintf(outf, "PKG_MAX_PWR_LVL2=%d ", (unsigned int)(msr >> 32) & 0x7FFF); + fprintf(outf, "LVL2_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF); + fprintf(outf, "PKG_TDP_LVL2=%d", (unsigned int)(msr) & 0x7FFF); + } + fprintf(outf, ")\n"); - old->tsc = new->tsc - old->tsc; + get_msr(base_cpu, MSR_CONFIG_TDP_CONTROL, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", base_cpu, msr); + if ((msr) & 0x3) + fprintf(outf, "TDP_LEVEL=%d ", (unsigned int)(msr) & 0x3); + fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1); + fprintf(outf, ")\n"); - /* check for TSC < 1 Mcycles over interval */ - if (old->tsc < (1000 * 1000)) - errx(-3, "Insanely slow TSC rate, TSC stops in idle?\n" - "You can disable all c-states by booting with \"idle=poll\"\n" - "or just the deep ones with \"processor.max_cstate=1\""); + get_msr(base_cpu, MSR_TURBO_ACTIVATION_RATIO, &msr); + fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", base_cpu, msr); + fprintf(outf, "MAX_NON_TURBO_RATIO=%d", (unsigned int)(msr) & 0xFF); + fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1); + fprintf(outf, ")\n"); +} - old->c1 = new->c1 - old->c1; +unsigned int irtl_time_units[] = { 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 }; - if (DO_BIC(BIC_Avg_MHz) || DO_BIC(BIC_Busy) || DO_BIC(BIC_Bzy_MHz) || soft_c1_residency_display(BIC_Avg_MHz)) { - if ((new->aperf > old->aperf) && (new->mperf > old->mperf)) { - old->aperf = new->aperf - old->aperf; - old->mperf = new->mperf - old->mperf; - } else { - return -1; - } +void print_irtl(void) +{ + unsigned long long msr; + + if (!platform->has_irtl_msrs || no_msr) + return; + + if (platform->supported_cstates & PC3) { + get_msr(base_cpu, MSR_PKGC3_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", base_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + } + + if (platform->supported_cstates & PC6) { + get_msr(base_cpu, MSR_PKGC6_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", base_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + } + + if (platform->supported_cstates & PC7) { + get_msr(base_cpu, MSR_PKGC7_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", base_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + } + + if (platform->supported_cstates & PC8) { + get_msr(base_cpu, MSR_PKGC8_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", base_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } - if (use_c1_residency_msr) { - /* - * Some models have a dedicated C1 residency MSR, - * which should be more accurate than the derivation below. - */ - } else { - /* - * As counter collection is not atomic, - * it is possible for mperf's non-halted cycles + idle states - * to exceed TSC's all cycles: show c1 = 0% in that case. - */ - if ((old->mperf + core_delta->c3 + core_delta->c6 + core_delta->c7) > (old->tsc * tsc_tweak)) - old->c1 = 0; - else { - /* normal case, derive c1 */ - old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3 - - core_delta->c6 - core_delta->c7; - } + if (platform->supported_cstates & PC9) { + get_msr(base_cpu, MSR_PKGC9_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", base_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } - if (old->mperf == 0) { - if (debug > 1) - fprintf(outf, "cpu%d MPERF 0!\n", old->cpu_id); - old->mperf = 1; /* divide by 0 protection */ + if (platform->supported_cstates & PC10) { + get_msr(base_cpu, MSR_PKGC10_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", base_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } +} - if (DO_BIC(BIC_IPC)) - old->instr_count = new->instr_count - old->instr_count; +void free_fd_percpu(void) +{ + int i; - if (DO_BIC(BIC_IRQ)) - old->irq_count = new->irq_count - old->irq_count; + if (!fd_percpu) + return; - if (DO_BIC(BIC_SMI)) - old->smi_count = new->smi_count - old->smi_count; + for (i = 0; i < topo.max_cpu_num + 1; ++i) { + if (fd_percpu[i] != 0) + close(fd_percpu[i]); + } - for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) - old->counter[i] = new->counter[i]; - else - old->counter[i] = new->counter[i] - old->counter[i]; + free(fd_percpu); + fd_percpu = NULL; +} + +void free_fd_instr_count_percpu(void) +{ + if (!fd_instr_count_percpu) + return; + + for (int i = 0; i < topo.max_cpu_num + 1; ++i) { + if (fd_instr_count_percpu[i] != 0) + close(fd_instr_count_percpu[i]); } - return 0; + + free(fd_instr_count_percpu); + fd_instr_count_percpu = NULL; } -int delta_cpu(struct thread_data *t, struct core_data *c, - struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2) +void free_fd_llc_percpu(void) { - int retval = 0; + if (!fd_llc_percpu) + return; - /* calculate core delta only for 1st thread in core */ - if (t->flags & CPU_IS_FIRST_THREAD_IN_CORE) - delta_core(c, c2); + for (int i = 0; i < topo.max_cpu_num + 1; ++i) { + if (fd_llc_percpu[i] != 0) + close(fd_llc_percpu[i]); + } - /* always calculate thread delta */ - retval = delta_thread(t, t2, c2); /* c2 is core delta */ - if (retval) - return retval; + free(fd_llc_percpu); + fd_llc_percpu = NULL; +} - /* calculate package delta only for 1st core in package */ - if (t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE) - retval = delta_package(p, p2); +void free_fd_cstate(void) +{ + if (!ccstate_counter_info) + return; - return retval; + const int counter_info_num = ccstate_counter_info_size; + + for (int counter_id = 0; counter_id < counter_info_num; ++counter_id) { + if (ccstate_counter_info[counter_id].fd_perf_core != -1) + close(ccstate_counter_info[counter_id].fd_perf_core); + + if (ccstate_counter_info[counter_id].fd_perf_pkg != -1) + close(ccstate_counter_info[counter_id].fd_perf_pkg); + } + + free(ccstate_counter_info); + ccstate_counter_info = NULL; + ccstate_counter_info_size = 0; } -void clear_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) +void free_fd_msr(void) { - int i; - struct msr_counter *mp; + if (!msr_counter_info) + return; - t->tv_begin.tv_sec = 0; - t->tv_begin.tv_usec = 0; - t->tv_end.tv_sec = 0; - t->tv_end.tv_usec = 0; - t->tv_delta.tv_sec = 0; - t->tv_delta.tv_usec = 0; + for (int cpu = 0; cpu < topo.max_cpu_num; ++cpu) { + if (msr_counter_info[cpu].fd_perf != -1) + close(msr_counter_info[cpu].fd_perf); + } - t->tsc = 0; - t->aperf = 0; - t->mperf = 0; - t->c1 = 0; + free(msr_counter_info); + msr_counter_info = NULL; + msr_counter_info_size = 0; +} - t->instr_count = 0; +void free_fd_rapl_percpu(void) +{ + if (!rapl_counter_info_perdomain) + return; - t->irq_count = 0; - t->smi_count = 0; + const int num_domains = rapl_counter_info_perdomain_size; - /* tells format_counters to dump all fields from this set */ - t->flags = CPU_IS_FIRST_THREAD_IN_CORE | CPU_IS_FIRST_CORE_IN_PACKAGE; + for (int domain_id = 0; domain_id < num_domains; ++domain_id) { + if (rapl_counter_info_perdomain[domain_id].fd_perf != -1) + close(rapl_counter_info_perdomain[domain_id].fd_perf); + } - c->c3 = 0; - c->c6 = 0; - c->c7 = 0; - c->mc6_us = 0; - c->core_temp_c = 0; - c->core_energy = 0; - c->core_throt_cnt = 0; + free(rapl_counter_info_perdomain); + rapl_counter_info_perdomain = NULL; + rapl_counter_info_perdomain_size = 0; +} - p->pkg_wtd_core_c0 = 0; - p->pkg_any_core_c0 = 0; - p->pkg_any_gfxe_c0 = 0; - p->pkg_both_core_gfxe_c0 = 0; +void free_fd_added_perf_counters_(struct perf_counter_info *pp) +{ + if (!pp) + return; - p->pc2 = 0; - if (DO_BIC(BIC_Pkgpc3)) - p->pc3 = 0; - if (DO_BIC(BIC_Pkgpc6)) - p->pc6 = 0; - if (DO_BIC(BIC_Pkgpc7)) - p->pc7 = 0; - p->pc8 = 0; - p->pc9 = 0; - p->pc10 = 0; - p->cpu_lpi = 0; - p->sys_lpi = 0; + if (!pp->fd_perf_per_domain) + return; - p->energy_pkg = 0; - p->energy_dram = 0; - p->energy_cores = 0; - p->energy_gfx = 0; - p->rapl_pkg_perf_status = 0; - p->rapl_dram_perf_status = 0; - p->pkg_temp_c = 0; + while (pp) { + for (size_t domain = 0; domain < pp->num_domains; ++domain) { + if (pp->fd_perf_per_domain[domain] != -1) { + close(pp->fd_perf_per_domain[domain]); + pp->fd_perf_per_domain[domain] = -1; + } + } - p->gfx_rc6_ms = 0; - p->uncore_mhz = 0; - p->gfx_mhz = 0; - p->gfx_act_mhz = 0; - for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) - t->counter[i] = 0; + free(pp->fd_perf_per_domain); + pp->fd_perf_per_domain = NULL; - for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) - c->counter[i] = 0; + pp = pp->next; + } +} - for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) - p->counter[i] = 0; +void free_fd_added_perf_counters(void) +{ + free_fd_added_perf_counters_(sys.perf_tp); + free_fd_added_perf_counters_(sys.perf_cp); + free_fd_added_perf_counters_(sys.perf_pp); } -int sum_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) +void free_all_buffers(void) { int i; - struct msr_counter *mp; - /* copy un-changing apic_id's */ - if (DO_BIC(BIC_APIC)) - average.threads.apic_id = t->apic_id; - if (DO_BIC(BIC_X2APIC)) - average.threads.x2apic_id = t->x2apic_id; + CPU_FREE(cpu_present_set); + cpu_present_set = NULL; + cpu_present_setsize = 0; - /* remember first tv_begin */ - if (average.threads.tv_begin.tv_sec == 0) - average.threads.tv_begin = t->tv_begin; + CPU_FREE(cpu_effective_set); + cpu_effective_set = NULL; + cpu_effective_setsize = 0; - /* remember last tv_end */ - average.threads.tv_end = t->tv_end; + CPU_FREE(cpu_allowed_set); + cpu_allowed_set = NULL; + cpu_allowed_setsize = 0; - average.threads.tsc += t->tsc; - average.threads.aperf += t->aperf; - average.threads.mperf += t->mperf; - average.threads.c1 += t->c1; + CPU_FREE(cpu_affinity_set); + cpu_affinity_set = NULL; + cpu_affinity_setsize = 0; - average.threads.instr_count += t->instr_count; + free(thread_even); + free(core_even); + free(package_even); - average.threads.irq_count += t->irq_count; - average.threads.smi_count += t->smi_count; + thread_even = NULL; + core_even = NULL; + package_even = NULL; - for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) - continue; - average.threads.counter[i] += t->counter[i]; - } + free(thread_odd); + free(core_odd); + free(package_odd); - /* sum per-core values only for 1st thread in core */ - if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE)) - return 0; + thread_odd = NULL; + core_odd = NULL; + package_odd = NULL; - average.cores.c3 += c->c3; - average.cores.c6 += c->c6; - average.cores.c7 += c->c7; - average.cores.mc6_us += c->mc6_us; + free(output_buffer); + output_buffer = NULL; + outp = NULL; - average.cores.core_temp_c = MAX(average.cores.core_temp_c, c->core_temp_c); - average.cores.core_throt_cnt = MAX(average.cores.core_throt_cnt, c->core_throt_cnt); + free_fd_percpu(); + free_fd_instr_count_percpu(); + free_fd_llc_percpu(); + free_fd_msr(); + free_fd_rapl_percpu(); + free_fd_cstate(); + free_fd_added_perf_counters(); - average.cores.core_energy += c->core_energy; + free(irq_column_2_cpu); + free(irqs_per_cpu); + free(nmi_per_cpu); - for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) - continue; - average.cores.counter[i] += c->counter[i]; + for (i = 0; i <= topo.max_cpu_num; ++i) { + if (cpus[i].put_ids) + CPU_FREE(cpus[i].put_ids); } + free(cpus); +} - /* sum per-pkg values only for 1st core in pkg */ - if (!(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) +/* + * Parse a file containing a single int. + * Return 0 if file can not be opened + * Exit if file can be opened, but can not be parsed + */ +int parse_int_file(const char *fmt, ...) +{ + va_list args; + char path[PATH_MAX]; + FILE *filep; + int value; + + va_start(args, fmt); + vsnprintf(path, sizeof(path), fmt, args); + va_end(args); + filep = fopen(path, "r"); + if (!filep) return 0; + if (fscanf(filep, "%d", &value) != 1) + err(1, "%s: failed to parse number from file", path); + fclose(filep); + return value; +} - if (DO_BIC(BIC_Totl_c0)) - average.packages.pkg_wtd_core_c0 += p->pkg_wtd_core_c0; - if (DO_BIC(BIC_Any_c0)) - average.packages.pkg_any_core_c0 += p->pkg_any_core_c0; - if (DO_BIC(BIC_GFX_c0)) - average.packages.pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0; - if (DO_BIC(BIC_CPUGFX)) - average.packages.pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0; +/* + * cpu_is_first_core_in_package(cpu) + * return 1 if given CPU is 1st core in package + */ +int cpu_is_first_core_in_package(int cpu) +{ + return cpu == parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_siblings_list", cpu); +} - average.packages.pc2 += p->pc2; - if (DO_BIC(BIC_Pkgpc3)) - average.packages.pc3 += p->pc3; - if (DO_BIC(BIC_Pkgpc6)) - average.packages.pc6 += p->pc6; - if (DO_BIC(BIC_Pkgpc7)) - average.packages.pc7 += p->pc7; - average.packages.pc8 += p->pc8; - average.packages.pc9 += p->pc9; - average.packages.pc10 += p->pc10; +int get_physical_package_id(int cpu) +{ + return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu); +} - average.packages.cpu_lpi = p->cpu_lpi; - average.packages.sys_lpi = p->sys_lpi; +int get_die_id(int cpu) +{ + return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/die_id", cpu); +} - average.packages.energy_pkg += p->energy_pkg; - average.packages.energy_dram += p->energy_dram; - average.packages.energy_cores += p->energy_cores; - average.packages.energy_gfx += p->energy_gfx; +int get_l3_id(int cpu) +{ + return parse_int_file("/sys/devices/system/cpu/cpu%d/cache/index3/id", cpu); +} - average.packages.gfx_rc6_ms = p->gfx_rc6_ms; - average.packages.uncore_mhz = p->uncore_mhz; - average.packages.gfx_mhz = p->gfx_mhz; - average.packages.gfx_act_mhz = p->gfx_act_mhz; +int get_core_id(int cpu) +{ + return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_id", cpu); +} - average.packages.pkg_temp_c = MAX(average.packages.pkg_temp_c, p->pkg_temp_c); +void set_node_data(void) +{ + int pkg, node, lnode, cpu, cpux; + int cpu_count; - average.packages.rapl_pkg_perf_status += p->rapl_pkg_perf_status; - average.packages.rapl_dram_perf_status += p->rapl_dram_perf_status; + /* initialize logical_node_id */ + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) + cpus[cpu].logical_node_id = -1; - for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if ((mp->format == FORMAT_RAW) && (topo.num_packages == 0)) - average.packages.counter[i] = p->counter[i]; - else - average.packages.counter[i] += p->counter[i]; + cpu_count = 0; + for (pkg = 0; pkg < topo.num_packages; pkg++) { + lnode = 0; + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { + if (cpus[cpu].physical_package_id != pkg) + continue; + /* find a cpu with an unset logical_node_id */ + if (cpus[cpu].logical_node_id != -1) + continue; + cpus[cpu].logical_node_id = lnode; + node = cpus[cpu].physical_node_id; + cpu_count++; + /* + * find all matching cpus on this pkg and set + * the logical_node_id + */ + for (cpux = cpu; cpux <= topo.max_cpu_num; cpux++) { + if ((cpus[cpux].physical_package_id == pkg) && (cpus[cpux].physical_node_id == node)) { + cpus[cpux].logical_node_id = lnode; + cpu_count++; + } + } + lnode++; + if (lnode > topo.nodes_per_pkg) + topo.nodes_per_pkg = lnode; + } + if (cpu_count >= topo.max_cpu_num) + break; } - return 0; } -/* - * sum the counters for all cpus in the system - * compute the weighted average - */ -void compute_average(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int get_physical_node_id(struct cpu_topology *thiscpu) { + char path[80]; + FILE *filep; int i; - struct msr_counter *mp; - - clear_counters(&average.threads, &average.cores, &average.packages); + int cpu = thiscpu->logical_cpu_id; - for_all_cpus(sum_counters, t, c, p); + for (i = 0; i <= topo.max_cpu_num; i++) { + sprintf(path, "/sys/devices/system/cpu/cpu%d/node%i/cpulist", cpu, i); + filep = fopen(path, "r"); + if (!filep) + continue; + fclose(filep); + return i; + } + return -1; +} - /* Use the global time delta for the average. */ - average.threads.tv_delta = tv_delta; +static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size) +{ + unsigned int start, end; + char *next = cpu_str; - average.threads.tsc /= topo.num_cpus; - average.threads.aperf /= topo.num_cpus; - average.threads.mperf /= topo.num_cpus; - average.threads.instr_count /= topo.num_cpus; - average.threads.c1 /= topo.num_cpus; + while (next && *next) { - if (average.threads.irq_count > 9999999) - sums_need_wide_columns = 1; + if (*next == '-') /* no negative cpu numbers */ + return 1; - average.cores.c3 /= topo.num_cores; - average.cores.c6 /= topo.num_cores; - average.cores.c7 /= topo.num_cores; - average.cores.mc6_us /= topo.num_cores; + if (*next == '\0' || *next == '\n') + break; - if (DO_BIC(BIC_Totl_c0)) - average.packages.pkg_wtd_core_c0 /= topo.num_packages; - if (DO_BIC(BIC_Any_c0)) - average.packages.pkg_any_core_c0 /= topo.num_packages; - if (DO_BIC(BIC_GFX_c0)) - average.packages.pkg_any_gfxe_c0 /= topo.num_packages; - if (DO_BIC(BIC_CPUGFX)) - average.packages.pkg_both_core_gfxe_c0 /= topo.num_packages; + start = strtoul(next, &next, 10); - average.packages.pc2 /= topo.num_packages; - if (DO_BIC(BIC_Pkgpc3)) - average.packages.pc3 /= topo.num_packages; - if (DO_BIC(BIC_Pkgpc6)) - average.packages.pc6 /= topo.num_packages; - if (DO_BIC(BIC_Pkgpc7)) - average.packages.pc7 /= topo.num_packages; + if (start >= CPU_SUBSET_MAXCPUS) + return 1; + CPU_SET_S(start, cpu_set_size, cpu_set); - average.packages.pc8 /= topo.num_packages; - average.packages.pc9 /= topo.num_packages; - average.packages.pc10 /= topo.num_packages; + if (*next == '\0' || *next == '\n') + break; - for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) - continue; - if (mp->type == COUNTER_ITEMS) { - if (average.threads.counter[i] > 9999999) - sums_need_wide_columns = 1; + if (*next == ',') { + next += 1; continue; } - average.threads.counter[i] /= topo.num_cpus; - } - for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) - continue; - if (mp->type == COUNTER_ITEMS) { - if (average.cores.counter[i] > 9999999) - sums_need_wide_columns = 1; + + if (*next == '-') { + next += 1; /* start range */ + } else if (*next == '.') { + next += 1; + if (*next == '.') + next += 1; /* start range */ + else + return 1; } - average.cores.counter[i] /= topo.num_cores; - } - for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) - continue; - if (mp->type == COUNTER_ITEMS) { - if (average.packages.counter[i] > 9999999) - sums_need_wide_columns = 1; + + end = strtoul(next, &next, 10); + if (end <= start) + return 1; + + while (++start <= end) { + if (start >= CPU_SUBSET_MAXCPUS) + return 1; + CPU_SET_S(start, cpu_set_size, cpu_set); } - average.packages.counter[i] /= topo.num_packages; + + if (*next == ',') + next += 1; + else if (*next != '\0' && *next != '\n') + return 1; } + + return 0; } -static unsigned long long rdtsc(void) +int get_thread_siblings(struct cpu_topology *thiscpu) { - unsigned int low, high; + char path[80], character; + FILE *filep; + unsigned long map; + int so, shift, sib_core; + int cpu = thiscpu->logical_cpu_id; + int offset = topo.max_cpu_num + 1; + size_t size; + int thread_id = 0; - asm volatile ("rdtsc":"=a" (low), "=d"(high)); + thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1)); + if (thiscpu->thread_id < 0) + thiscpu->thread_id = thread_id++; + if (!thiscpu->put_ids) + return -1; - return low | ((unsigned long long)high) << 32; + size = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); + CPU_ZERO_S(size, thiscpu->put_ids); + + sprintf(path, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", cpu); + filep = fopen(path, "r"); + + if (!filep) { + warnx("%s: open failed", path); + return -1; + } + do { + offset -= BITMASK_SIZE; + if (fscanf(filep, "%lx%c", &map, &character) != 2) + err(1, "%s: failed to parse file", path); + for (shift = 0; shift < BITMASK_SIZE; shift++) { + if ((map >> shift) & 0x1) { + so = shift + offset; + sib_core = get_core_id(so); + if (sib_core == thiscpu->physical_core_id) { + CPU_SET_S(so, size, thiscpu->put_ids); + if ((so != cpu) && (cpus[so].thread_id < 0)) + cpus[so].thread_id = thread_id++; + } + } + } + } while (character == ','); + fclose(filep); + + return CPU_COUNT_S(size, thiscpu->put_ids); } /* - * Open a file, and exit on failure + * run func(thread, core, package) in topology order + * skip non-present cpus */ -FILE *fopen_or_die(const char *path, const char *mode) + +int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *, + struct pkg_data *, struct thread_data *, struct core_data *, + struct pkg_data *), struct thread_data *thread_base, + struct core_data *core_base, struct pkg_data *pkg_base, + struct thread_data *thread_base2, struct core_data *core_base2, struct pkg_data *pkg_base2) { - FILE *filep = fopen(path, mode); + int retval, pkg_no, node_no, core_no, thread_no; - if (!filep) - err(1, "%s: open failed", path); - return filep; + retval = 0; + + for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) { + for (node_no = 0; node_no < topo.nodes_per_pkg; ++node_no) { + for (core_no = 0; core_no < topo.cores_per_node; ++core_no) { + for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) { + struct thread_data *t, *t2; + struct core_data *c, *c2; + + t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no); + + if (cpu_is_not_allowed(t->cpu_id)) + continue; + + t2 = GET_THREAD(thread_base2, thread_no, core_no, node_no, pkg_no); + + c = GET_CORE(core_base, core_no, node_no, pkg_no); + c2 = GET_CORE(core_base2, core_no, node_no, pkg_no); + + retval |= func(t, c, &pkg_base[pkg_no], t2, c2, &pkg_base2[pkg_no]); + } + } + } + } + return retval; } /* - * snapshot_sysfs_counter() - * - * return snapshot of given counter + * run func(cpu) on every cpu in /proc/stat + * return max_cpu number */ -unsigned long long snapshot_sysfs_counter(char *path) +int for_all_proc_cpus(int (func) (int)) { FILE *fp; + int cpu_num; int retval; - unsigned long long counter; - - fp = fopen_or_die(path, "r"); - - retval = fscanf(fp, "%lld", &counter); - if (retval != 1) - err(1, "snapshot_sysfs_counter(%s)", path); - - fclose(fp); - return counter; -} + fp = fopen_or_die(proc_stat, "r"); -int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp) -{ - if (mp->msr_num != 0) { - if (get_msr(cpu, mp->msr_num, counterp)) - return -1; - } else { - char path[128 + PATH_BYTES]; + retval = fscanf(fp, "cpu %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n"); + if (retval != 0) + err(1, "%s: failed to parse format", proc_stat); - if (mp->flags & SYSFS_PERCPU) { - sprintf(path, "/sys/devices/system/cpu/cpu%d/%s", cpu, mp->path); + while (1) { + retval = fscanf(fp, "cpu%u %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n", &cpu_num); + if (retval != 1) + break; - *counterp = snapshot_sysfs_counter(path); - } else { - *counterp = snapshot_sysfs_counter(mp->path); + retval = func(cpu_num); + if (retval) { + fclose(fp); + return (retval); } } - + fclose(fp); return 0; } -unsigned long long get_uncore_mhz(int package, int die) -{ - char path[128]; - - sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", package, - die); +#define PATH_EFFECTIVE_CPUS "/sys/fs/cgroup/cpuset.cpus.effective" - return (snapshot_sysfs_counter(path) / 1000); -} +static char cpu_effective_str[1024]; -int get_epb(int cpu) +static int update_effective_str(bool startup) { - char path[128 + PATH_BYTES]; - unsigned long long msr; - int ret, epb = -1; FILE *fp; + char *pos; + char buf[1024]; + int ret; - sprintf(path, "/sys/devices/system/cpu/cpu%d/power/energy_perf_bias", cpu); + if (cpu_effective_str[0] == '\0' && !startup) + return 0; - fp = fopen(path, "r"); + fp = fopen(PATH_EFFECTIVE_CPUS, "r"); if (!fp) - goto msr_fallback; + return 0; - ret = fscanf(fp, "%d", &epb); - if (ret != 1) - err(1, "%s(%s)", __func__, path); + pos = fgets(buf, 1024, fp); + if (!pos) + err(1, "%s: file read failed\n", PATH_EFFECTIVE_CPUS); fclose(fp); - return epb; - -msr_fallback: - get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr); + ret = strncmp(cpu_effective_str, buf, 1024); + if (!ret) + return 0; - return msr & 0xf; + strncpy(cpu_effective_str, buf, 1024); + return 1; } -void get_apic_id(struct thread_data *t) +static void update_effective_set(bool startup) { - unsigned int eax, ebx, ecx, edx; - - if (DO_BIC(BIC_APIC)) { - eax = ebx = ecx = edx = 0; - __cpuid(1, eax, ebx, ecx, edx); + update_effective_str(startup); - t->apic_id = (ebx >> 24) & 0xff; - } - - if (!DO_BIC(BIC_X2APIC)) - return; - - if (authentic_amd || hygon_genuine) { - unsigned int topology_extensions; - - if (max_extended_level < 0x8000001e) - return; - - eax = ebx = ecx = edx = 0; - __cpuid(0x80000001, eax, ebx, ecx, edx); - topology_extensions = ecx & (1 << 22); - - if (topology_extensions == 0) - return; - - eax = ebx = ecx = edx = 0; - __cpuid(0x8000001e, eax, ebx, ecx, edx); - - t->x2apic_id = eax; - return; - } - - if (!genuine_intel) - return; - - if (max_level < 0xb) - return; + if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_effective_setsize)) + err(1, "%s: cpu str malformat %s\n", PATH_EFFECTIVE_CPUS, cpu_effective_str); +} - ecx = 0; - __cpuid(0xb, eax, ebx, ecx, edx); - t->x2apic_id = edx; +void linux_perf_init(void); +void msr_perf_init(void); +void rapl_perf_init(void); +void cstate_perf_init(void); +void perf_llc_init(void); +void added_perf_counters_init(void); +void pmt_init(void); - if (debug && (t->apic_id != (t->x2apic_id & 0xff))) - fprintf(outf, "cpu%d: BIOS BUG: apic 0x%x x2apic 0x%x\n", t->cpu_id, t->apic_id, t->x2apic_id); +void re_initialize(void) +{ + free_all_buffers(); + setup_all_buffers(false); + linux_perf_init(); + msr_perf_init(); + rapl_perf_init(); + cstate_perf_init(); + perf_llc_init(); + added_perf_counters_init(); + pmt_init(); + fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus); } -int get_core_throt_cnt(int cpu, unsigned long long *cnt) +void set_max_cpu_num(void) { - char path[128 + PATH_BYTES]; - unsigned long long tmp; - FILE *fp; - int ret; + FILE *filep; + int base_cpu; + unsigned long dummy; + char pathname[64]; - sprintf(path, "/sys/devices/system/cpu/cpu%d/thermal_throttle/core_throttle_count", cpu); - fp = fopen(path, "r"); - if (!fp) - return -1; - ret = fscanf(fp, "%lld", &tmp); - fclose(fp); - if (ret != 1) - return -1; - *cnt = tmp; + base_cpu = sched_getcpu(); + if (base_cpu < 0) + err(1, "cannot find calling cpu ID"); + sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", base_cpu); - return 0; + filep = fopen_or_die(pathname, "r"); + topo.max_cpu_num = 0; + while (fscanf(filep, "%lx,", &dummy) == 1) + topo.max_cpu_num += BITMASK_SIZE; + fclose(filep); + topo.max_cpu_num--; /* 0 based */ } /* - * get_counters(...) - * migrate to cpu - * acquire and record local counters for that cpu + * count_cpus() + * remember the last one seen, it will be the max */ -int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int count_cpus(int cpu) { - int cpu = t->cpu_id; - unsigned long long msr; - int aperf_mperf_retry_count = 0; - struct msr_counter *mp; - int i; - - if (cpu_migrate(cpu)) { - fprintf(outf, "get_counters: Could not migrate to CPU %d\n", cpu); - return -1; - } - - gettimeofday(&t->tv_begin, (struct timezone *)NULL); - - if (first_counter_read) - get_apic_id(t); -retry: - t->tsc = rdtsc(); /* we are running on local CPU of interest */ - - if (DO_BIC(BIC_Avg_MHz) || DO_BIC(BIC_Busy) || DO_BIC(BIC_Bzy_MHz) || soft_c1_residency_display(BIC_Avg_MHz)) { - unsigned long long tsc_before, tsc_between, tsc_after, aperf_time, mperf_time; - - /* - * The TSC, APERF and MPERF must be read together for - * APERF/MPERF and MPERF/TSC to give accurate results. - * - * Unfortunately, APERF and MPERF are read by - * individual system call, so delays may occur - * between them. If the time to read them - * varies by a large amount, we re-read them. - */ - - /* - * This initial dummy APERF read has been seen to - * reduce jitter in the subsequent reads. - */ - - if (get_msr(cpu, MSR_IA32_APERF, &t->aperf)) - return -3; + UNUSED(cpu); - t->tsc = rdtsc(); /* re-read close to APERF */ + topo.num_cpus++; + return 0; +} - tsc_before = t->tsc; +int mark_cpu_present(int cpu) +{ + CPU_SET_S(cpu, cpu_present_setsize, cpu_present_set); + return 0; +} - if (get_msr(cpu, MSR_IA32_APERF, &t->aperf)) - return -3; +int init_thread_id(int cpu) +{ + cpus[cpu].thread_id = -1; + return 0; +} - tsc_between = rdtsc(); +int set_my_cpu_type(void) +{ + unsigned int eax, ebx, ecx, edx; + unsigned int max_level; - if (get_msr(cpu, MSR_IA32_MPERF, &t->mperf)) - return -4; + __cpuid(0, max_level, ebx, ecx, edx); - tsc_after = rdtsc(); + if (max_level < CPUID_LEAF_MODEL_ID) + return 0; - aperf_time = tsc_between - tsc_before; - mperf_time = tsc_after - tsc_between; + __cpuid(CPUID_LEAF_MODEL_ID, eax, ebx, ecx, edx); - /* - * If the system call latency to read APERF and MPERF - * differ by more than 2x, then try again. - */ - if ((aperf_time > (2 * mperf_time)) || (mperf_time > (2 * aperf_time))) { - aperf_mperf_retry_count++; - if (aperf_mperf_retry_count < 5) - goto retry; - else - warnx("cpu%d jitter %lld %lld", cpu, aperf_time, mperf_time); - } - aperf_mperf_retry_count = 0; + return (eax >> CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT); +} - t->aperf = t->aperf * aperf_mperf_multiplier; - t->mperf = t->mperf * aperf_mperf_multiplier; - } +int set_cpu_hybrid_type(int cpu) +{ + if (cpu_migrate(cpu)) + return -1; - if (DO_BIC(BIC_IPC)) - if (read(get_instr_count_fd(cpu), &t->instr_count, sizeof(long long)) != sizeof(long long)) - return -4; + int type = set_my_cpu_type(); - if (DO_BIC(BIC_IRQ)) - t->irq_count = irqs_per_cpu[cpu]; - if (DO_BIC(BIC_SMI)) { - if (get_msr(cpu, MSR_SMI_COUNT, &msr)) - return -5; - t->smi_count = msr & 0xFFFFFFFF; - } - if (DO_BIC(BIC_CPU_c1) && use_c1_residency_msr) { - if (get_msr(cpu, MSR_CORE_C1_RES, &t->c1)) - return -6; - } + cpus[cpu].type = type; + return 0; +} - for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - if (get_mp(cpu, mp, &t->counter[i])) - return -10; - } +/* + * snapshot_proc_interrupts() + * + * read and record summary of /proc/interrupts + * + * return 1 if config change requires a restart, else return 0 + */ +int snapshot_proc_interrupts(void) +{ + static FILE *fp; + int column, retval; - /* collect core counters only for 1st thread in core */ - if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE)) - goto done; + if (fp == NULL) + fp = fopen_or_die("/proc/interrupts", "r"); + else + rewind(fp); - if (DO_BIC(BIC_CPU_c3) || soft_c1_residency_display(BIC_CPU_c3)) { - if (get_msr(cpu, MSR_CORE_C3_RESIDENCY, &c->c3)) - return -6; - } + /* read 1st line of /proc/interrupts to get cpu* name for each column */ + for (column = 0; column < topo.num_cpus; ++column) { + int cpu_number; - if ((DO_BIC(BIC_CPU_c6) || soft_c1_residency_display(BIC_CPU_c6)) && !do_knl_cstates) { - if (get_msr(cpu, MSR_CORE_C6_RESIDENCY, &c->c6)) - return -7; - } else if (do_knl_cstates && soft_c1_residency_display(BIC_CPU_c6)) { - if (get_msr(cpu, MSR_KNL_CORE_C6_RESIDENCY, &c->c6)) - return -7; - } + retval = fscanf(fp, " CPU%d", &cpu_number); + if (retval != 1) + break; - if (DO_BIC(BIC_CPU_c7) || soft_c1_residency_display(BIC_CPU_c7)) { - if (get_msr(cpu, MSR_CORE_C7_RESIDENCY, &c->c7)) - return -8; - else if (t->is_atom) { - /* - * For Atom CPUs that has core cstate deeper than c6, - * MSR_CORE_C6_RESIDENCY returns residency of cc6 and deeper. - * Minus CC7 (and deeper cstates) residency to get - * accturate cc6 residency. - */ - c->c6 -= c->c7; + if (cpu_number > topo.max_cpu_num) { + warn("/proc/interrupts: cpu%d: > %d", cpu_number, topo.max_cpu_num); + return 1; } - } - if (DO_BIC(BIC_Mod_c6)) - if (get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us)) - return -8; - - if (DO_BIC(BIC_CoreTmp)) { - if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr)) - return -9; - c->core_temp_c = tj_max - ((msr >> 16) & 0x7F); - } - - if (DO_BIC(BIC_CORE_THROT_CNT)) - get_core_throt_cnt(cpu, &c->core_throt_cnt); - - if (do_rapl & RAPL_AMD_F17H) { - if (get_msr(cpu, MSR_CORE_ENERGY_STAT, &msr)) - return -14; - c->core_energy = msr & 0xFFFFFFFF; - } - - for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - if (get_mp(cpu, mp, &c->counter[i])) - return -10; + irq_column_2_cpu[column] = cpu_number; + irqs_per_cpu[cpu_number] = 0; + nmi_per_cpu[cpu_number] = 0; } - /* collect package counters only for 1st core in package */ - if (!(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) - goto done; + /* read /proc/interrupt count lines and sum up irqs per cpu */ + while (1) { + int column; + char buf[64]; + int this_row_is_nmi = 0; - if (DO_BIC(BIC_Totl_c0)) { - if (get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, &p->pkg_wtd_core_c0)) - return -10; - } - if (DO_BIC(BIC_Any_c0)) { - if (get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, &p->pkg_any_core_c0)) - return -11; - } - if (DO_BIC(BIC_GFX_c0)) { - if (get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, &p->pkg_any_gfxe_c0)) - return -12; - } - if (DO_BIC(BIC_CPUGFX)) { - if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0)) - return -13; - } - if (DO_BIC(BIC_Pkgpc3)) - if (get_msr(cpu, MSR_PKG_C3_RESIDENCY, &p->pc3)) - return -9; - if (DO_BIC(BIC_Pkgpc6)) { - if (do_slm_cstates) { - if (get_msr(cpu, MSR_ATOM_PKG_C6_RESIDENCY, &p->pc6)) - return -10; - } else { - if (get_msr(cpu, MSR_PKG_C6_RESIDENCY, &p->pc6)) - return -10; + retval = fscanf(fp, " %s:", buf); /* irq# "N:" */ + if (retval != 1) + break; + + if (strncmp(buf, "NMI", strlen("NMI")) == 0) + this_row_is_nmi = 1; + + /* read the count per cpu */ + for (column = 0; column < topo.num_cpus; ++column) { + + int cpu_number, irq_count; + + retval = fscanf(fp, " %d", &irq_count); + + if (retval != 1) + break; + + cpu_number = irq_column_2_cpu[column]; + irqs_per_cpu[cpu_number] += irq_count; + if (this_row_is_nmi) + nmi_per_cpu[cpu_number] += irq_count; } + while (getc(fp) != '\n') ; /* flush interrupt description */ + } + return 0; +} - if (DO_BIC(BIC_Pkgpc2)) - if (get_msr(cpu, MSR_PKG_C2_RESIDENCY, &p->pc2)) - return -11; - if (DO_BIC(BIC_Pkgpc7)) - if (get_msr(cpu, MSR_PKG_C7_RESIDENCY, &p->pc7)) - return -12; - if (DO_BIC(BIC_Pkgpc8)) - if (get_msr(cpu, MSR_PKG_C8_RESIDENCY, &p->pc8)) - return -13; - if (DO_BIC(BIC_Pkgpc9)) - if (get_msr(cpu, MSR_PKG_C9_RESIDENCY, &p->pc9)) - return -13; - if (DO_BIC(BIC_Pkgpc10)) - if (get_msr(cpu, MSR_PKG_C10_RESIDENCY, &p->pc10)) - return -13; +/* + * snapshot_graphics() + * + * record snapshot of specified graphics sysfs knob + * + * return 1 if config change requires a restart, else return 0 + */ +int snapshot_graphics(int idx) +{ + int retval; - if (DO_BIC(BIC_CPU_LPI)) - p->cpu_lpi = cpuidle_cur_cpu_lpi_us; - if (DO_BIC(BIC_SYS_LPI)) - p->sys_lpi = cpuidle_cur_sys_lpi_us; + rewind(gfx_info[idx].fp); + fflush(gfx_info[idx].fp); - if (do_rapl & RAPL_PKG) { - if (get_msr_sum(cpu, MSR_PKG_ENERGY_STATUS, &msr)) - return -13; - p->energy_pkg = msr; - } - if (do_rapl & RAPL_CORES_ENERGY_STATUS) { - if (get_msr_sum(cpu, MSR_PP0_ENERGY_STATUS, &msr)) - return -14; - p->energy_cores = msr; - } - if (do_rapl & RAPL_DRAM) { - if (get_msr_sum(cpu, MSR_DRAM_ENERGY_STATUS, &msr)) - return -15; - p->energy_dram = msr; - } - if (do_rapl & RAPL_GFX) { - if (get_msr_sum(cpu, MSR_PP1_ENERGY_STATUS, &msr)) - return -16; - p->energy_gfx = msr; - } - if (do_rapl & RAPL_PKG_PERF_STATUS) { - if (get_msr_sum(cpu, MSR_PKG_PERF_STATUS, &msr)) - return -16; - p->rapl_pkg_perf_status = msr; - } - if (do_rapl & RAPL_DRAM_PERF_STATUS) { - if (get_msr_sum(cpu, MSR_DRAM_PERF_STATUS, &msr)) - return -16; - p->rapl_dram_perf_status = msr; - } - if (do_rapl & RAPL_AMD_F17H) { - if (get_msr_sum(cpu, MSR_PKG_ENERGY_STAT, &msr)) - return -13; - p->energy_pkg = msr; + switch (idx) { + case GFX_rc6: + case SAM_mc6: + retval = fscanf(gfx_info[idx].fp, "%lld", &gfx_info[idx].val_ull); + if (retval != 1) + err(1, "rc6"); + return 0; + case GFX_MHz: + case GFX_ACTMHz: + case SAM_MHz: + case SAM_ACTMHz: + retval = fscanf(gfx_info[idx].fp, "%d", &gfx_info[idx].val); + if (retval != 1) + err(1, "MHz"); + return 0; + default: + return -EINVAL; } - if (DO_BIC(BIC_PkgTmp)) { - if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr)) - return -17; - p->pkg_temp_c = tj_max - ((msr >> 16) & 0x7F); +} + +/* + * snapshot_cpu_lpi() + * + * record snapshot of + * /sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us + */ +int snapshot_cpu_lpi_us(void) +{ + FILE *fp; + int retval; + + fp = fopen_or_die("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", "r"); + + retval = fscanf(fp, "%lld", &cpuidle_cur_cpu_lpi_us); + if (retval != 1) { + fprintf(stderr, "Disabling Low Power Idle CPU output\n"); + BIC_NOT_PRESENT(BIC_CPU_LPI); + fclose(fp); + return -1; } - if (DO_BIC(BIC_GFX_rc6)) - p->gfx_rc6_ms = gfx_cur_rc6_ms; + fclose(fp); - /* n.b. assume die0 uncore frequency applies to whole package */ - if (DO_BIC(BIC_UNCORE_MHZ)) - p->uncore_mhz = get_uncore_mhz(p->package_id, 0); + return 0; +} - if (DO_BIC(BIC_GFXMHz)) - p->gfx_mhz = gfx_cur_mhz; +/* + * snapshot_sys_lpi() + * + * record snapshot of sys_lpi_file + */ +int snapshot_sys_lpi_us(void) +{ + FILE *fp; + int retval; - if (DO_BIC(BIC_GFXACTMHz)) - p->gfx_act_mhz = gfx_act_mhz; + fp = fopen_or_die(sys_lpi_file, "r"); - for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if (get_mp(cpu, mp, &p->counter[i])) - return -10; + retval = fscanf(fp, "%lld", &cpuidle_cur_sys_lpi_us); + if (retval != 1) { + fprintf(stderr, "Disabling Low Power Idle System output\n"); + BIC_NOT_PRESENT(BIC_SYS_LPI); + fclose(fp); + return -1; } -done: - gettimeofday(&t->tv_end, (struct timezone *)NULL); + fclose(fp); return 0; } /* - * MSR_PKG_CST_CONFIG_CONTROL decoding for pkg_cstate_limit: - * If you change the values, note they are used both in comparisons - * (>= PCL__7) and to index pkg_cstate_limit_strings[]. + * snapshot /proc and /sys files + * + * return 1 if configuration restart needed, else return 0 */ +int snapshot_proc_sysfs_files(void) +{ + gettimeofday(&procsysfs_tv_begin, (struct timezone *)NULL); -#define PCLUKN 0 /* Unknown */ -#define PCLRSV 1 /* Reserved */ -#define PCL__0 2 /* PC0 */ -#define PCL__1 3 /* PC1 */ -#define PCL__2 4 /* PC2 */ -#define PCL__3 5 /* PC3 */ -#define PCL__4 6 /* PC4 */ -#define PCL__6 7 /* PC6 */ -#define PCL_6N 8 /* PC6 No Retention */ -#define PCL_6R 9 /* PC6 Retention */ -#define PCL__7 10 /* PC7 */ -#define PCL_7S 11 /* PC7 Shrink */ -#define PCL__8 12 /* PC8 */ -#define PCL__9 13 /* PC9 */ -#define PCL_10 14 /* PC10 */ -#define PCLUNL 15 /* Unlimited */ + if (DO_BIC(BIC_IRQ) || DO_BIC(BIC_NMI)) + if (snapshot_proc_interrupts()) + return 1; -int pkg_cstate_limit = PCLUKN; -char *pkg_cstate_limit_strings[] = { "reserved", "unknown", "pc0", "pc1", "pc2", - "pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited" -}; + if (DO_BIC(BIC_GFX_rc6)) + snapshot_graphics(GFX_rc6); -int nhm_pkg_cstate_limits[16] = - { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, - PCLRSV, PCLRSV -}; + if (DO_BIC(BIC_GFXMHz)) + snapshot_graphics(GFX_MHz); -int snb_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, - PCLRSV, PCLRSV -}; + if (DO_BIC(BIC_GFXACTMHz)) + snapshot_graphics(GFX_ACTMHz); -int hsw_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, - PCLRSV, PCLRSV -}; + if (DO_BIC(BIC_SAM_mc6)) + snapshot_graphics(SAM_mc6); -int slv_pkg_cstate_limits[16] = - { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, - PCL__6, PCL__7 -}; + if (DO_BIC(BIC_SAMMHz)) + snapshot_graphics(SAM_MHz); -int amt_pkg_cstate_limits[16] = - { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, - PCLRSV, PCLRSV -}; + if (DO_BIC(BIC_SAMACTMHz)) + snapshot_graphics(SAM_ACTMHz); -int phi_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, - PCLRSV, PCLRSV -}; + if (DO_BIC(BIC_CPU_LPI)) + snapshot_cpu_lpi_us(); -int glm_pkg_cstate_limits[16] = - { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, - PCLRSV, PCLRSV -}; + if (DO_BIC(BIC_SYS_LPI)) + snapshot_sys_lpi_us(); -int skx_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, - PCLRSV, PCLRSV -}; + return 0; +} -int icx_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, - PCLRSV, PCLRSV -}; +int exit_requested; -static void calculate_tsc_tweak() +static void signal_handler(int signal) { - tsc_tweak = base_hz / tsc_hz; + switch (signal) { + case SIGINT: + exit_requested = 1; + if (debug) + fprintf(stderr, " SIGINT\n"); + break; + case SIGUSR1: + if (debug > 1) + fprintf(stderr, "SIGUSR1\n"); + break; + } } -void prewake_cstate_probe(unsigned int family, unsigned int model); - -static void dump_nhm_platform_info(void) +void setup_signal_handler(void) { - unsigned long long msr; - unsigned int ratio; - - get_msr(base_cpu, MSR_PLATFORM_INFO, &msr); - - fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", base_cpu, msr); - - ratio = (msr >> 40) & 0xFF; - fprintf(outf, "%d * %.1f = %.1f MHz max efficiency frequency\n", ratio, bclk, ratio * bclk); - - ratio = (msr >> 8) & 0xFF; - fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk); + struct sigaction sa; - get_msr(base_cpu, MSR_IA32_POWER_CTL, &msr); - fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", - base_cpu, msr, msr & 0x2 ? "EN" : "DIS"); + memset(&sa, 0, sizeof(sa)); - /* C-state Pre-wake Disable (CSTATE_PREWAKE_DISABLE) */ - if (dis_cstate_prewake) - fprintf(outf, "C-state Pre-wake: %sabled\n", msr & 0x40000000 ? "DIS" : "EN"); + sa.sa_handler = &signal_handler; - return; + if (sigaction(SIGINT, &sa, NULL) < 0) + err(1, "sigaction SIGINT"); + if (sigaction(SIGUSR1, &sa, NULL) < 0) + err(1, "sigaction SIGUSR1"); } -static void dump_hsw_turbo_ratio_limits(void) +void do_sleep(void) { - unsigned long long msr; - unsigned int ratio; + struct timeval tout; + struct timespec rest; + fd_set readfds; + int retval; - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT2, &msr); + FD_ZERO(&readfds); + FD_SET(0, &readfds); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", base_cpu, msr); + if (ignore_stdin) { + nanosleep(&interval_ts, NULL); + return; + } - ratio = (msr >> 8) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 18 active cores\n", ratio, bclk, ratio * bclk); + tout = interval_tv; + retval = select(1, &readfds, NULL, NULL, &tout); - ratio = (msr >> 0) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 17 active cores\n", ratio, bclk, ratio * bclk); - return; + if (retval == 1) { + switch (getc(stdin)) { + case 'q': + exit_requested = 1; + break; + case EOF: + /* + * 'stdin' is a pipe closed on the other end. There + * won't be any further input. + */ + ignore_stdin = 1; + /* Sleep the rest of the time */ + rest.tv_sec = (tout.tv_sec + tout.tv_usec / 1000000); + rest.tv_nsec = (tout.tv_usec % 1000000) * 1000; + nanosleep(&rest, NULL); + } + } } -static void dump_ivt_turbo_ratio_limits(void) +int get_msr_sum(int cpu, off_t offset, unsigned long long *msr) { - unsigned long long msr; - unsigned int ratio; + int ret, idx; + unsigned long long msr_cur, msr_last; - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &msr); + assert(!no_msr); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, msr); + if (!per_cpu_msr_sum) + return 1; + + idx = offset_to_idx(offset); + if (idx < 0) + return idx; + /* get_msr_sum() = sum + (get_msr() - last) */ + ret = get_msr(cpu, offset, &msr_cur); + if (ret) + return ret; + msr_last = per_cpu_msr_sum[cpu].entries[idx].last; + DELTA_WRAP32(msr_cur, msr_last); + *msr = msr_last + per_cpu_msr_sum[cpu].entries[idx].sum; + + return 0; +} - ratio = (msr >> 56) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 16 active cores\n", ratio, bclk, ratio * bclk); +timer_t timerid; - ratio = (msr >> 48) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 15 active cores\n", ratio, bclk, ratio * bclk); +/* Timer callback, update the sum of MSRs periodically. */ +static int update_msr_sum(PER_THREAD_PARAMS) +{ + int i, ret; + int cpu = t->cpu_id; - ratio = (msr >> 40) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 14 active cores\n", ratio, bclk, ratio * bclk); + UNUSED(c); + UNUSED(p); - ratio = (msr >> 32) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 13 active cores\n", ratio, bclk, ratio * bclk); + assert(!no_msr); - ratio = (msr >> 24) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 12 active cores\n", ratio, bclk, ratio * bclk); + for (i = IDX_PKG_ENERGY; i < IDX_COUNT; i++) { + unsigned long long msr_cur, msr_last; + off_t offset; - ratio = (msr >> 16) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 11 active cores\n", ratio, bclk, ratio * bclk); + if (!idx_valid(i)) + continue; + offset = idx_to_offset(i); + if (offset < 0) + continue; + ret = get_msr(cpu, offset, &msr_cur); + if (ret) { + fprintf(outf, "Can not update msr(0x%llx)\n", (unsigned long long)offset); + continue; + } - ratio = (msr >> 8) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 10 active cores\n", ratio, bclk, ratio * bclk); + msr_last = per_cpu_msr_sum[cpu].entries[i].last; + per_cpu_msr_sum[cpu].entries[i].last = msr_cur & 0xffffffff; - ratio = (msr >> 0) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 9 active cores\n", ratio, bclk, ratio * bclk); - return; + DELTA_WRAP32(msr_cur, msr_last); + per_cpu_msr_sum[cpu].entries[i].sum += msr_last; + } + return 0; } -int has_turbo_ratio_group_limits(int family, int model) +static void msr_record_handler(union sigval v) { + UNUSED(v); - if (!genuine_intel) - return 0; - - if (family != 6) - return 0; - - switch (model) { - case INTEL_FAM6_ATOM_GOLDMONT: - case INTEL_FAM6_SKYLAKE_X: - case INTEL_FAM6_ICELAKE_X: - case INTEL_FAM6_SAPPHIRERAPIDS_X: - case INTEL_FAM6_ATOM_GOLDMONT_D: - case INTEL_FAM6_ATOM_TREMONT_D: - return 1; - default: - return 0; - } + for_all_cpus(update_msr_sum, EVEN_COUNTERS); } -static void dump_turbo_ratio_limits(int trl_msr_offset, int family, int model) +void msr_sum_record(void) { - unsigned long long msr, core_counts; - int shift; + struct itimerspec its; + struct sigevent sev; - get_msr(base_cpu, trl_msr_offset, &msr); - fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n", - base_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr); + per_cpu_msr_sum = calloc(topo.max_cpu_num + 1, sizeof(struct msr_sum_array)); + if (!per_cpu_msr_sum) { + fprintf(outf, "Can not allocate memory for long time MSR.\n"); + return; + } + /* + * Signal handler might be restricted, so use thread notifier instead. + */ + memset(&sev, 0, sizeof(struct sigevent)); + sev.sigev_notify = SIGEV_THREAD; + sev.sigev_notify_function = msr_record_handler; - if (has_turbo_ratio_group_limits(family, model)) { - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, core_counts); - } else { - core_counts = 0x0807060504030201; + sev.sigev_value.sival_ptr = &timerid; + if (timer_create(CLOCK_REALTIME, &sev, &timerid) == -1) { + fprintf(outf, "Can not create timer.\n"); + goto release_msr; } - for (shift = 56; shift >= 0; shift -= 8) { - unsigned int ratio, group_size; + its.it_value.tv_sec = 0; + its.it_value.tv_nsec = 1; + /* + * A wraparound time has been calculated early. + * Some sources state that the peak power for a + * microprocessor is usually 1.5 times the TDP rating, + * use 2 * TDP for safety. + */ + its.it_interval.tv_sec = rapl_joule_counter_range / 2; + its.it_interval.tv_nsec = 0; - ratio = (msr >> shift) & 0xFF; - group_size = (core_counts >> shift) & 0xFF; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", - ratio, bclk, ratio * bclk, group_size); + if (timer_settime(timerid, 0, &its, NULL) == -1) { + fprintf(outf, "Can not set timer.\n"); + goto release_timer; } - return; + +release_timer: + timer_delete(timerid); +release_msr: + free(per_cpu_msr_sum); + per_cpu_msr_sum = NULL; } -static void dump_atom_turbo_ratio_limits(void) +/* + * set_my_sched_priority(pri) + * return previous priority on success + * return value < -20 on failure + */ +int set_my_sched_priority(int priority) { - unsigned long long msr; - unsigned int ratio; - - get_msr(base_cpu, MSR_ATOM_CORE_RATIOS, &msr); - fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF); - - ratio = (msr >> 0) & 0x3F; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz minimum operating frequency\n", ratio, bclk, ratio * bclk); - - ratio = (msr >> 8) & 0x3F; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz low frequency mode (LFM)\n", ratio, bclk, ratio * bclk); - - ratio = (msr >> 16) & 0x3F; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk); - - get_msr(base_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr); - fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF); + int retval; + int original_priority; - ratio = (msr >> 24) & 0x3F; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 4 active cores\n", ratio, bclk, ratio * bclk); + errno = 0; + original_priority = getpriority(PRIO_PROCESS, 0); + if (errno && (original_priority == -1)) + return -21; - ratio = (msr >> 16) & 0x3F; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 3 active cores\n", ratio, bclk, ratio * bclk); + retval = setpriority(PRIO_PROCESS, 0, priority); + if (retval) + return -21; - ratio = (msr >> 8) & 0x3F; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 2 active cores\n", ratio, bclk, ratio * bclk); + errno = 0; + retval = getpriority(PRIO_PROCESS, 0); + if (retval != priority) + return -21; - ratio = (msr >> 0) & 0x3F; - if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo 1 active core\n", ratio, bclk, ratio * bclk); + return original_priority; } -static void dump_knl_turbo_ratio_limits(void) +void turbostat_loop() { - const unsigned int buckets_no = 7; - - unsigned long long msr; - int delta_cores, delta_ratio; - int i, b_nr; - unsigned int cores[buckets_no]; - unsigned int ratio[buckets_no]; - - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT, &msr); + int retval; + int restarted = 0; + unsigned int done_iters = 0; - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", base_cpu, msr); + setup_signal_handler(); /* - * Turbo encoding in KNL is as follows: - * [0] -- Reserved - * [7:1] -- Base value of number of active cores of bucket 1. - * [15:8] -- Base value of freq ratio of bucket 1. - * [20:16] -- +ve delta of number of active cores of bucket 2. - * i.e. active cores of bucket 2 = - * active cores of bucket 1 + delta - * [23:21] -- Negative delta of freq ratio of bucket 2. - * i.e. freq ratio of bucket 2 = - * freq ratio of bucket 1 - delta - * [28:24]-- +ve delta of number of active cores of bucket 3. - * [31:29]-- -ve delta of freq ratio of bucket 3. - * [36:32]-- +ve delta of number of active cores of bucket 4. - * [39:37]-- -ve delta of freq ratio of bucket 4. - * [44:40]-- +ve delta of number of active cores of bucket 5. - * [47:45]-- -ve delta of freq ratio of bucket 5. - * [52:48]-- +ve delta of number of active cores of bucket 6. - * [55:53]-- -ve delta of freq ratio of bucket 6. - * [60:56]-- +ve delta of number of active cores of bucket 7. - * [63:61]-- -ve delta of freq ratio of bucket 7. + * elevate own priority for interval mode + * + * ignore on error - we probably don't have permission to set it, but + * it's not a big deal */ + set_my_sched_priority(-20); - b_nr = 0; - cores[b_nr] = (msr & 0xFF) >> 1; - ratio[b_nr] = (msr >> 8) & 0xFF; +restart: + restarted++; - for (i = 16; i < 64; i += 8) { - delta_cores = (msr >> i) & 0x1F; - delta_ratio = (msr >> (i + 5)) & 0x7; + snapshot_proc_sysfs_files(); + retval = for_all_cpus(get_counters, EVEN_COUNTERS); + first_counter_read = 0; + if (retval < -1) { + exit(retval); + } else if (retval == -1) { + if (restarted > 10) { + exit(retval); + } + re_initialize(); + goto restart; + } + restarted = 0; + done_iters = 0; + gettimeofday(&tv_even, (struct timezone *)NULL); - cores[b_nr + 1] = cores[b_nr] + delta_cores; - ratio[b_nr + 1] = ratio[b_nr] - delta_ratio; - b_nr++; + while (1) { + if (for_all_proc_cpus(cpu_is_not_present)) { + re_initialize(); + goto restart; + } + if (update_effective_str(false)) { + re_initialize(); + goto restart; + } + do_sleep(); + if (snapshot_proc_sysfs_files()) + goto restart; + retval = for_all_cpus(get_counters, ODD_COUNTERS); + if (retval < -1) { + exit(retval); + } else if (retval == -1) { + re_initialize(); + goto restart; + } + gettimeofday(&tv_odd, (struct timezone *)NULL); + timersub(&tv_odd, &tv_even, &tv_delta); + if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS)) { + re_initialize(); + goto restart; + } + delta_platform(&platform_counters_odd, &platform_counters_even); + compute_average(EVEN_COUNTERS); + format_all_counters(EVEN_COUNTERS); + flush_output_stdout(); + if (exit_requested) + break; + if (num_iterations && ++done_iters >= num_iterations) + break; + do_sleep(); + if (snapshot_proc_sysfs_files()) + goto restart; + retval = for_all_cpus(get_counters, EVEN_COUNTERS); + if (retval < -1) { + exit(retval); + } else if (retval == -1) { + re_initialize(); + goto restart; + } + gettimeofday(&tv_even, (struct timezone *)NULL); + timersub(&tv_even, &tv_odd, &tv_delta); + if (for_all_cpus_2(delta_cpu, EVEN_COUNTERS, ODD_COUNTERS)) { + re_initialize(); + goto restart; + } + delta_platform(&platform_counters_even, &platform_counters_odd); + compute_average(ODD_COUNTERS); + format_all_counters(ODD_COUNTERS); + flush_output_stdout(); + if (exit_requested) + break; + if (num_iterations && ++done_iters >= num_iterations) + break; } - - for (i = buckets_no - 1; i >= 0; i--) - if (i > 0 ? ratio[i] != ratio[i - 1] : 1) - fprintf(outf, - "%d * %.1f = %.1f MHz max turbo %d active cores\n", - ratio[i], bclk, ratio[i] * bclk, cores[i]); } -static void dump_nhm_cst_cfg(void) +int probe_dev_msr(void) { - unsigned long long msr; - - get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); - - fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", base_cpu, msr); - - fprintf(outf, " (%s%s%s%s%slocked, pkg-cstate-limit=%d (%s)", - (msr & SNB_C3_AUTO_UNDEMOTE) ? "UNdemote-C3, " : "", - (msr & SNB_C1_AUTO_UNDEMOTE) ? "UNdemote-C1, " : "", - (msr & NHM_C3_AUTO_DEMOTE) ? "demote-C3, " : "", - (msr & NHM_C1_AUTO_DEMOTE) ? "demote-C1, " : "", - (msr & (1 << 15)) ? "" : "UN", (unsigned int)msr & 0xF, pkg_cstate_limit_strings[pkg_cstate_limit]); - -#define AUTOMATIC_CSTATE_CONVERSION (1UL << 16) - if (has_automatic_cstate_conversion) { - fprintf(outf, ", automatic c-state conversion=%s", (msr & AUTOMATIC_CSTATE_CONVERSION) ? "on" : "off"); - } - - fprintf(outf, ")\n"); + struct stat sb; + char pathname[32]; - return; + sprintf(pathname, "/dev/msr%d", base_cpu); + return !stat(pathname, &sb); } -static void dump_config_tdp(void) +int probe_dev_cpu_msr(void) { - unsigned long long msr; - - get_msr(base_cpu, MSR_CONFIG_TDP_NOMINAL, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", base_cpu, msr); - fprintf(outf, " (base_ratio=%d)\n", (unsigned int)msr & 0xFF); + struct stat sb; + char pathname[32]; - get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_1, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", base_cpu, msr); - if (msr) { - fprintf(outf, "PKG_MIN_PWR_LVL1=%d ", (unsigned int)(msr >> 48) & 0x7FFF); - fprintf(outf, "PKG_MAX_PWR_LVL1=%d ", (unsigned int)(msr >> 32) & 0x7FFF); - fprintf(outf, "LVL1_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF); - fprintf(outf, "PKG_TDP_LVL1=%d", (unsigned int)(msr) & 0x7FFF); - } - fprintf(outf, ")\n"); + sprintf(pathname, "/dev/cpu/%d/msr", base_cpu); + return !stat(pathname, &sb); +} - get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_2, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", base_cpu, msr); - if (msr) { - fprintf(outf, "PKG_MIN_PWR_LVL2=%d ", (unsigned int)(msr >> 48) & 0x7FFF); - fprintf(outf, "PKG_MAX_PWR_LVL2=%d ", (unsigned int)(msr >> 32) & 0x7FFF); - fprintf(outf, "LVL2_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF); - fprintf(outf, "PKG_TDP_LVL2=%d", (unsigned int)(msr) & 0x7FFF); +int probe_msr_driver(void) +{ + if (probe_dev_msr()) { + use_android_msr_path = 1; + return 1; } - fprintf(outf, ")\n"); - - get_msr(base_cpu, MSR_CONFIG_TDP_CONTROL, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", base_cpu, msr); - if ((msr) & 0x3) - fprintf(outf, "TDP_LEVEL=%d ", (unsigned int)(msr) & 0x3); - fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1); - fprintf(outf, ")\n"); - - get_msr(base_cpu, MSR_TURBO_ACTIVATION_RATIO, &msr); - fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", base_cpu, msr); - fprintf(outf, "MAX_NON_TURBO_RATIO=%d", (unsigned int)(msr) & 0xFF); - fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1); - fprintf(outf, ")\n"); + return probe_dev_cpu_msr(); } -unsigned int irtl_time_units[] = { 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 }; - -void print_irtl(void) +void check_msr_driver(void) { - unsigned long long msr; - - get_msr(base_cpu, MSR_PKGC3_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); - - get_msr(base_cpu, MSR_PKGC6_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); - - get_msr(base_cpu, MSR_PKGC7_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); - - if (!do_irtl_hsw) + if (probe_msr_driver()) return; - get_msr(base_cpu, MSR_PKGC8_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); - - get_msr(base_cpu, MSR_PKGC9_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); - - get_msr(base_cpu, MSR_PKGC10_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + if (system("/sbin/modprobe msr > /dev/null 2>&1")) + no_msr = 1; + if (!probe_msr_driver()) + no_msr = 1; } -void free_fd_percpu(void) +/* + * check for CAP_SYS_RAWIO + * return 0 on success + * return 1 on fail + */ +int check_for_cap_sys_rawio(void) { - int i; - - for (i = 0; i < topo.max_cpu_num + 1; ++i) { - if (fd_percpu[i] != 0) - close(fd_percpu[i]); - } - - free(fd_percpu); -} + cap_t caps; + cap_flag_value_t cap_flag_value; + int ret = 0; -void free_all_buffers(void) -{ - int i; + caps = cap_get_proc(); + if (caps == NULL) { + /* + * CONFIG_MULTIUSER=n kernels have no cap_get_proc() + * Allow them to continue and attempt to access MSRs + */ + if (errno == ENOSYS) + return 0; - CPU_FREE(cpu_present_set); - cpu_present_set = NULL; - cpu_present_setsize = 0; + return 1; + } - CPU_FREE(cpu_affinity_set); - cpu_affinity_set = NULL; - cpu_affinity_setsize = 0; + if (cap_get_flag(caps, CAP_SYS_RAWIO, CAP_EFFECTIVE, &cap_flag_value)) { + ret = 1; + goto free_and_exit; + } - free(thread_even); - free(core_even); - free(package_even); + if (cap_flag_value != CAP_SET) { + ret = 1; + goto free_and_exit; + } - thread_even = NULL; - core_even = NULL; - package_even = NULL; +free_and_exit: + if (cap_free(caps) == -1) + err(-6, "cap_free\n"); - free(thread_odd); - free(core_odd); - free(package_odd); + return ret; +} - thread_odd = NULL; - core_odd = NULL; - package_odd = NULL; +void check_msr_permission(void) +{ + int failed = 0; + char pathname[32]; - free(output_buffer); - output_buffer = NULL; - outp = NULL; + if (no_msr) + return; - free_fd_percpu(); + /* check for CAP_SYS_RAWIO */ + failed += check_for_cap_sys_rawio(); - free(irq_column_2_cpu); - free(irqs_per_cpu); + /* test file permissions */ + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", base_cpu); + if (euidaccess(pathname, R_OK)) { + failed++; + } - for (i = 0; i <= topo.max_cpu_num; ++i) { - if (cpus[i].put_ids) - CPU_FREE(cpus[i].put_ids); + if (failed) { + warnx("Failed to access %s. Some of the counters may not be available\n" + "\tRun as root to enable them or use %s to disable the access explicitly", pathname, "--no-msr"); + no_msr = 1; } - free(cpus); } -/* - * Parse a file containing a single int. - * Return 0 if file can not be opened - * Exit if file can be opened, but can not be parsed - */ -int parse_int_file(const char *fmt, ...) +void probe_bclk(void) { - va_list args; - char path[PATH_MAX]; - FILE *filep; - int value; + unsigned long long msr; + unsigned int base_ratio; - va_start(args, fmt); - vsnprintf(path, sizeof(path), fmt, args); - va_end(args); - filep = fopen(path, "r"); - if (!filep) - return 0; - if (fscanf(filep, "%d", &value) != 1) - err(1, "%s: failed to parse number from file", path); - fclose(filep); - return value; -} + if (!platform->has_nhm_msrs || no_msr) + return; -/* - * cpu_is_first_core_in_package(cpu) - * return 1 if given CPU is 1st core in package - */ -int cpu_is_first_core_in_package(int cpu) -{ - return cpu == parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_siblings_list", cpu); -} + if (platform->bclk_freq == BCLK_100MHZ) + bclk = 100.00; + else if (platform->bclk_freq == BCLK_133MHZ) + bclk = 133.33; + else if (platform->bclk_freq == BCLK_SLV) + bclk = slm_bclk(); + else + return; -int get_physical_package_id(int cpu) -{ - return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu); -} + get_msr(base_cpu, MSR_PLATFORM_INFO, &msr); + base_ratio = (msr >> 8) & 0xFF; -int get_die_id(int cpu) -{ - return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/die_id", cpu); -} + base_hz = base_ratio * bclk * 1000000; + has_base_hz = 1; -int get_core_id(int cpu) -{ - return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_id", cpu); + if (platform->enable_tsc_tweak) + tsc_tweak = base_hz / tsc_hz; } -void set_node_data(void) +static void remove_underbar(char *s) { - int pkg, node, lnode, cpu, cpux; - int cpu_count; - - /* initialize logical_node_id */ - for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) - cpus[cpu].logical_node_id = -1; - - cpu_count = 0; - for (pkg = 0; pkg < topo.num_packages; pkg++) { - lnode = 0; - for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { - if (cpus[cpu].physical_package_id != pkg) - continue; - /* find a cpu with an unset logical_node_id */ - if (cpus[cpu].logical_node_id != -1) - continue; - cpus[cpu].logical_node_id = lnode; - node = cpus[cpu].physical_node_id; - cpu_count++; - /* - * find all matching cpus on this pkg and set - * the logical_node_id - */ - for (cpux = cpu; cpux <= topo.max_cpu_num; cpux++) { - if ((cpus[cpux].physical_package_id == pkg) && (cpus[cpux].physical_node_id == node)) { - cpus[cpux].logical_node_id = lnode; - cpu_count++; - } - } - lnode++; - if (lnode > topo.nodes_per_pkg) - topo.nodes_per_pkg = lnode; - } - if (cpu_count >= topo.max_cpu_num) - break; + char *to = s; + + while (*s) { + if (*s != '_') + *to++ = *s; + s++; } + + *to = 0; } -int get_physical_node_id(struct cpu_topology *thiscpu) +static void dump_turbo_ratio_info(void) { - char path[80]; - FILE *filep; - int i; - int cpu = thiscpu->logical_cpu_id; + if (!has_turbo) + return; - for (i = 0; i <= topo.max_cpu_num; i++) { - sprintf(path, "/sys/devices/system/cpu/cpu%d/node%i/cpulist", cpu, i); - filep = fopen(path, "r"); - if (!filep) - continue; - fclose(filep); - return i; + if (!platform->has_nhm_msrs || no_msr) + return; + + if (platform->trl_msrs & TRL_LIMIT2) + dump_turbo_ratio_limit2(); + + if (platform->trl_msrs & TRL_LIMIT1) + dump_turbo_ratio_limit1(); + + if (platform->trl_msrs & TRL_BASE) { + dump_turbo_ratio_limits(MSR_TURBO_RATIO_LIMIT); + + if (is_hybrid) + dump_turbo_ratio_limits(MSR_SECONDARY_TURBO_RATIO_LIMIT); } - return -1; + + if (platform->trl_msrs & TRL_ATOM) + dump_atom_turbo_ratio_limits(); + + if (platform->trl_msrs & TRL_KNL) + dump_knl_turbo_ratio_limits(); + + if (platform->has_config_tdp) + dump_config_tdp(); } -int get_thread_siblings(struct cpu_topology *thiscpu) +static int read_sysfs_int(char *path) { - char path[80], character; - FILE *filep; - unsigned long map; - int so, shift, sib_core; - int cpu = thiscpu->logical_cpu_id; - int offset = topo.max_cpu_num + 1; - size_t size; - int thread_id = 0; + FILE *input; + int retval = -1; - thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1)); - if (thiscpu->thread_id < 0) - thiscpu->thread_id = thread_id++; - if (!thiscpu->put_ids) - return -1; + input = fopen(path, "r"); + if (input == NULL) { + if (debug) + fprintf(outf, "NSFOD %s\n", path); + return (-1); + } + if (fscanf(input, "%d", &retval) != 1) + err(1, "%s: failed to read int from file", path); + fclose(input); - size = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); - CPU_ZERO_S(size, thiscpu->put_ids); + return (retval); +} - sprintf(path, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", cpu); - filep = fopen(path, "r"); +static void dump_sysfs_file(char *path) +{ + FILE *input; + char cpuidle_buf[64]; - if (!filep) { - warnx("%s: open failed", path); - return -1; + input = fopen(path, "r"); + if (input == NULL) { + if (debug) + fprintf(outf, "NSFOD %s\n", path); + return; } - do { - offset -= BITMASK_SIZE; - if (fscanf(filep, "%lx%c", &map, &character) != 2) - err(1, "%s: failed to parse file", path); - for (shift = 0; shift < BITMASK_SIZE; shift++) { - if ((map >> shift) & 0x1) { - so = shift + offset; - sib_core = get_core_id(so); - if (sib_core == thiscpu->physical_core_id) { - CPU_SET_S(so, size, thiscpu->put_ids); - if ((so != cpu) && (cpus[so].thread_id < 0)) - cpus[so].thread_id = thread_id++; - } - } - } - } while (character == ','); - fclose(filep); + if (!fgets(cpuidle_buf, sizeof(cpuidle_buf), input)) + err(1, "%s: failed to read file", path); + fclose(input); - return CPU_COUNT_S(size, thiscpu->put_ids); + fprintf(outf, "%s: %s", strrchr(path, '/') + 1, cpuidle_buf); } -/* - * run func(thread, core, package) in topology order - * skip non-present cpus - */ - -int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *, - struct pkg_data *, struct thread_data *, struct core_data *, - struct pkg_data *), struct thread_data *thread_base, - struct core_data *core_base, struct pkg_data *pkg_base, - struct thread_data *thread_base2, struct core_data *core_base2, struct pkg_data *pkg_base2) +static void probe_intel_uncore_frequency_legacy(void) { - int retval, pkg_no, node_no, core_no, thread_no; + int i, j; + char path[256]; - for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) { - for (node_no = 0; node_no < topo.nodes_per_pkg; ++node_no) { - for (core_no = 0; core_no < topo.cores_per_node; ++core_no) { - for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) { - struct thread_data *t, *t2; - struct core_data *c, *c2; - struct pkg_data *p, *p2; + for (i = 0; i < topo.num_packages; ++i) { + for (j = 0; j <= topo.max_die_id; ++j) { + int k, l; + char path_base[128]; - t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no); + sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i, j); - if (cpu_is_not_present(t->cpu_id)) - continue; + sprintf(path, "%s/current_freq_khz", path_base); + if (access(path, R_OK)) + continue; - t2 = GET_THREAD(thread_base2, thread_no, core_no, node_no, pkg_no); + BIC_PRESENT(BIC_UNCORE_MHZ); - c = GET_CORE(core_base, core_no, node_no, pkg_no); - c2 = GET_CORE(core_base2, core_no, node_no, pkg_no); + if (quiet) + return; + + sprintf(path, "%s/min_freq_khz", path_base); + k = read_sysfs_int(path); + sprintf(path, "%s/max_freq_khz", path_base); + l = read_sysfs_int(path); + fprintf(outf, "Uncore Frequency package%d die%d: %d - %d MHz ", i, j, k / 1000, l / 1000); - p = GET_PKG(pkg_base, pkg_no); - p2 = GET_PKG(pkg_base2, pkg_no); + sprintf(path, "%s/initial_min_freq_khz", path_base); + k = read_sysfs_int(path); + sprintf(path, "%s/initial_max_freq_khz", path_base); + l = read_sysfs_int(path); + fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000); - retval = func(t, c, p, t2, c2, p2); - if (retval) - return retval; - } - } + sprintf(path, "%s/current_freq_khz", path_base); + k = read_sysfs_int(path); + fprintf(outf, " %d MHz\n", k / 1000); } } - return 0; } -/* - * run func(cpu) on every cpu in /proc/stat - * return max_cpu number - */ -int for_all_proc_cpus(int (func) (int)) +static void probe_intel_uncore_frequency_cluster(void) { - FILE *fp; - int cpu_num; - int retval; + int i, uncore_max_id; + char path[256]; + char path_base[128]; - fp = fopen_or_die(proc_stat, "r"); + if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00/current_freq_khz", R_OK)) + return; - retval = fscanf(fp, "cpu %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n"); - if (retval != 0) - err(1, "%s: failed to parse format", proc_stat); + for (uncore_max_id = 0;; ++uncore_max_id) { - while (1) { - retval = fscanf(fp, "cpu%u %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n", &cpu_num); - if (retval != 1) - break; + sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", uncore_max_id); - retval = func(cpu_num); - if (retval) { - fclose(fp); - return (retval); + /* uncore## start at 00 and skips no numbers, so stop upon first missing */ + if (access(path_base, R_OK)) { + uncore_max_id -= 1; + break; } } - fclose(fp); - return 0; -} + for (i = uncore_max_id; i >= 0; --i) { + int k, l; + int package_id, domain_id, cluster_id; + char name_buf[16]; -void re_initialize(void) -{ - free_all_buffers(); - setup_all_buffers(); - fprintf(outf, "turbostat: re-initialized with num_cpus %d\n", topo.num_cpus); -} + sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", i); -void set_max_cpu_num(void) -{ - FILE *filep; - int base_cpu; - unsigned long dummy; - char pathname[64]; + if (access(path_base, R_OK)) + err(1, "%s: %s\n", __func__, path_base); - base_cpu = sched_getcpu(); - if (base_cpu < 0) - err(1, "cannot find calling cpu ID"); - sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", base_cpu); + sprintf(path, "%s/package_id", path_base); + package_id = read_sysfs_int(path); - filep = fopen_or_die(pathname, "r"); - topo.max_cpu_num = 0; - while (fscanf(filep, "%lx,", &dummy) == 1) - topo.max_cpu_num += BITMASK_SIZE; - fclose(filep); - topo.max_cpu_num--; /* 0 based */ -} + sprintf(path, "%s/domain_id", path_base); + domain_id = read_sysfs_int(path); -/* - * count_cpus() - * remember the last one seen, it will be the max - */ -int count_cpus(int cpu) -{ - UNUSED(cpu); + sprintf(path, "%s/fabric_cluster_id", path_base); + cluster_id = read_sysfs_int(path); - topo.num_cpus++; - return 0; + sprintf(path, "%s/current_freq_khz", path_base); + sprintf(name_buf, "UMHz%d.%d", domain_id, cluster_id); + + /* + * Once add_couter() is called, that counter is always read + * and reported -- So it is effectively (enabled & present). + * Only call add_counter() here if legacy BIC_UNCORE_MHZ (UncMHz) + * is (enabled). Since we are in this routine, we + * know we will not probe and set (present) the legacy counter. + * + * This allows "--show/--hide UncMHz" to be effective for + * the clustered MHz counters, as a group. + */ + if BIC_IS_ENABLED + (BIC_UNCORE_MHZ) + add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, package_id); + + if (quiet) + continue; + + sprintf(path, "%s/min_freq_khz", path_base); + k = read_sysfs_int(path); + sprintf(path, "%s/max_freq_khz", path_base); + l = read_sysfs_int(path); + fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", package_id, domain_id, cluster_id, k / 1000, l / 1000); + + sprintf(path, "%s/initial_min_freq_khz", path_base); + k = read_sysfs_int(path); + sprintf(path, "%s/initial_max_freq_khz", path_base); + l = read_sysfs_int(path); + fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000); + + sprintf(path, "%s/current_freq_khz", path_base); + k = read_sysfs_int(path); + fprintf(outf, " %d MHz\n", k / 1000); + } } -int mark_cpu_present(int cpu) +static void probe_intel_uncore_frequency(void) { - CPU_SET_S(cpu, cpu_present_setsize, cpu_present_set); - return 0; + if (!genuine_intel) + return; + + if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00", R_OK) == 0) + probe_intel_uncore_frequency_cluster(); + else + probe_intel_uncore_frequency_legacy(); } -int init_thread_id(int cpu) +static void set_graphics_fp(char *path, int idx) { - cpus[cpu].thread_id = -1; - return 0; + if (!access(path, R_OK)) + gfx_info[idx].fp = fopen_or_die(path, "r"); } -/* - * snapshot_proc_interrupts() - * - * read and record summary of /proc/interrupts - * - * return 1 if config change requires a restart, else return 0 - */ -int snapshot_proc_interrupts(void) -{ - static FILE *fp; - int column, retval; +/* Enlarge this if there are /sys/class/drm/card2 ... */ +#define GFX_MAX_CARDS 2 - if (fp == NULL) - fp = fopen_or_die("/proc/interrupts", "r"); - else - rewind(fp); +static void probe_graphics(void) +{ + char path[PATH_MAX]; + int i; - /* read 1st line of /proc/interrupts to get cpu* name for each column */ - for (column = 0; column < topo.num_cpus; ++column) { - int cpu_number; + /* Xe graphics sysfs knobs */ + if (!access("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", R_OK)) { + FILE *fp; + char buf[8]; + bool gt0_is_gt; - retval = fscanf(fp, " CPU%d", &cpu_number); - if (retval != 1) - break; + fp = fopen("/sys/class/drm/card0/device/tile0/gt0/gtidle/name", "r"); + if (!fp) + goto next; - if (cpu_number > topo.max_cpu_num) { - warn("/proc/interrupts: cpu%d: > %d", cpu_number, topo.max_cpu_num); - return 1; + if (!fread(buf, sizeof(char), 7, fp)) { + fclose(fp); + goto next; } + fclose(fp); - irq_column_2_cpu[column] = cpu_number; - irqs_per_cpu[cpu_number] = 0; - } + if (!strncmp(buf, "gt0-rc", strlen("gt0-rc"))) + gt0_is_gt = true; + else if (!strncmp(buf, "gt0-mc", strlen("gt0-mc"))) + gt0_is_gt = false; + else + goto next; - /* read /proc/interrupt count lines and sum up irqs per cpu */ - while (1) { - int column; - char buf[64]; + set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", gt0_is_gt ? GFX_rc6 : SAM_mc6); + + set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/cur_freq", gt0_is_gt ? GFX_MHz : SAM_MHz); + + set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/act_freq", gt0_is_gt ? GFX_ACTMHz : SAM_ACTMHz); + + set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/gtidle/idle_residency_ms", gt0_is_gt ? SAM_mc6 : GFX_rc6); + + set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/cur_freq", gt0_is_gt ? SAM_MHz : GFX_MHz); + + set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/act_freq", gt0_is_gt ? SAM_ACTMHz : GFX_ACTMHz); - retval = fscanf(fp, " %s:", buf); /* flush irq# "N:" */ - if (retval != 1) + goto end; + } + +next: + /* New i915 graphics sysfs knobs */ + for (i = 0; i < GFX_MAX_CARDS; i++) { + snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rc6_residency_ms", i); + if (!access(path, R_OK)) break; + } - /* read the count per cpu */ - for (column = 0; column < topo.num_cpus; ++column) { + if (i == GFX_MAX_CARDS) + goto legacy_i915; - int cpu_number, irq_count; + snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rc6_residency_ms", i); + set_graphics_fp(path, GFX_rc6); - retval = fscanf(fp, " %d", &irq_count); - if (retval != 1) - break; + snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rps_cur_freq_mhz", i); + set_graphics_fp(path, GFX_MHz); - cpu_number = irq_column_2_cpu[column]; - irqs_per_cpu[cpu_number] += irq_count; + snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rps_act_freq_mhz", i); + set_graphics_fp(path, GFX_ACTMHz); - } + snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rc6_residency_ms", i); + set_graphics_fp(path, SAM_mc6); - while (getc(fp) != '\n') ; /* flush interrupt description */ + snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rps_cur_freq_mhz", i); + set_graphics_fp(path, SAM_MHz); - } - return 0; -} + snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rps_act_freq_mhz", i); + set_graphics_fp(path, SAM_ACTMHz); -/* - * snapshot_gfx_rc6_ms() - * - * record snapshot of - * /sys/class/drm/card0/power/rc6_residency_ms - * - * return 1 if config change requires a restart, else return 0 - */ -int snapshot_gfx_rc6_ms(void) -{ - FILE *fp; - int retval; + goto end; - fp = fopen_or_die("/sys/class/drm/card0/power/rc6_residency_ms", "r"); +legacy_i915: + /* Fall back to traditional i915 graphics sysfs knobs */ + set_graphics_fp("/sys/class/drm/card0/power/rc6_residency_ms", GFX_rc6); - retval = fscanf(fp, "%lld", &gfx_cur_rc6_ms); - if (retval != 1) - err(1, "GFX rc6"); + set_graphics_fp("/sys/class/drm/card0/gt_cur_freq_mhz", GFX_MHz); + if (!gfx_info[GFX_MHz].fp) + set_graphics_fp("/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", GFX_MHz); - fclose(fp); + set_graphics_fp("/sys/class/drm/card0/gt_act_freq_mhz", GFX_ACTMHz); + if (!gfx_info[GFX_ACTMHz].fp) + set_graphics_fp("/sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz", GFX_ACTMHz); - return 0; +end: + if (gfx_info[GFX_rc6].fp) + BIC_PRESENT(BIC_GFX_rc6); + if (gfx_info[GFX_MHz].fp) + BIC_PRESENT(BIC_GFXMHz); + if (gfx_info[GFX_ACTMHz].fp) + BIC_PRESENT(BIC_GFXACTMHz); + if (gfx_info[SAM_mc6].fp) + BIC_PRESENT(BIC_SAM_mc6); + if (gfx_info[SAM_MHz].fp) + BIC_PRESENT(BIC_SAMMHz); + if (gfx_info[SAM_ACTMHz].fp) + BIC_PRESENT(BIC_SAMACTMHz); } -/* - * snapshot_gfx_mhz() - * - * record snapshot of - * /sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz - * - * return 1 if config change requires a restart, else return 0 - */ -int snapshot_gfx_mhz(void) +static void dump_sysfs_cstate_config(void) { - static FILE *fp; - int retval; + char path[64]; + char name_buf[16]; + char desc[64]; + FILE *input; + int state; + char *sp; - if (fp == NULL) - fp = fopen_or_die("/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", "r"); - else { - rewind(fp); - fflush(fp); + if (access("/sys/devices/system/cpu/cpuidle", R_OK)) { + fprintf(outf, "cpuidle not loaded\n"); + return; } - retval = fscanf(fp, "%d", &gfx_cur_mhz); - if (retval != 1) - err(1, "GFX MHz"); + dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_driver"); + dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor"); + dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor_ro"); - return 0; -} + for (state = 0; state < 10; ++state) { -/* - * snapshot_gfx_cur_mhz() - * - * record snapshot of - * /sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz - * - * return 1 if config change requires a restart, else return 0 - */ -int snapshot_gfx_act_mhz(void) -{ - static FILE *fp; - int retval; + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state); + input = fopen(path, "r"); + if (input == NULL) + continue; + if (!fgets(name_buf, sizeof(name_buf), input)) + err(1, "%s: failed to read file", path); - if (fp == NULL) - fp = fopen_or_die("/sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz", "r"); - else { - rewind(fp); - fflush(fp); - } + /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */ + sp = strchr(name_buf, '-'); + if (!sp) + sp = strchrnul(name_buf, '\n'); + *sp = '\0'; + fclose(input); - retval = fscanf(fp, "%d", &gfx_act_mhz); - if (retval != 1) - err(1, "GFX ACT MHz"); + remove_underbar(name_buf); - return 0; + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", base_cpu, state); + input = fopen(path, "r"); + if (input == NULL) + continue; + if (!fgets(desc, sizeof(desc), input)) + err(1, "%s: failed to read file", path); + + fprintf(outf, "cpu%d: %s: %s", base_cpu, name_buf, desc); + fclose(input); + } } -/* - * snapshot_cpu_lpi() - * - * record snapshot of - * /sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us - */ -int snapshot_cpu_lpi_us(void) +static void dump_sysfs_pstate_config(void) { - FILE *fp; - int retval; + char path[64]; + char driver_buf[64]; + char governor_buf[64]; + FILE *input; + int turbo; - fp = fopen_or_die("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", "r"); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", base_cpu); + input = fopen(path, "r"); + if (input == NULL) { + fprintf(outf, "NSFOD %s\n", path); + return; + } + if (!fgets(driver_buf, sizeof(driver_buf), input)) + err(1, "%s: failed to read file", path); + fclose(input); - retval = fscanf(fp, "%lld", &cpuidle_cur_cpu_lpi_us); - if (retval != 1) { - fprintf(stderr, "Disabling Low Power Idle CPU output\n"); - BIC_NOT_PRESENT(BIC_CPU_LPI); - fclose(fp); - return -1; + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", base_cpu); + input = fopen(path, "r"); + if (input == NULL) { + fprintf(outf, "NSFOD %s\n", path); + return; } + if (!fgets(governor_buf, sizeof(governor_buf), input)) + err(1, "%s: failed to read file", path); + fclose(input); - fclose(fp); + fprintf(outf, "cpu%d: cpufreq driver: %s", base_cpu, driver_buf); + fprintf(outf, "cpu%d: cpufreq governor: %s", base_cpu, governor_buf); - return 0; + sprintf(path, "/sys/devices/system/cpu/cpufreq/boost"); + input = fopen(path, "r"); + if (input != NULL) { + if (fscanf(input, "%d", &turbo) != 1) + err(1, "%s: failed to parse number from file", path); + fprintf(outf, "cpufreq boost: %d\n", turbo); + fclose(input); + } + + sprintf(path, "/sys/devices/system/cpu/intel_pstate/no_turbo"); + input = fopen(path, "r"); + if (input != NULL) { + if (fscanf(input, "%d", &turbo) != 1) + err(1, "%s: failed to parse number from file", path); + fprintf(outf, "cpufreq intel_pstate no_turbo: %d\n", turbo); + fclose(input); + } } /* - * snapshot_sys_lpi() - * - * record snapshot of sys_lpi_file + * print_epb() + * Decode the ENERGY_PERF_BIAS MSR */ -int snapshot_sys_lpi_us(void) +int print_epb(PER_THREAD_PARAMS) { - FILE *fp; - int retval; + char *epb_string; + int cpu, epb; - fp = fopen_or_die(sys_lpi_file, "r"); + UNUSED(c); + UNUSED(p); - retval = fscanf(fp, "%lld", &cpuidle_cur_sys_lpi_us); - if (retval != 1) { - fprintf(stderr, "Disabling Low Power Idle System output\n"); - BIC_NOT_PRESENT(BIC_SYS_LPI); - fclose(fp); + if (!has_epb) + return 0; + + cpu = t->cpu_id; + + /* EPB is per-package */ + if (!is_cpu_first_thread_in_package(t, c, p)) + return 0; + + if (cpu_migrate(cpu)) { + fprintf(outf, "print_epb: Could not migrate to CPU %d\n", cpu); return -1; } - fclose(fp); + + epb = get_epb(cpu); + if (epb < 0) + return 0; + + switch (epb) { + case ENERGY_PERF_BIAS_PERFORMANCE: + epb_string = "performance"; + break; + case ENERGY_PERF_BIAS_NORMAL: + epb_string = "balanced"; + break; + case ENERGY_PERF_BIAS_POWERSAVE: + epb_string = "powersave"; + break; + default: + epb_string = "custom"; + break; + } + fprintf(outf, "cpu%d: EPB: %d (%s)\n", cpu, epb, epb_string); return 0; } /* - * snapshot /proc and /sys files - * - * return 1 if configuration restart needed, else return 0 + * print_hwp() + * Decode the MSR_HWP_CAPABILITIES */ -int snapshot_proc_sysfs_files(void) +int print_hwp(PER_THREAD_PARAMS) { - if (DO_BIC(BIC_IRQ)) - if (snapshot_proc_interrupts()) - return 1; - - if (DO_BIC(BIC_GFX_rc6)) - snapshot_gfx_rc6_ms(); - - if (DO_BIC(BIC_GFXMHz)) - snapshot_gfx_mhz(); + unsigned long long msr; + int cpu; - if (DO_BIC(BIC_GFXACTMHz)) - snapshot_gfx_act_mhz(); + UNUSED(c); + UNUSED(p); - if (DO_BIC(BIC_CPU_LPI)) - snapshot_cpu_lpi_us(); + if (no_msr) + return 0; - if (DO_BIC(BIC_SYS_LPI)) - snapshot_sys_lpi_us(); + if (!has_hwp) + return 0; - return 0; -} + cpu = t->cpu_id; -int exit_requested; + /* MSR_HWP_CAPABILITIES is per-package */ + if (!is_cpu_first_thread_in_package(t, c, p)) + return 0; -static void signal_handler(int signal) -{ - switch (signal) { - case SIGINT: - exit_requested = 1; - if (debug) - fprintf(stderr, " SIGINT\n"); - break; - case SIGUSR1: - if (debug > 1) - fprintf(stderr, "SIGUSR1\n"); - break; + if (cpu_migrate(cpu)) { + fprintf(outf, "print_hwp: Could not migrate to CPU %d\n", cpu); + return -1; } -} -void setup_signal_handler(void) -{ - struct sigaction sa; + if (get_msr(cpu, MSR_PM_ENABLE, &msr)) + return 0; - memset(&sa, 0, sizeof(sa)); + fprintf(outf, "cpu%d: MSR_PM_ENABLE: 0x%08llx (%sHWP)\n", cpu, msr, (msr & (1 << 0)) ? "" : "No-"); - sa.sa_handler = &signal_handler; + /* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */ + if ((msr & (1 << 0)) == 0) + return 0; - if (sigaction(SIGINT, &sa, NULL) < 0) - err(1, "sigaction SIGINT"); - if (sigaction(SIGUSR1, &sa, NULL) < 0) - err(1, "sigaction SIGUSR1"); -} + if (get_msr(cpu, MSR_HWP_CAPABILITIES, &msr)) + return 0; -void do_sleep(void) -{ - struct timeval tout; - struct timespec rest; - fd_set readfds; - int retval; + fprintf(outf, "cpu%d: MSR_HWP_CAPABILITIES: 0x%08llx " + "(high %d guar %d eff %d low %d)\n", + cpu, msr, + (unsigned int)HWP_HIGHEST_PERF(msr), + (unsigned int)HWP_GUARANTEED_PERF(msr), (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr)); - FD_ZERO(&readfds); - FD_SET(0, &readfds); + if (get_msr(cpu, MSR_HWP_REQUEST, &msr)) + return 0; - if (ignore_stdin) { - nanosleep(&interval_ts, NULL); - return; - } + fprintf(outf, "cpu%d: MSR_HWP_REQUEST: 0x%08llx " + "(min %d max %d des %d epp 0x%x window 0x%x pkg 0x%x)\n", + cpu, msr, + (unsigned int)(((msr) >> 0) & 0xff), + (unsigned int)(((msr) >> 8) & 0xff), + (unsigned int)(((msr) >> 16) & 0xff), + (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1)); - tout = interval_tv; - retval = select(1, &readfds, NULL, NULL, &tout); + if (has_hwp_pkg) { + if (get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr)) + return 0; - if (retval == 1) { - switch (getc(stdin)) { - case 'q': - exit_requested = 1; - break; - case EOF: - /* - * 'stdin' is a pipe closed on the other end. There - * won't be any further input. - */ - ignore_stdin = 1; - /* Sleep the rest of the time */ - rest.tv_sec = (tout.tv_sec + tout.tv_usec / 1000000); - rest.tv_nsec = (tout.tv_usec % 1000000) * 1000; - nanosleep(&rest, NULL); - } + fprintf(outf, "cpu%d: MSR_HWP_REQUEST_PKG: 0x%08llx " + "(min %d max %d des %d epp 0x%x window 0x%x)\n", + cpu, msr, + (unsigned int)(((msr) >> 0) & 0xff), + (unsigned int)(((msr) >> 8) & 0xff), + (unsigned int)(((msr) >> 16) & 0xff), (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3)); } -} - -int get_msr_sum(int cpu, off_t offset, unsigned long long *msr) -{ - int ret, idx; - unsigned long long msr_cur, msr_last; + if (has_hwp_notify) { + if (get_msr(cpu, MSR_HWP_INTERRUPT, &msr)) + return 0; - if (!per_cpu_msr_sum) - return 1; + fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx " + "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis"); + } + if (get_msr(cpu, MSR_HWP_STATUS, &msr)) + return 0; - idx = offset_to_idx(offset); - if (idx < 0) - return idx; - /* get_msr_sum() = sum + (get_msr() - last) */ - ret = get_msr(cpu, offset, &msr_cur); - if (ret) - return ret; - msr_last = per_cpu_msr_sum[cpu].entries[idx].last; - DELTA_WRAP32(msr_cur, msr_last); - *msr = msr_last + per_cpu_msr_sum[cpu].entries[idx].sum; + fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx " + "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-"); return 0; } -timer_t timerid; - -/* Timer callback, update the sum of MSRs periodically. */ -static int update_msr_sum(struct thread_data *t, struct core_data *c, struct pkg_data *p) +/* + * print_perf_limit() + */ +int print_perf_limit(PER_THREAD_PARAMS) { - int i, ret; - int cpu = t->cpu_id; + unsigned long long msr; + int cpu; UNUSED(c); UNUSED(p); - for (i = IDX_PKG_ENERGY; i < IDX_COUNT; i++) { - unsigned long long msr_cur, msr_last; - off_t offset; + if (no_msr) + return 0; - if (!idx_valid(i)) - continue; - offset = idx_to_offset(i); - if (offset < 0) - continue; - ret = get_msr(cpu, offset, &msr_cur); - if (ret) { - fprintf(outf, "Can not update msr(0x%llx)\n", (unsigned long long)offset); - continue; - } + cpu = t->cpu_id; - msr_last = per_cpu_msr_sum[cpu].entries[i].last; - per_cpu_msr_sum[cpu].entries[i].last = msr_cur & 0xffffffff; + /* per-package */ + if (!is_cpu_first_thread_in_package(t, c, p)) + return 0; - DELTA_WRAP32(msr_cur, msr_last); - per_cpu_msr_sum[cpu].entries[i].sum += msr_last; + if (cpu_migrate(cpu)) { + fprintf(outf, "print_perf_limit: Could not migrate to CPU %d\n", cpu); + return -1; + } + + if (platform->plr_msrs & PLR_CORE) { + get_msr(cpu, MSR_CORE_PERF_LIMIT_REASONS, &msr); + fprintf(outf, "cpu%d: MSR_CORE_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr); + fprintf(outf, " (Active: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)", + (msr & 1 << 15) ? "bit15, " : "", + (msr & 1 << 14) ? "bit14, " : "", + (msr & 1 << 13) ? "Transitions, " : "", + (msr & 1 << 12) ? "MultiCoreTurbo, " : "", + (msr & 1 << 11) ? "PkgPwrL2, " : "", + (msr & 1 << 10) ? "PkgPwrL1, " : "", + (msr & 1 << 9) ? "CorePwr, " : "", + (msr & 1 << 8) ? "Amps, " : "", + (msr & 1 << 6) ? "VR-Therm, " : "", + (msr & 1 << 5) ? "Auto-HWP, " : "", + (msr & 1 << 4) ? "Graphics, " : "", + (msr & 1 << 2) ? "bit2, " : "", (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : ""); + fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n", + (msr & 1 << 31) ? "bit31, " : "", + (msr & 1 << 30) ? "bit30, " : "", + (msr & 1 << 29) ? "Transitions, " : "", + (msr & 1 << 28) ? "MultiCoreTurbo, " : "", + (msr & 1 << 27) ? "PkgPwrL2, " : "", + (msr & 1 << 26) ? "PkgPwrL1, " : "", + (msr & 1 << 25) ? "CorePwr, " : "", + (msr & 1 << 24) ? "Amps, " : "", + (msr & 1 << 22) ? "VR-Therm, " : "", + (msr & 1 << 21) ? "Auto-HWP, " : "", + (msr & 1 << 20) ? "Graphics, " : "", + (msr & 1 << 18) ? "bit18, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : ""); + + } + if (platform->plr_msrs & PLR_GFX) { + get_msr(cpu, MSR_GFX_PERF_LIMIT_REASONS, &msr); + fprintf(outf, "cpu%d: MSR_GFX_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr); + fprintf(outf, " (Active: %s%s%s%s%s%s%s%s)", + (msr & 1 << 0) ? "PROCHOT, " : "", + (msr & 1 << 1) ? "ThermStatus, " : "", + (msr & 1 << 4) ? "Graphics, " : "", + (msr & 1 << 6) ? "VR-Therm, " : "", + (msr & 1 << 8) ? "Amps, " : "", + (msr & 1 << 9) ? "GFXPwr, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); + fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s)\n", + (msr & 1 << 16) ? "PROCHOT, " : "", + (msr & 1 << 17) ? "ThermStatus, " : "", + (msr & 1 << 20) ? "Graphics, " : "", + (msr & 1 << 22) ? "VR-Therm, " : "", + (msr & 1 << 24) ? "Amps, " : "", + (msr & 1 << 25) ? "GFXPwr, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); + } + if (platform->plr_msrs & PLR_RING) { + get_msr(cpu, MSR_RING_PERF_LIMIT_REASONS, &msr); + fprintf(outf, "cpu%d: MSR_RING_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr); + fprintf(outf, " (Active: %s%s%s%s%s%s)", + (msr & 1 << 0) ? "PROCHOT, " : "", + (msr & 1 << 1) ? "ThermStatus, " : "", + (msr & 1 << 6) ? "VR-Therm, " : "", + (msr & 1 << 8) ? "Amps, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); + fprintf(outf, " (Logged: %s%s%s%s%s%s)\n", + (msr & 1 << 16) ? "PROCHOT, " : "", + (msr & 1 << 17) ? "ThermStatus, " : "", + (msr & 1 << 22) ? "VR-Therm, " : "", + (msr & 1 << 24) ? "Amps, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); } return 0; } -static void msr_record_handler(union sigval v) +#define RAPL_POWER_GRANULARITY 0x7FFF /* 15 bit power granularity */ +#define RAPL_TIME_GRANULARITY 0x3F /* 6 bit time granularity */ + +double get_quirk_tdp(void) { - UNUSED(v); + if (platform->rapl_quirk_tdp) + return platform->rapl_quirk_tdp; - for_all_cpus(update_msr_sum, EVEN_COUNTERS); + return 135.0; } -void msr_sum_record(void) +double get_tdp_intel(void) { - struct itimerspec its; - struct sigevent sev; + unsigned long long msr; - per_cpu_msr_sum = calloc(topo.max_cpu_num + 1, sizeof(struct msr_sum_array)); - if (!per_cpu_msr_sum) { - fprintf(outf, "Can not allocate memory for long time MSR.\n"); - return; - } - /* - * Signal handler might be restricted, so use thread notifier instead. - */ - memset(&sev, 0, sizeof(struct sigevent)); - sev.sigev_notify = SIGEV_THREAD; - sev.sigev_notify_function = msr_record_handler; + if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) + if (!get_msr(base_cpu, MSR_PKG_POWER_INFO, &msr)) + return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units; + return get_quirk_tdp(); +} - sev.sigev_value.sival_ptr = &timerid; - if (timer_create(CLOCK_REALTIME, &sev, &timerid) == -1) { - fprintf(outf, "Can not create timer.\n"); - goto release_msr; - } +double get_tdp_amd(void) +{ + return get_quirk_tdp(); +} - its.it_value.tv_sec = 0; - its.it_value.tv_nsec = 1; - /* - * A wraparound time has been calculated early. - * Some sources state that the peak power for a - * microprocessor is usually 1.5 times the TDP rating, - * use 2 * TDP for safety. - */ - its.it_interval.tv_sec = rapl_joule_counter_range / 2; - its.it_interval.tv_nsec = 0; +void rapl_probe_intel(void) +{ + unsigned long long msr; + unsigned int time_unit; + double tdp; - if (timer_settime(timerid, 0, &its, NULL) == -1) { - fprintf(outf, "Can not set timer.\n"); - goto release_timer; + if (rapl_joules) { + CLR_BIC(BIC_SysWatt, &bic_enabled); + CLR_BIC(BIC_PkgWatt, &bic_enabled); + CLR_BIC(BIC_CorWatt, &bic_enabled); + CLR_BIC(BIC_RAMWatt, &bic_enabled); + CLR_BIC(BIC_GFXWatt, &bic_enabled); + } else { + CLR_BIC(BIC_Sys_J, &bic_enabled); + CLR_BIC(BIC_Pkg_J, &bic_enabled); + CLR_BIC(BIC_Cor_J, &bic_enabled); + CLR_BIC(BIC_RAM_J, &bic_enabled); + CLR_BIC(BIC_GFX_J, &bic_enabled); } - return; -release_timer: - timer_delete(timerid); -release_msr: - free(per_cpu_msr_sum); -} + if (!valid_rapl_msrs || no_msr) + return; -/* - * set_my_sched_priority(pri) - * return previous - */ -int set_my_sched_priority(int priority) -{ - int retval; - int original_priority; + if (!(valid_rapl_msrs & RAPL_PKG_PERF_STATUS)) + CLR_BIC(BIC_PKG__, &bic_enabled); + if (!(valid_rapl_msrs & RAPL_DRAM_PERF_STATUS)) + CLR_BIC(BIC_RAM__, &bic_enabled); + + /* units on package 0, verify later other packages match */ + if (get_msr(base_cpu, MSR_RAPL_POWER_UNIT, &msr)) + return; + + rapl_power_units = 1.0 / (1 << (msr & 0xF)); + if (platform->has_rapl_divisor) + rapl_energy_units = 1.0 * (1 << (msr >> 8 & 0x1F)) / 1000000; + else + rapl_energy_units = 1.0 / (1 << (msr >> 8 & 0x1F)); + + if (platform->has_fixed_rapl_unit) + rapl_dram_energy_units = (15.3 / 1000000); + else + rapl_dram_energy_units = rapl_energy_units; + + if (platform->has_fixed_rapl_psys_unit) + rapl_psys_energy_units = 1.0; + else + rapl_psys_energy_units = rapl_energy_units; - errno = 0; - original_priority = getpriority(PRIO_PROCESS, 0); - if (errno && (original_priority == -1)) - err(errno, "getpriority"); + time_unit = msr >> 16 & 0xF; + if (time_unit == 0) + time_unit = 0xA; - retval = setpriority(PRIO_PROCESS, 0, priority); - if (retval) - errx(retval, "capget(CAP_SYS_NICE) failed,try \"# setcap cap_sys_nice=ep %s\"", progname); + rapl_time_units = 1.0 / (1 << (time_unit)); - errno = 0; - retval = getpriority(PRIO_PROCESS, 0); - if (retval != priority) - err(retval, "getpriority(%d) != setpriority(%d)", retval, priority); + tdp = get_tdp_intel(); - return original_priority; + rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp; + if (!quiet) + fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp); } -void turbostat_loop() +void rapl_probe_amd(void) { - int retval; - int restarted = 0; - unsigned int done_iters = 0; + unsigned long long msr; + double tdp; - setup_signal_handler(); + if (rapl_joules) { + CLR_BIC(BIC_SysWatt, &bic_enabled); + CLR_BIC(BIC_CorWatt, &bic_enabled); + } else { + CLR_BIC(BIC_Pkg_J, &bic_enabled); + CLR_BIC(BIC_Cor_J, &bic_enabled); + } - /* - * elevate own priority for interval mode - */ - set_my_sched_priority(-20); + if (!valid_rapl_msrs || no_msr) + return; -restart: - restarted++; + if (get_msr(base_cpu, MSR_RAPL_PWR_UNIT, &msr)) + return; - snapshot_proc_sysfs_files(); - retval = for_all_cpus(get_counters, EVEN_COUNTERS); - first_counter_read = 0; - if (retval < -1) { - exit(retval); - } else if (retval == -1) { - if (restarted > 10) { - exit(retval); - } - re_initialize(); - goto restart; - } - restarted = 0; - done_iters = 0; - gettimeofday(&tv_even, (struct timezone *)NULL); + rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf)); + rapl_energy_units = ldexp(1.0, -(msr >> 8 & 0x1f)); + rapl_power_units = ldexp(1.0, -(msr & 0xf)); - while (1) { - if (for_all_proc_cpus(cpu_is_not_present)) { - re_initialize(); - goto restart; - } - do_sleep(); - if (snapshot_proc_sysfs_files()) - goto restart; - retval = for_all_cpus(get_counters, ODD_COUNTERS); - if (retval < -1) { - exit(retval); - } else if (retval == -1) { - re_initialize(); - goto restart; - } - gettimeofday(&tv_odd, (struct timezone *)NULL); - timersub(&tv_odd, &tv_even, &tv_delta); - if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS)) { - re_initialize(); - goto restart; - } - compute_average(EVEN_COUNTERS); - format_all_counters(EVEN_COUNTERS); - flush_output_stdout(); - if (exit_requested) - break; - if (num_iterations && ++done_iters >= num_iterations) - break; - do_sleep(); - if (snapshot_proc_sysfs_files()) - goto restart; - retval = for_all_cpus(get_counters, EVEN_COUNTERS); - if (retval < -1) { - exit(retval); - } else if (retval == -1) { - re_initialize(); - goto restart; - } - gettimeofday(&tv_even, (struct timezone *)NULL); - timersub(&tv_even, &tv_odd, &tv_delta); - if (for_all_cpus_2(delta_cpu, EVEN_COUNTERS, ODD_COUNTERS)) { - re_initialize(); - goto restart; - } - compute_average(ODD_COUNTERS); - format_all_counters(ODD_COUNTERS); - flush_output_stdout(); - if (exit_requested) - break; - if (num_iterations && ++done_iters >= num_iterations) - break; - } + tdp = get_tdp_amd(); + + rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp; + if (!quiet) + fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp); } -void check_dev_msr() +void print_power_limit_msr(int cpu, unsigned long long msr, char *label) { - struct stat sb; - char pathname[32]; + fprintf(outf, "cpu%d: %s: %sabled (%0.3f Watts, %f sec, clamp %sabled)\n", + cpu, label, + ((msr >> 15) & 1) ? "EN" : "DIS", + ((msr >> 0) & 0x7FFF) * rapl_power_units, + (1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units, (((msr >> 16) & 1) ? "EN" : "DIS")); - sprintf(pathname, "/dev/cpu/%d/msr", base_cpu); - if (stat(pathname, &sb)) - if (system("/sbin/modprobe msr > /dev/null 2>&1")) - err(-5, "no /dev/cpu/0/msr, Try \"# modprobe msr\" "); + return; } -/* - * check for CAP_SYS_RAWIO - * return 0 on success - * return 1 on fail - */ -int check_for_cap_sys_rawio(void) +static int fread_int(char *path, int *val) { - cap_t caps; - cap_flag_value_t cap_flag_value; + FILE *filep; + int ret; - caps = cap_get_proc(); - if (caps == NULL) - err(-6, "cap_get_proc\n"); + filep = fopen(path, "r"); + if (!filep) + return -1; - if (cap_get_flag(caps, CAP_SYS_RAWIO, CAP_EFFECTIVE, &cap_flag_value)) - err(-6, "cap_get\n"); + ret = fscanf(filep, "%d", val); + fclose(filep); + return ret; +} - if (cap_flag_value != CAP_SET) { - warnx("capget(CAP_SYS_RAWIO) failed," " try \"# setcap cap_sys_rawio=ep %s\"", progname); - return 1; - } +static int fread_ull(char *path, unsigned long long *val) +{ + FILE *filep; + int ret; - if (cap_free(caps) == -1) - err(-6, "cap_free\n"); + filep = fopen(path, "r"); + if (!filep) + return -1; - return 0; + ret = fscanf(filep, "%llu", val); + fclose(filep); + return ret; } -void check_permissions(void) +static int fread_str(char *path, char *buf, int size) { - int do_exit = 0; - char pathname[32]; + FILE *filep; + int ret; + char *cp; - /* check for CAP_SYS_RAWIO */ - do_exit += check_for_cap_sys_rawio(); + filep = fopen(path, "r"); + if (!filep) + return -1; - /* test file permissions */ - sprintf(pathname, "/dev/cpu/%d/msr", base_cpu); - if (euidaccess(pathname, R_OK)) { - do_exit++; - warn("/dev/cpu/0/msr open failed, try chown or chmod +r /dev/cpu/*/msr"); - } + ret = fread(buf, 1, size, filep); + fclose(filep); - /* if all else fails, thell them to be root */ - if (do_exit) - if (getuid() != 0) - warnx("... or simply run as root"); + /* replace '\n' with '\0' */ + cp = strchr(buf, '\n'); + if (cp != NULL) + *cp = '\0'; - if (do_exit) - exit(-6); + return ret; } -/* - * NHM adds support for additional MSRs: - * - * MSR_SMI_COUNT 0x00000034 - * - * MSR_PLATFORM_INFO 0x000000ce - * MSR_PKG_CST_CONFIG_CONTROL 0x000000e2 - * - * MSR_MISC_PWR_MGMT 0x000001aa - * - * MSR_PKG_C3_RESIDENCY 0x000003f8 - * MSR_PKG_C6_RESIDENCY 0x000003f9 - * MSR_CORE_C3_RESIDENCY 0x000003fc - * MSR_CORE_C6_RESIDENCY 0x000003fd - * - * Side effect: - * sets global pkg_cstate_limit to decode MSR_PKG_CST_CONFIG_CONTROL - * sets has_misc_feature_control - */ -int probe_nhm_msrs(unsigned int family, unsigned int model) +#define PATH_RAPL_SYSFS "/sys/class/powercap" + +static int dump_one_domain(char *domain_path) { - unsigned long long msr; - unsigned int base_ratio; - int *pkg_cstate_limits; + char path[PATH_MAX]; + char str[PATH_MAX]; + unsigned long long val; + int constraint; + int enable; + int ret; - if (!genuine_intel) - return 0; + snprintf(path, PATH_MAX, "%s/name", domain_path); + ret = fread_str(path, str, PATH_MAX); + if (ret <= 0) + return -1; - if (family != 6) - return 0; + fprintf(outf, "%s: %s", domain_path + strlen(PATH_RAPL_SYSFS) + 1, str); - bclk = discover_bclk(family, model); + snprintf(path, PATH_MAX, "%s/enabled", domain_path); + ret = fread_int(path, &enable); + if (ret <= 0) + return -1; - switch (model) { - case INTEL_FAM6_NEHALEM: /* Core i7 and i5 Processor - Clarksfield, Lynnfield, Jasper Forest */ - case INTEL_FAM6_NEHALEM_EX: /* Nehalem-EX Xeon - Beckton */ - pkg_cstate_limits = nhm_pkg_cstate_limits; - break; - case INTEL_FAM6_SANDYBRIDGE: /* SNB */ - case INTEL_FAM6_SANDYBRIDGE_X: /* SNB Xeon */ - case INTEL_FAM6_IVYBRIDGE: /* IVB */ - case INTEL_FAM6_IVYBRIDGE_X: /* IVB Xeon */ - pkg_cstate_limits = snb_pkg_cstate_limits; - has_misc_feature_control = 1; - break; - case INTEL_FAM6_HASWELL: /* HSW */ - case INTEL_FAM6_HASWELL_G: /* HSW */ - case INTEL_FAM6_HASWELL_X: /* HSX */ - case INTEL_FAM6_HASWELL_L: /* HSW */ - case INTEL_FAM6_BROADWELL: /* BDW */ - case INTEL_FAM6_BROADWELL_G: /* BDW */ - case INTEL_FAM6_BROADWELL_X: /* BDX */ - case INTEL_FAM6_SKYLAKE_L: /* SKL */ - case INTEL_FAM6_CANNONLAKE_L: /* CNL */ - pkg_cstate_limits = hsw_pkg_cstate_limits; - has_misc_feature_control = 1; - break; - case INTEL_FAM6_SKYLAKE_X: /* SKX */ - case INTEL_FAM6_SAPPHIRERAPIDS_X: /* SPR */ - pkg_cstate_limits = skx_pkg_cstate_limits; - has_misc_feature_control = 1; - break; - case INTEL_FAM6_ICELAKE_X: /* ICX */ - pkg_cstate_limits = icx_pkg_cstate_limits; - has_misc_feature_control = 1; - break; - case INTEL_FAM6_ATOM_SILVERMONT: /* BYT */ - no_MSR_MISC_PWR_MGMT = 1; - /* FALLTHRU */ - case INTEL_FAM6_ATOM_SILVERMONT_D: /* AVN */ - pkg_cstate_limits = slv_pkg_cstate_limits; - break; - case INTEL_FAM6_ATOM_AIRMONT: /* AMT */ - pkg_cstate_limits = amt_pkg_cstate_limits; - no_MSR_MISC_PWR_MGMT = 1; - break; - case INTEL_FAM6_XEON_PHI_KNL: /* PHI */ - pkg_cstate_limits = phi_pkg_cstate_limits; - break; - case INTEL_FAM6_ATOM_GOLDMONT: /* BXT */ - case INTEL_FAM6_ATOM_GOLDMONT_PLUS: - case INTEL_FAM6_ATOM_GOLDMONT_D: /* DNV */ - case INTEL_FAM6_ATOM_TREMONT: /* EHL */ - case INTEL_FAM6_ATOM_TREMONT_D: /* JVL */ - pkg_cstate_limits = glm_pkg_cstate_limits; - break; - default: + if (!enable) { + fputs(" disabled\n", outf); return 0; } - get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); - pkg_cstate_limit = pkg_cstate_limits[msr & 0xF]; - get_msr(base_cpu, MSR_PLATFORM_INFO, &msr); - base_ratio = (msr >> 8) & 0xFF; + for (constraint = 0;; constraint++) { + snprintf(path, PATH_MAX, "%s/constraint_%d_time_window_us", domain_path, constraint); + ret = fread_ull(path, &val); + if (ret <= 0) + break; - base_hz = base_ratio * bclk * 1000000; - has_base_hz = 1; - return 1; -} + if (val > 1000000) + fprintf(outf, " %0.1fs", (double)val / 1000000); + else if (val > 1000) + fprintf(outf, " %0.1fms", (double)val / 1000); + else + fprintf(outf, " %0.1fus", (double)val); -/* - * SLV client has support for unique MSRs: - * - * MSR_CC6_DEMOTION_POLICY_CONFIG - * MSR_MC6_DEMOTION_POLICY_CONFIG - */ + snprintf(path, PATH_MAX, "%s/constraint_%d_power_limit_uw", domain_path, constraint); + ret = fread_ull(path, &val); + if (ret > 0 && val) + fprintf(outf, ":%lluW", val / 1000000); -int has_slv_msrs(unsigned int family, unsigned int model) -{ - if (!genuine_intel) - return 0; + snprintf(path, PATH_MAX, "%s/constraint_%d_max_power_uw", domain_path, constraint); + ret = fread_ull(path, &val); + if (ret > 0 && val) + fprintf(outf, ",max:%lluW", val / 1000000); + } + fputc('\n', outf); - if (family != 6) - return 0; + return 0; +} + +static int print_rapl_sysfs(void) +{ + DIR *dir, *cdir; + struct dirent *entry, *centry; + char path[PATH_MAX]; + char str[PATH_MAX]; - switch (model) { - case INTEL_FAM6_ATOM_SILVERMONT: - case INTEL_FAM6_ATOM_SILVERMONT_MID: - case INTEL_FAM6_ATOM_AIRMONT_MID: + if ((dir = opendir(PATH_RAPL_SYSFS)) == NULL) { + warn("open %s failed", PATH_RAPL_SYSFS); return 1; } + + while ((entry = readdir(dir)) != NULL) { + if (strlen(entry->d_name) > 100) + continue; + + if (strncmp(entry->d_name, "intel-rapl", strlen("intel-rapl"))) + continue; + + snprintf(path, PATH_MAX, "%s/%s/name", PATH_RAPL_SYSFS, entry->d_name); + + /* Parse top level domains first, including package and psys */ + fread_str(path, str, PATH_MAX); + if (strncmp(str, "package", strlen("package")) && strncmp(str, "psys", strlen("psys"))) + continue; + + snprintf(path, PATH_MAX, "%s/%s", PATH_RAPL_SYSFS, entry->d_name); + if ((cdir = opendir(path)) == NULL) { + perror("opendir() error"); + return 1; + } + + dump_one_domain(path); + + while ((centry = readdir(cdir)) != NULL) { + if (strncmp(centry->d_name, "intel-rapl", strlen("intel-rapl"))) + continue; + snprintf(path, PATH_MAX, "%s/%s/%s", PATH_RAPL_SYSFS, entry->d_name, centry->d_name); + dump_one_domain(path); + } + closedir(cdir); + } + + closedir(dir); return 0; } -int is_dnv(unsigned int family, unsigned int model) +int print_rapl(PER_THREAD_PARAMS) { + unsigned long long msr; + const char *msr_name; + int cpu; - if (!genuine_intel) + UNUSED(c); + UNUSED(p); + + if (!valid_rapl_msrs) return 0; - if (family != 6) + /* RAPL counters are per package, so print only for 1st thread/package */ + if (!is_cpu_first_thread_in_package(t, c, p)) return 0; - switch (model) { - case INTEL_FAM6_ATOM_GOLDMONT_D: - return 1; + cpu = t->cpu_id; + if (cpu_migrate(cpu)) { + fprintf(outf, "print_rapl: Could not migrate to CPU %d\n", cpu); + return -1; + } + + if (valid_rapl_msrs & RAPL_AMD_F17H) { + msr_name = "MSR_RAPL_PWR_UNIT"; + if (get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr)) + return -1; + } else { + msr_name = "MSR_RAPL_POWER_UNIT"; + if (get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr)) + return -1; } - return 0; -} -int is_bdx(unsigned int family, unsigned int model) -{ + fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, + rapl_power_units, rapl_energy_units, rapl_time_units); - if (!genuine_intel) - return 0; + if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) { - if (family != 6) - return 0; + if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr)) + return -5; - switch (model) { - case INTEL_FAM6_BROADWELL_X: - return 1; - } - return 0; -} + fprintf(outf, "cpu%d: MSR_PKG_POWER_INFO: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n", + cpu, msr, + ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units, + ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units, + ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); -int is_skx(unsigned int family, unsigned int model) -{ + } + if (valid_rapl_msrs & RAPL_PKG) { - if (!genuine_intel) - return 0; + if (get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr)) + return -9; - if (family != 6) - return 0; + fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 63) & 1 ? "" : "UN"); - switch (model) { - case INTEL_FAM6_SKYLAKE_X: - return 1; - } - return 0; -} + print_power_limit_msr(cpu, msr, "PKG Limit #1"); + fprintf(outf, "cpu%d: PKG Limit #2: %sabled (%0.3f Watts, %f* sec, clamp %sabled)\n", + cpu, + ((msr >> 47) & 1) ? "EN" : "DIS", + ((msr >> 32) & 0x7FFF) * rapl_power_units, + (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, ((msr >> 48) & 1) ? "EN" : "DIS"); -int is_icx(unsigned int family, unsigned int model) -{ + if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr)) + return -9; - if (!genuine_intel) - return 0; + fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr); + fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n", + cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN"); + } - if (family != 6) - return 0; + if (valid_rapl_msrs & RAPL_DRAM_POWER_INFO) { + if (get_msr(cpu, MSR_DRAM_POWER_INFO, &msr)) + return -6; - switch (model) { - case INTEL_FAM6_ICELAKE_X: - return 1; + fprintf(outf, "cpu%d: MSR_DRAM_POWER_INFO,: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n", + cpu, msr, + ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units, + ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units, + ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); } - return 0; -} + if (valid_rapl_msrs & RAPL_DRAM) { + if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr)) + return -9; + fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); -int is_spr(unsigned int family, unsigned int model) -{ + print_power_limit_msr(cpu, msr, "DRAM Limit"); + } + if (valid_rapl_msrs & RAPL_CORE_POLICY) { + if (get_msr(cpu, MSR_PP0_POLICY, &msr)) + return -7; - if (!genuine_intel) - return 0; + fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF); + } + if (valid_rapl_msrs & RAPL_CORE_POWER_LIMIT) { + if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr)) + return -9; + fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); + print_power_limit_msr(cpu, msr, "Cores Limit"); + } + if (valid_rapl_msrs & RAPL_GFX) { + if (get_msr(cpu, MSR_PP1_POLICY, &msr)) + return -8; - if (family != 6) - return 0; + fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF); - switch (model) { - case INTEL_FAM6_SAPPHIRERAPIDS_X: - return 1; + if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr)) + return -9; + fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); + print_power_limit_msr(cpu, msr, "GFX Limit"); } return 0; } -int is_ehl(unsigned int family, unsigned int model) +/* + * probe_rapl_msrs + * + * initialize global valid_rapl_msrs to platform->plat_rapl_msrs + * only if PKG_ENERGY counter is enumerated and reads non-zero + */ +void probe_rapl_msrs(void) { - if (!genuine_intel) - return 0; + int ret; + off_t offset; + unsigned long long msr_value; - if (family != 6) - return 0; + if (no_msr) + return; - switch (model) { - case INTEL_FAM6_ATOM_TREMONT: - return 1; + if ((platform->plat_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H)) == 0) + return; + + offset = idx_to_offset(IDX_PKG_ENERGY); + if (offset < 0) + return; + + ret = get_msr(base_cpu, offset, &msr_value); + if (ret) { + if (debug) + fprintf(outf, "Can not read RAPL_PKG_ENERGY MSR(0x%llx)\n", (unsigned long long)offset); + return; } - return 0; + if (msr_value == 0) { + if (debug) + fprintf(outf, "RAPL_PKG_ENERGY MSR(0x%llx) == ZERO: disabling all RAPL MSRs\n", (unsigned long long)offset); + return; + } + + valid_rapl_msrs = platform->plat_rapl_msrs; /* success */ } -int is_jvl(unsigned int family, unsigned int model) +/* + * probe_rapl() + * + * sets rapl_power_units, rapl_energy_units, rapl_time_units + */ +void probe_rapl(void) { - if (!genuine_intel) - return 0; + probe_rapl_msrs(); - if (family != 6) - return 0; + if (genuine_intel) + rapl_probe_intel(); + if (authentic_amd || hygon_genuine) + rapl_probe_amd(); - switch (model) { - case INTEL_FAM6_ATOM_TREMONT_D: - return 1; - } - return 0; -} + if (quiet) + return; -int has_turbo_ratio_limit(unsigned int family, unsigned int model) -{ - if (has_slv_msrs(family, model)) - return 0; + print_rapl_sysfs(); - if (family != 6) - return 0; + if (!valid_rapl_msrs || no_msr) + return; - switch (model) { - /* Nehalem compatible, but do not include turbo-ratio limit support */ - case INTEL_FAM6_NEHALEM_EX: /* Nehalem-EX Xeon - Beckton */ - case INTEL_FAM6_XEON_PHI_KNL: /* PHI - Knights Landing (different MSR definition) */ - return 0; - default: - return 1; - } + for_all_cpus(print_rapl, ODD_COUNTERS); } -int has_atom_turbo_ratio_limit(unsigned int family, unsigned int model) +/* + * MSR_IA32_TEMPERATURE_TARGET indicates the temperature where + * the Thermal Control Circuit (TCC) activates. + * This is usually equal to tjMax. + * + * Older processors do not have this MSR, so there we guess, + * but also allow cmdline over-ride with -T. + * + * Several MSR temperature values are in units of degrees-C + * below this value, including the Digital Thermal Sensor (DTS), + * Package Thermal Management Sensor (PTM), and thermal event thresholds. + */ +int set_temperature_target(PER_THREAD_PARAMS) { - if (has_slv_msrs(family, model)) - return 1; + unsigned long long msr; + unsigned int tcc_default, tcc_offset; + int cpu; - return 0; -} + UNUSED(c); + UNUSED(p); -int has_ivt_turbo_ratio_limit(unsigned int family, unsigned int model) -{ - if (!genuine_intel) + /* tj_max is used only for dts or ptm */ + if (!(do_dts || do_ptm)) return 0; - if (family != 6) + /* this is a per-package concept */ + if (!is_cpu_first_thread_in_package(t, c, p)) return 0; - switch (model) { - case INTEL_FAM6_IVYBRIDGE_X: /* IVB Xeon */ - case INTEL_FAM6_HASWELL_X: /* HSW Xeon */ - return 1; - default: - return 0; + cpu = t->cpu_id; + if (cpu_migrate(cpu)) { + fprintf(outf, "Could not migrate to CPU %d\n", cpu); + return -1; } -} -int has_hsw_turbo_ratio_limit(unsigned int family, unsigned int model) -{ - if (!genuine_intel) + if (tj_max_override != 0) { + tj_max = tj_max_override; + fprintf(outf, "cpu%d: Using cmdline TCC Target (%d C)\n", cpu, tj_max); return 0; + } - if (family != 6) - return 0; + /* Temperature Target MSR is Nehalem and newer only */ + if (!platform->has_nhm_msrs || no_msr) + goto guess; - switch (model) { - case INTEL_FAM6_HASWELL_X: /* HSW Xeon */ - return 1; - default: - return 0; - } -} + if (get_msr(base_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr)) + goto guess; -int has_knl_turbo_ratio_limit(unsigned int family, unsigned int model) -{ - if (!genuine_intel) - return 0; + tcc_default = (msr >> 16) & 0xFF; - if (family != 6) - return 0; + if (!quiet) { + int bits = platform->tcc_offset_bits; + unsigned long long enabled = 0; - switch (model) { - case INTEL_FAM6_XEON_PHI_KNL: /* Knights Landing */ - return 1; - default: - return 0; + if (bits && !get_msr(base_cpu, MSR_PLATFORM_INFO, &enabled)) + enabled = (enabled >> 30) & 1; + + if (bits && enabled) { + tcc_offset = (msr >> 24) & GENMASK(bits - 1, 0); + fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C) (%d default - %d offset)\n", + cpu, msr, tcc_default - tcc_offset, tcc_default, tcc_offset); + } else { + fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C)\n", cpu, msr, tcc_default); + } } -} -int has_glm_turbo_ratio_limit(unsigned int family, unsigned int model) -{ - if (!genuine_intel) - return 0; + if (!tcc_default) + goto guess; - if (family != 6) - return 0; + tj_max = tcc_default; - switch (model) { - case INTEL_FAM6_ATOM_GOLDMONT: - case INTEL_FAM6_SKYLAKE_X: - case INTEL_FAM6_ICELAKE_X: - case INTEL_FAM6_SAPPHIRERAPIDS_X: - return 1; - default: - return 0; - } + return 0; + +guess: + tj_max = TJMAX_DEFAULT; + fprintf(outf, "cpu%d: Guessing tjMax %d C, Please use -T to specify\n", cpu, tj_max); + + return 0; } -int has_config_tdp(unsigned int family, unsigned int model) +int print_thermal(PER_THREAD_PARAMS) { - if (!genuine_intel) - return 0; + unsigned long long msr; + unsigned int dts, dts2; + int cpu; + + UNUSED(c); + UNUSED(p); - if (family != 6) + if (no_msr) return 0; - switch (model) { - case INTEL_FAM6_IVYBRIDGE: /* IVB */ - case INTEL_FAM6_HASWELL: /* HSW */ - case INTEL_FAM6_HASWELL_X: /* HSX */ - case INTEL_FAM6_HASWELL_L: /* HSW */ - case INTEL_FAM6_HASWELL_G: /* HSW */ - case INTEL_FAM6_BROADWELL: /* BDW */ - case INTEL_FAM6_BROADWELL_G: /* BDW */ - case INTEL_FAM6_BROADWELL_X: /* BDX */ - case INTEL_FAM6_SKYLAKE_L: /* SKL */ - case INTEL_FAM6_CANNONLAKE_L: /* CNL */ - case INTEL_FAM6_SKYLAKE_X: /* SKX */ - case INTEL_FAM6_ICELAKE_X: /* ICX */ - case INTEL_FAM6_SAPPHIRERAPIDS_X: /* SPR */ - case INTEL_FAM6_XEON_PHI_KNL: /* Knights Landing */ - return 1; - default: + if (!(do_dts || do_ptm)) return 0; - } -} -/* - * tcc_offset_bits: - * 0: Tcc Offset not supported (Default) - * 6: Bit 29:24 of MSR_PLATFORM_INFO - * 4: Bit 27:24 of MSR_PLATFORM_INFO - */ -void check_tcc_offset(int model) -{ - unsigned long long msr; + cpu = t->cpu_id; - if (!genuine_intel) - return; + /* DTS is per-core, no need to print for each thread */ + if (!is_cpu_first_thread_in_core(t, c)) + return 0; - switch (model) { - case INTEL_FAM6_SKYLAKE_L: - case INTEL_FAM6_SKYLAKE: - case INTEL_FAM6_KABYLAKE_L: - case INTEL_FAM6_KABYLAKE: - case INTEL_FAM6_ICELAKE_L: - case INTEL_FAM6_ICELAKE: - case INTEL_FAM6_TIGERLAKE_L: - case INTEL_FAM6_TIGERLAKE: - case INTEL_FAM6_COMETLAKE: - if (!get_msr(base_cpu, MSR_PLATFORM_INFO, &msr)) { - msr = (msr >> 30) & 1; - if (msr) - tcc_offset_bits = 6; - } - return; - default: - return; + if (cpu_migrate(cpu)) { + fprintf(outf, "print_thermal: Could not migrate to CPU %d\n", cpu); + return -1; } -} - -static void remove_underbar(char *s) -{ - char *to = s; - while (*s) { - if (*s != '_') - *to++ = *s; - s++; - } + if (do_ptm && is_cpu_first_core_in_package(t, p)) { + if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr)) + return 0; - *to = 0; -} + dts = (msr >> 16) & 0x7F; + fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_STATUS: 0x%08llx (%d C)\n", cpu, msr, tj_max - dts); -static void dump_turbo_ratio_info(unsigned int family, unsigned int model) -{ - if (!has_turbo) - return; + if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &msr)) + return 0; - if (has_hsw_turbo_ratio_limit(family, model)) - dump_hsw_turbo_ratio_limits(); + dts = (msr >> 16) & 0x7F; + dts2 = (msr >> 8) & 0x7F; + fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2); + } - if (has_ivt_turbo_ratio_limit(family, model)) - dump_ivt_turbo_ratio_limits(); + if (do_dts && debug) { + unsigned int resolution; - if (has_turbo_ratio_limit(family, model)) { - dump_turbo_ratio_limits(MSR_TURBO_RATIO_LIMIT, family, model); + if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr)) + return 0; - if (is_hybrid) - dump_turbo_ratio_limits(MSR_SECONDARY_TURBO_RATIO_LIMIT, family, model); - } + dts = (msr >> 16) & 0x7F; + resolution = (msr >> 27) & 0xF; + fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", cpu, msr, tj_max - dts, resolution); - if (has_atom_turbo_ratio_limit(family, model)) - dump_atom_turbo_ratio_limits(); + if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr)) + return 0; - if (has_knl_turbo_ratio_limit(family, model)) - dump_knl_turbo_ratio_limits(); + dts = (msr >> 16) & 0x7F; + dts2 = (msr >> 8) & 0x7F; + fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2); + } - if (has_config_tdp(family, model)) - dump_config_tdp(); + return 0; } -static void dump_cstate_pstate_config_info(unsigned int family, unsigned int model) +void probe_thermal(void) { - if (!do_nhm_platform_info) + if (!access("/sys/devices/system/cpu/cpu0/thermal_throttle/core_throttle_count", R_OK)) + BIC_PRESENT(BIC_CORE_THROT_CNT); + else + BIC_NOT_PRESENT(BIC_CORE_THROT_CNT); + + for_all_cpus(set_temperature_target, ODD_COUNTERS); + + if (quiet) return; - dump_nhm_platform_info(); - dump_turbo_ratio_info(family, model); - dump_nhm_cst_cfg(); + for_all_cpus(print_thermal, ODD_COUNTERS); } -static int read_sysfs_int(char *path) +int get_cpu_type(PER_THREAD_PARAMS) { - FILE *input; - int retval = -1; + unsigned int eax, ebx, ecx, edx; - input = fopen(path, "r"); - if (input == NULL) { - if (debug) - fprintf(outf, "NSFOD %s\n", path); - return (-1); + UNUSED(c); + UNUSED(p); + + if (!genuine_intel) + return 0; + + if (cpu_migrate(t->cpu_id)) { + fprintf(outf, "Could not migrate to CPU %d\n", t->cpu_id); + return -1; } - if (fscanf(input, "%d", &retval) != 1) - err(1, "%s: failed to read int from file", path); - fclose(input); - return (retval); + if (max_level < 0x1a) + return 0; + + __cpuid(0x1a, eax, ebx, ecx, edx); + eax = (eax >> 24) & 0xFF; + if (eax == 0x20) + t->is_atom = true; + return 0; } -static void dump_sysfs_file(char *path) +void decode_feature_control_msr(void) { - FILE *input; - char cpuidle_buf[64]; + unsigned long long msr; - input = fopen(path, "r"); - if (input == NULL) { - if (debug) - fprintf(outf, "NSFOD %s\n", path); + if (no_msr) return; - } - if (!fgets(cpuidle_buf, sizeof(cpuidle_buf), input)) - err(1, "%s: failed to read file", path); - fclose(input); - fprintf(outf, "%s: %s", strrchr(path, '/') + 1, cpuidle_buf); + if (!get_msr(base_cpu, MSR_IA32_FEAT_CTL, &msr)) + fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n", + base_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : ""); } -static void intel_uncore_frequency_probe(void) +void decode_misc_enable_msr(void) { - int i, j; - char path[128]; + unsigned long long msr; - if (!genuine_intel) + if (no_msr) return; - if (access("/sys/devices/system/cpu/intel_uncore_frequency/package_00_die_00", R_OK)) + if (!genuine_intel) return; - if (!access("/sys/devices/system/cpu/intel_uncore_frequency/package_00_die_00/current_freq_khz", R_OK)) - BIC_PRESENT(BIC_UNCORE_MHZ); + if (!get_msr(base_cpu, MSR_IA32_MISC_ENABLE, &msr)) + fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n", + base_cpu, msr, + msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-", + msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-", + msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-", + msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : ""); +} - if (quiet) - return; +void decode_misc_feature_control(void) +{ + unsigned long long msr; - for (i = 0; i < topo.num_packages; ++i) { - for (j = 0; j < topo.num_die; ++j) { - int k, l; + if (no_msr) + return; - sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_0%d_die_0%d/min_freq_khz", - i, j); - k = read_sysfs_int(path); - sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_0%d_die_0%d/max_freq_khz", - i, j); - l = read_sysfs_int(path); - fprintf(outf, "Uncore Frequency pkg%d die%d: %d - %d MHz ", i, j, k / 1000, l / 1000); + if (!platform->has_msr_misc_feature_control) + return; - sprintf(path, - "/sys/devices/system/cpu/intel_uncore_frequency/package_0%d_die_0%d/initial_min_freq_khz", - i, j); - k = read_sysfs_int(path); - sprintf(path, - "/sys/devices/system/cpu/intel_uncore_frequency/package_0%d_die_0%d/initial_max_freq_khz", - i, j); - l = read_sysfs_int(path); - fprintf(outf, "(%d - %d MHz)\n", k / 1000, l / 1000); - } - } + if (!get_msr(base_cpu, MSR_MISC_FEATURE_CONTROL, &msr)) + fprintf(outf, + "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n", + base_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", + msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : ""); } -static void dump_sysfs_cstate_config(void) +/* + * Decode MSR_MISC_PWR_MGMT + * + * Decode the bits according to the Nehalem documentation + * bit[0] seems to continue to have same meaning going forward + * bit[1] less so... + */ +void decode_misc_pwr_mgmt_msr(void) { - char path[64]; - char name_buf[16]; - char desc[64]; - FILE *input; - int state; - char *sp; + unsigned long long msr; - if (access("/sys/devices/system/cpu/cpuidle", R_OK)) { - fprintf(outf, "cpuidle not loaded\n"); + if (no_msr) return; - } - dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_driver"); - dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor"); - dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor_ro"); + if (!platform->has_msr_misc_pwr_mgmt) + return; - for (state = 0; state < 10; ++state) { + if (!get_msr(base_cpu, MSR_MISC_PWR_MGMT, &msr)) + fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n", + base_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS"); +} - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state); - input = fopen(path, "r"); - if (input == NULL) - continue; - if (!fgets(name_buf, sizeof(name_buf), input)) - err(1, "%s: failed to read file", path); +/* + * Decode MSR_CC6_DEMOTION_POLICY_CONFIG, MSR_MC6_DEMOTION_POLICY_CONFIG + * + * This MSRs are present on Silvermont processors, + * Intel Atom processor E3000 series (Baytrail), and friends. + */ +void decode_c6_demotion_policy_msr(void) +{ + unsigned long long msr; - /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */ - sp = strchr(name_buf, '-'); - if (!sp) - sp = strchrnul(name_buf, '\n'); - *sp = '\0'; - fclose(input); + if (no_msr) + return; - remove_underbar(name_buf); + if (!platform->has_msr_c6_demotion_policy_config) + return; - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", base_cpu, state); - input = fopen(path, "r"); - if (input == NULL) - continue; - if (!fgets(desc, sizeof(desc), input)) - err(1, "%s: failed to read file", path); + if (!get_msr(base_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr)) + fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", + base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); - fprintf(outf, "cpu%d: %s: %s", base_cpu, name_buf, desc); - fclose(input); - } + if (!get_msr(base_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr)) + fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", + base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); } -static void dump_sysfs_pstate_config(void) +void print_dev_latency(void) { - char path[64]; - char driver_buf[64]; - char governor_buf[64]; - FILE *input; - int turbo; + char *path = "/dev/cpu_dma_latency"; + int fd; + int value; + int retval; - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", base_cpu); - input = fopen(path, "r"); - if (input == NULL) { - fprintf(outf, "NSFOD %s\n", path); + fd = open(path, O_RDONLY); + if (fd < 0) { + if (debug) + warnx("Read %s failed", path); return; } - if (!fgets(driver_buf, sizeof(driver_buf), input)) - err(1, "%s: failed to read file", path); - fclose(input); - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", base_cpu); - input = fopen(path, "r"); - if (input == NULL) { - fprintf(outf, "NSFOD %s\n", path); + retval = read(fd, (void *)&value, sizeof(int)); + if (retval != sizeof(int)) { + warn("read failed %s", path); + close(fd); return; } - if (!fgets(governor_buf, sizeof(governor_buf), input)) - err(1, "%s: failed to read file", path); - fclose(input); + fprintf(outf, "/dev/cpu_dma_latency: %d usec (%s)\n", value, value == 2000000000 ? "default" : "constrained"); - fprintf(outf, "cpu%d: cpufreq driver: %s", base_cpu, driver_buf); - fprintf(outf, "cpu%d: cpufreq governor: %s", base_cpu, governor_buf); + close(fd); +} - sprintf(path, "/sys/devices/system/cpu/cpufreq/boost"); - input = fopen(path, "r"); - if (input != NULL) { - if (fscanf(input, "%d", &turbo) != 1) - err(1, "%s: failed to parse number from file", path); - fprintf(outf, "cpufreq boost: %d\n", turbo); - fclose(input); - } +static int has_perf_instr_count_access(void) +{ + int fd; - sprintf(path, "/sys/devices/system/cpu/intel_pstate/no_turbo"); - input = fopen(path, "r"); - if (input != NULL) { - if (fscanf(input, "%d", &turbo) != 1) - err(1, "%s: failed to parse number from file", path); - fprintf(outf, "cpufreq intel_pstate no_turbo: %d\n", turbo); - fclose(input); - } + if (no_perf) + return 0; + + fd = open_perf_counter(base_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0); + if (fd != -1) + close(fd); + + if (fd == -1) + warnx("Failed to access %s. Some of the counters may not be available\n" + "\tRun as root to enable them or use %s to disable the access explicitly", "perf instructions retired counter", + "'--hide IPC' or '--no-perf'"); + + return (fd != -1); } -/* - * print_epb() - * Decode the ENERGY_PERF_BIAS MSR - */ -int print_epb(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int add_rapl_perf_counter(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, double *scale_, enum rapl_unit *unit_) { - char *epb_string; - int cpu, epb; + int ret = -1; - UNUSED(c); - UNUSED(p); + if (no_perf) + return -1; - if (!has_epb) - return 0; + if (!cai->perf_name) + return -1; - cpu = t->cpu_id; + const double scale = read_perf_scale(cai->perf_subsys, cai->perf_name); - /* EPB is per-package */ - if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE) || !(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) - return 0; + if (scale == 0.0) + goto end; - if (cpu_migrate(cpu)) { - fprintf(outf, "print_epb: Could not migrate to CPU %d\n", cpu); - return -1; - } + const enum rapl_unit unit = read_perf_rapl_unit(cai->perf_subsys, cai->perf_name); + + if (unit == RAPL_UNIT_INVALID) + goto end; + + const unsigned int rapl_type = read_perf_type(cai->perf_subsys); + const unsigned int rapl_energy_pkg_config = read_perf_config(cai->perf_subsys, cai->perf_name); + + ret = open_perf_counter(cpu, rapl_type, rapl_energy_pkg_config, rci->fd_perf, PERF_FORMAT_GROUP); + if (ret == -1) + goto end; - epb = get_epb(cpu); - if (epb < 0) - return 0; + /* If it's the first counter opened, make it a group descriptor */ + if (rci->fd_perf == -1) + rci->fd_perf = ret; - switch (epb) { - case ENERGY_PERF_BIAS_PERFORMANCE: - epb_string = "performance"; - break; - case ENERGY_PERF_BIAS_NORMAL: - epb_string = "balanced"; - break; - case ENERGY_PERF_BIAS_POWERSAVE: - epb_string = "powersave"; - break; - default: - epb_string = "custom"; - break; - } - fprintf(outf, "cpu%d: EPB: %d (%s)\n", cpu, epb, epb_string); + *scale_ = scale; + *unit_ = unit; - return 0; +end: + if (debug >= 2) + fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu); + + return ret; } /* - * print_hwp() - * Decode the MSR_HWP_CAPABILITIES + * Linux-perf manages the HW instructions-retired counter + * by enabling when requested, and hiding rollover */ -int print_hwp(struct thread_data *t, struct core_data *c, struct pkg_data *p) +void linux_perf_init(void) { - unsigned long long msr; - int cpu; + if (access("/proc/sys/kernel/perf_event_paranoid", F_OK)) + return; - UNUSED(c); - UNUSED(p); + if (BIC_IS_ENABLED(BIC_IPC) && cpuid_has_aperf_mperf) { + fd_instr_count_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); + if (fd_instr_count_percpu == NULL) + err(-1, "calloc fd_instr_count_percpu"); + } + if (BIC_IS_ENABLED(BIC_LLC_RPS)) { + fd_llc_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); + if (fd_llc_percpu == NULL) + err(-1, "calloc fd_llc_percpu"); + } +} - if (!has_hwp) - return 0; +void rapl_perf_init(void) +{ + const unsigned int num_domains = get_rapl_num_domains(); + bool *domain_visited = calloc(num_domains, sizeof(bool)); - cpu = t->cpu_id; + rapl_counter_info_perdomain = calloc(num_domains, sizeof(*rapl_counter_info_perdomain)); + if (rapl_counter_info_perdomain == NULL) + err(-1, "calloc rapl_counter_info_percpu"); + rapl_counter_info_perdomain_size = num_domains; - /* MSR_HWP_CAPABILITIES is per-package */ - if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE) || !(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) - return 0; + /* + * Initialize rapl_counter_info_percpu + */ + for (unsigned int domain_id = 0; domain_id < num_domains; ++domain_id) { + struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[domain_id]; - if (cpu_migrate(cpu)) { - fprintf(outf, "print_hwp: Could not migrate to CPU %d\n", cpu); - return -1; + rci->fd_perf = -1; + for (size_t i = 0; i < NUM_RAPL_COUNTERS; ++i) { + rci->data[i] = 0; + rci->source[i] = COUNTER_SOURCE_NONE; + } } - if (get_msr(cpu, MSR_PM_ENABLE, &msr)) - return 0; + /* + * Open/probe the counters + * If can't get it via perf, fallback to MSR + */ + for (size_t i = 0; i < ARRAY_SIZE(rapl_counter_arch_infos); ++i) { - fprintf(outf, "cpu%d: MSR_PM_ENABLE: 0x%08llx (%sHWP)\n", cpu, msr, (msr & (1 << 0)) ? "" : "No-"); + const struct rapl_counter_arch_info *const cai = &rapl_counter_arch_infos[i]; + bool has_counter = 0; + double scale; + enum rapl_unit unit; + unsigned int next_domain; - /* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */ - if ((msr & (1 << 0)) == 0) - return 0; + if (!BIC_IS_ENABLED(cai->bic_number)) + continue; - if (get_msr(cpu, MSR_HWP_CAPABILITIES, &msr)) - return 0; + memset(domain_visited, 0, num_domains * sizeof(*domain_visited)); - fprintf(outf, "cpu%d: MSR_HWP_CAPABILITIES: 0x%08llx " - "(high %d guar %d eff %d low %d)\n", - cpu, msr, - (unsigned int)HWP_HIGHEST_PERF(msr), - (unsigned int)HWP_GUARANTEED_PERF(msr), - (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr)); + for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) { - if (get_msr(cpu, MSR_HWP_REQUEST, &msr)) - return 0; + if (cpu_is_not_allowed(cpu)) + continue; - fprintf(outf, "cpu%d: MSR_HWP_REQUEST: 0x%08llx " - "(min %d max %d des %d epp 0x%x window 0x%x pkg 0x%x)\n", - cpu, msr, - (unsigned int)(((msr) >> 0) & 0xff), - (unsigned int)(((msr) >> 8) & 0xff), - (unsigned int)(((msr) >> 16) & 0xff), - (unsigned int)(((msr) >> 24) & 0xff), - (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1)); + /* Skip already seen and handled RAPL domains */ + next_domain = get_rapl_domain_id(cpu); - if (has_hwp_pkg) { - if (get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr)) - return 0; + assert(next_domain < num_domains); - fprintf(outf, "cpu%d: MSR_HWP_REQUEST_PKG: 0x%08llx " - "(min %d max %d des %d epp 0x%x window 0x%x)\n", - cpu, msr, - (unsigned int)(((msr) >> 0) & 0xff), - (unsigned int)(((msr) >> 8) & 0xff), - (unsigned int)(((msr) >> 16) & 0xff), - (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3)); - } - if (has_hwp_notify) { - if (get_msr(cpu, MSR_HWP_INTERRUPT, &msr)) - return 0; + if (domain_visited[next_domain]) + continue; - fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx " - "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", - cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis"); + domain_visited[next_domain] = 1; + + if ((cai->flags & RAPL_COUNTER_FLAG_PLATFORM_COUNTER) && (cpu != base_cpu)) + continue; + + struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[next_domain]; + + /* + * rapl_counter_arch_infos[] can have multiple entries describing the same + * counter, due to the difference from different platforms/Vendors. + * E.g. rapl_counter_arch_infos[0] and rapl_counter_arch_infos[1] share the + * same perf_subsys and perf_name, but with different MSR address. + * rapl_counter_arch_infos[0] is for Intel and rapl_counter_arch_infos[1] + * is for AMD. + * In this case, it is possible that multiple rapl_counter_arch_infos[] + * entries are probed just because their perf/msr is duplicate and valid. + * + * Thus need a check to avoid re-probe the same counters. + */ + if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE) + break; + + /* Use perf API for this counter */ + if (add_rapl_perf_counter(cpu, rci, cai, &scale, &unit) != -1) { + rci->source[cai->rci_index] = COUNTER_SOURCE_PERF; + rci->scale[cai->rci_index] = scale * cai->compat_scale; + rci->unit[cai->rci_index] = unit; + rci->flags[cai->rci_index] = cai->flags; + + /* Use MSR for this counter */ + } else if (add_rapl_msr_counter(cpu, cai) >= 0) { + rci->source[cai->rci_index] = COUNTER_SOURCE_MSR; + rci->msr[cai->rci_index] = cai->msr; + rci->msr_mask[cai->rci_index] = cai->msr_mask; + rci->msr_shift[cai->rci_index] = cai->msr_shift; + rci->unit[cai->rci_index] = RAPL_UNIT_JOULES; + rci->scale[cai->rci_index] = *cai->platform_rapl_msr_scale * cai->compat_scale; + rci->flags[cai->rci_index] = cai->flags; + } + + if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE) + has_counter = 1; + } + + /* If any CPU has access to the counter, make it present */ + if (has_counter) + BIC_PRESENT(cai->bic_number); } - if (get_msr(cpu, MSR_HWP_STATUS, &msr)) - return 0; - fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx " - "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", - cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-"); + free(domain_visited); +} - return 0; +/* Assumes msr_counter_info is populated */ +static int has_amperf_access(void) +{ + return cpuid_has_aperf_mperf && msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present && + msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present; } -/* - * print_perf_limit() - */ -int print_perf_limit(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int *get_cstate_perf_group_fd(struct cstate_counter_info_t *cci, const char *group_name) { - unsigned long long msr; - int cpu; + if (strcmp(group_name, "cstate_core") == 0) + return &cci->fd_perf_core; - UNUSED(c); - UNUSED(p); + if (strcmp(group_name, "cstate_pkg") == 0) + return &cci->fd_perf_pkg; - cpu = t->cpu_id; + return NULL; +} - /* per-package */ - if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE) || !(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) - return 0; +int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai) +{ + int ret = -1; - if (cpu_migrate(cpu)) { - fprintf(outf, "print_perf_limit: Could not migrate to CPU %d\n", cpu); + if (no_perf) return -1; - } - if (do_core_perf_limit_reasons) { - get_msr(cpu, MSR_CORE_PERF_LIMIT_REASONS, &msr); - fprintf(outf, "cpu%d: MSR_CORE_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr); - fprintf(outf, " (Active: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)", - (msr & 1 << 15) ? "bit15, " : "", - (msr & 1 << 14) ? "bit14, " : "", - (msr & 1 << 13) ? "Transitions, " : "", - (msr & 1 << 12) ? "MultiCoreTurbo, " : "", - (msr & 1 << 11) ? "PkgPwrL2, " : "", - (msr & 1 << 10) ? "PkgPwrL1, " : "", - (msr & 1 << 9) ? "CorePwr, " : "", - (msr & 1 << 8) ? "Amps, " : "", - (msr & 1 << 6) ? "VR-Therm, " : "", - (msr & 1 << 5) ? "Auto-HWP, " : "", - (msr & 1 << 4) ? "Graphics, " : "", - (msr & 1 << 2) ? "bit2, " : "", - (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : ""); - fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n", - (msr & 1 << 31) ? "bit31, " : "", - (msr & 1 << 30) ? "bit30, " : "", - (msr & 1 << 29) ? "Transitions, " : "", - (msr & 1 << 28) ? "MultiCoreTurbo, " : "", - (msr & 1 << 27) ? "PkgPwrL2, " : "", - (msr & 1 << 26) ? "PkgPwrL1, " : "", - (msr & 1 << 25) ? "CorePwr, " : "", - (msr & 1 << 24) ? "Amps, " : "", - (msr & 1 << 22) ? "VR-Therm, " : "", - (msr & 1 << 21) ? "Auto-HWP, " : "", - (msr & 1 << 20) ? "Graphics, " : "", - (msr & 1 << 18) ? "bit18, " : "", - (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : ""); + if (!cai->perf_name) + return -1; - } - if (do_gfx_perf_limit_reasons) { - get_msr(cpu, MSR_GFX_PERF_LIMIT_REASONS, &msr); - fprintf(outf, "cpu%d: MSR_GFX_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr); - fprintf(outf, " (Active: %s%s%s%s%s%s%s%s)", - (msr & 1 << 0) ? "PROCHOT, " : "", - (msr & 1 << 1) ? "ThermStatus, " : "", - (msr & 1 << 4) ? "Graphics, " : "", - (msr & 1 << 6) ? "VR-Therm, " : "", - (msr & 1 << 8) ? "Amps, " : "", - (msr & 1 << 9) ? "GFXPwr, " : "", - (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); - fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s)\n", - (msr & 1 << 16) ? "PROCHOT, " : "", - (msr & 1 << 17) ? "ThermStatus, " : "", - (msr & 1 << 20) ? "Graphics, " : "", - (msr & 1 << 22) ? "VR-Therm, " : "", - (msr & 1 << 24) ? "Amps, " : "", - (msr & 1 << 25) ? "GFXPwr, " : "", - (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); - } - if (do_ring_perf_limit_reasons) { - get_msr(cpu, MSR_RING_PERF_LIMIT_REASONS, &msr); - fprintf(outf, "cpu%d: MSR_RING_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr); - fprintf(outf, " (Active: %s%s%s%s%s%s)", - (msr & 1 << 0) ? "PROCHOT, " : "", - (msr & 1 << 1) ? "ThermStatus, " : "", - (msr & 1 << 6) ? "VR-Therm, " : "", - (msr & 1 << 8) ? "Amps, " : "", - (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); - fprintf(outf, " (Logged: %s%s%s%s%s%s)\n", - (msr & 1 << 16) ? "PROCHOT, " : "", - (msr & 1 << 17) ? "ThermStatus, " : "", - (msr & 1 << 22) ? "VR-Therm, " : "", - (msr & 1 << 24) ? "Amps, " : "", - (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); - } - return 0; -} + int *pfd_group = get_cstate_perf_group_fd(cci, cai->perf_subsys); -#define RAPL_POWER_GRANULARITY 0x7FFF /* 15 bit power granularity */ -#define RAPL_TIME_GRANULARITY 0x3F /* 6 bit time granularity */ + if (pfd_group == NULL) + goto end; -double get_tdp_intel(unsigned int model) -{ - unsigned long long msr; + const unsigned int type = read_perf_type(cai->perf_subsys); + const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name); - if (do_rapl & RAPL_PKG_POWER_INFO) - if (!get_msr(base_cpu, MSR_PKG_POWER_INFO, &msr)) - return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units; + ret = open_perf_counter(cpu, type, config, *pfd_group, PERF_FORMAT_GROUP); - switch (model) { - case INTEL_FAM6_ATOM_SILVERMONT: - case INTEL_FAM6_ATOM_SILVERMONT_D: - return 30.0; - default: - return 135.0; - } + if (ret == -1) + goto end; + + /* If it's the first counter opened, make it a group descriptor */ + if (*pfd_group == -1) + *pfd_group = ret; + +end: + if (debug >= 2) + fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu); + + return ret; } -double get_tdp_amd(unsigned int family) +int add_msr_perf_counter(int cpu, struct msr_counter_info_t *cci, const struct msr_counter_arch_info *cai) { - UNUSED(family); + int ret = -1; + + if (no_perf) + return -1; + + if (!cai->perf_name) + return -1; + + const unsigned int type = read_perf_type(cai->perf_subsys); + const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name); + + ret = open_perf_counter(cpu, type, config, cci->fd_perf, PERF_FORMAT_GROUP); - /* This is the max stock TDP of HEDT/Server Fam17h+ chips */ - return 280.0; + if (ret == -1) + goto end; + + /* If it's the first counter opened, make it a group descriptor */ + if (cci->fd_perf == -1) + cci->fd_perf = ret; + +end: + if (debug) + fprintf(stderr, "%s: %s/%s: %d (cpu: %d)\n", __func__, cai->perf_subsys, cai->perf_name, ret, cpu); + + return ret; } -/* - * rapl_dram_energy_units_probe() - * Energy units are either hard-coded, or come from RAPL Energy Unit MSR. - */ -static double rapl_dram_energy_units_probe(int model, double rapl_energy_units) +void msr_perf_init_(void) { - /* only called for genuine_intel, family 6 */ + const int mci_num = topo.max_cpu_num + 1; - switch (model) { - case INTEL_FAM6_HASWELL_X: /* HSX */ - case INTEL_FAM6_BROADWELL_X: /* BDX */ - case INTEL_FAM6_SKYLAKE_X: /* SKX */ - case INTEL_FAM6_XEON_PHI_KNL: /* KNL */ - case INTEL_FAM6_ICELAKE_X: /* ICX */ - return (rapl_dram_energy_units = 15.3 / 1000000); - default: - return (rapl_energy_units); + msr_counter_info = calloc(mci_num, sizeof(*msr_counter_info)); + if (!msr_counter_info) + err(1, "calloc msr_counter_info"); + msr_counter_info_size = mci_num; + + for (int cpu = 0; cpu < mci_num; ++cpu) + msr_counter_info[cpu].fd_perf = -1; + + for (int cidx = 0; cidx < NUM_MSR_COUNTERS; ++cidx) { + + struct msr_counter_arch_info *cai = &msr_counter_arch_infos[cidx]; + + cai->present = false; + + for (int cpu = 0; cpu < mci_num; ++cpu) { + + struct msr_counter_info_t *const cci = &msr_counter_info[cpu]; + + if (cpu_is_not_allowed(cpu)) + continue; + + if (cai->needed) { + /* Use perf API for this counter */ + if (add_msr_perf_counter(cpu, cci, cai) != -1) { + cci->source[cai->rci_index] = COUNTER_SOURCE_PERF; + cai->present = true; + + /* User MSR for this counter */ + } else if (add_msr_counter(cpu, cai->msr) >= 0) { + cci->source[cai->rci_index] = COUNTER_SOURCE_MSR; + cci->msr[cai->rci_index] = cai->msr; + cci->msr_mask[cai->rci_index] = cai->msr_mask; + cai->present = true; + } + } + } } } -void rapl_probe_intel(unsigned int family, unsigned int model) +/* Initialize data for reading perf counters from the MSR group. */ +void msr_perf_init(void) { - unsigned long long msr; - unsigned int time_unit; - double tdp; + bool need_amperf = false, need_smi = false; + const bool need_soft_c1 = (!platform->has_msr_core_c1_res) && (platform->supported_cstates & CC1); - if (family != 6) - return; + need_amperf = BIC_IS_ENABLED(BIC_Avg_MHz) || BIC_IS_ENABLED(BIC_Busy) || BIC_IS_ENABLED(BIC_Bzy_MHz) + || BIC_IS_ENABLED(BIC_IPC) || need_soft_c1; - switch (model) { - case INTEL_FAM6_SANDYBRIDGE: - case INTEL_FAM6_IVYBRIDGE: - case INTEL_FAM6_HASWELL: /* HSW */ - case INTEL_FAM6_HASWELL_L: /* HSW */ - case INTEL_FAM6_HASWELL_G: /* HSW */ - case INTEL_FAM6_BROADWELL: /* BDW */ - case INTEL_FAM6_BROADWELL_G: /* BDW */ - do_rapl = RAPL_PKG | RAPL_CORES | RAPL_CORE_POLICY | RAPL_GFX | RAPL_PKG_POWER_INFO; - if (rapl_joules) { - BIC_PRESENT(BIC_Pkg_J); - BIC_PRESENT(BIC_Cor_J); - BIC_PRESENT(BIC_GFX_J); - } else { - BIC_PRESENT(BIC_PkgWatt); - BIC_PRESENT(BIC_CorWatt); - BIC_PRESENT(BIC_GFXWatt); - } - break; - case INTEL_FAM6_ATOM_GOLDMONT: /* BXT */ - case INTEL_FAM6_ATOM_GOLDMONT_PLUS: - do_rapl = RAPL_PKG | RAPL_PKG_POWER_INFO; - if (rapl_joules) - BIC_PRESENT(BIC_Pkg_J); - else - BIC_PRESENT(BIC_PkgWatt); - break; - case INTEL_FAM6_ATOM_TREMONT: /* EHL */ - do_rapl = - RAPL_PKG | RAPL_CORES | RAPL_CORE_POLICY | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_PKG_PERF_STATUS - | RAPL_GFX | RAPL_PKG_POWER_INFO; - if (rapl_joules) { - BIC_PRESENT(BIC_Pkg_J); - BIC_PRESENT(BIC_Cor_J); - BIC_PRESENT(BIC_RAM_J); - BIC_PRESENT(BIC_GFX_J); - } else { - BIC_PRESENT(BIC_PkgWatt); - BIC_PRESENT(BIC_CorWatt); - BIC_PRESENT(BIC_RAMWatt); - BIC_PRESENT(BIC_GFXWatt); - } - break; - case INTEL_FAM6_ATOM_TREMONT_D: /* JVL */ - do_rapl = RAPL_PKG | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO; - BIC_PRESENT(BIC_PKG__); - if (rapl_joules) - BIC_PRESENT(BIC_Pkg_J); - else - BIC_PRESENT(BIC_PkgWatt); - break; - case INTEL_FAM6_SKYLAKE_L: /* SKL */ - case INTEL_FAM6_CANNONLAKE_L: /* CNL */ - do_rapl = - RAPL_PKG | RAPL_CORES | RAPL_CORE_POLICY | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_PKG_PERF_STATUS - | RAPL_GFX | RAPL_PKG_POWER_INFO; - BIC_PRESENT(BIC_PKG__); - BIC_PRESENT(BIC_RAM__); - if (rapl_joules) { - BIC_PRESENT(BIC_Pkg_J); - BIC_PRESENT(BIC_Cor_J); - BIC_PRESENT(BIC_RAM_J); - BIC_PRESENT(BIC_GFX_J); - } else { - BIC_PRESENT(BIC_PkgWatt); - BIC_PRESENT(BIC_CorWatt); - BIC_PRESENT(BIC_RAMWatt); - BIC_PRESENT(BIC_GFXWatt); - } - break; - case INTEL_FAM6_HASWELL_X: /* HSX */ - case INTEL_FAM6_BROADWELL_X: /* BDX */ - case INTEL_FAM6_SKYLAKE_X: /* SKX */ - case INTEL_FAM6_ICELAKE_X: /* ICX */ - case INTEL_FAM6_SAPPHIRERAPIDS_X: /* SPR */ - case INTEL_FAM6_XEON_PHI_KNL: /* KNL */ - do_rapl = - RAPL_PKG | RAPL_DRAM | RAPL_DRAM_POWER_INFO | RAPL_DRAM_PERF_STATUS | RAPL_PKG_PERF_STATUS | - RAPL_PKG_POWER_INFO; - BIC_PRESENT(BIC_PKG__); - BIC_PRESENT(BIC_RAM__); - if (rapl_joules) { - BIC_PRESENT(BIC_Pkg_J); - BIC_PRESENT(BIC_RAM_J); - } else { - BIC_PRESENT(BIC_PkgWatt); - BIC_PRESENT(BIC_RAMWatt); - } - break; - case INTEL_FAM6_SANDYBRIDGE_X: - case INTEL_FAM6_IVYBRIDGE_X: - do_rapl = - RAPL_PKG | RAPL_CORES | RAPL_CORE_POLICY | RAPL_DRAM | RAPL_DRAM_POWER_INFO | RAPL_PKG_PERF_STATUS | - RAPL_DRAM_PERF_STATUS | RAPL_PKG_POWER_INFO; - BIC_PRESENT(BIC_PKG__); - BIC_PRESENT(BIC_RAM__); - if (rapl_joules) { - BIC_PRESENT(BIC_Pkg_J); - BIC_PRESENT(BIC_Cor_J); - BIC_PRESENT(BIC_RAM_J); - } else { - BIC_PRESENT(BIC_PkgWatt); - BIC_PRESENT(BIC_CorWatt); - BIC_PRESENT(BIC_RAMWatt); - } - break; - case INTEL_FAM6_ATOM_SILVERMONT: /* BYT */ - case INTEL_FAM6_ATOM_SILVERMONT_D: /* AVN */ - do_rapl = RAPL_PKG | RAPL_CORES; - if (rapl_joules) { - BIC_PRESENT(BIC_Pkg_J); - BIC_PRESENT(BIC_Cor_J); - } else { - BIC_PRESENT(BIC_PkgWatt); - BIC_PRESENT(BIC_CorWatt); - } - break; - case INTEL_FAM6_ATOM_GOLDMONT_D: /* DNV */ - do_rapl = - RAPL_PKG | RAPL_DRAM | RAPL_DRAM_POWER_INFO | RAPL_DRAM_PERF_STATUS | RAPL_PKG_PERF_STATUS | - RAPL_PKG_POWER_INFO | RAPL_CORES_ENERGY_STATUS; - BIC_PRESENT(BIC_PKG__); - BIC_PRESENT(BIC_RAM__); - if (rapl_joules) { - BIC_PRESENT(BIC_Pkg_J); - BIC_PRESENT(BIC_Cor_J); - BIC_PRESENT(BIC_RAM_J); - } else { - BIC_PRESENT(BIC_PkgWatt); - BIC_PRESENT(BIC_CorWatt); - BIC_PRESENT(BIC_RAMWatt); - } - break; - default: - return; + if (BIC_IS_ENABLED(BIC_SMI)) + need_smi = true; + + /* Enable needed counters */ + msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].needed = need_amperf; + msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].needed = need_amperf; + msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].needed = need_smi; + + msr_perf_init_(); + + const bool has_amperf = has_amperf_access(); + const bool has_smi = msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].present; + + has_aperf_access = has_amperf; + + if (has_amperf) { + BIC_PRESENT(BIC_Avg_MHz); + BIC_PRESENT(BIC_Busy); + BIC_PRESENT(BIC_Bzy_MHz); + BIC_PRESENT(BIC_SMI); + } + + if (has_smi) + BIC_PRESENT(BIC_SMI); +} + +void cstate_perf_init_(bool soft_c1) +{ + bool has_counter; + bool *cores_visited = NULL, *pkg_visited = NULL; + const int cores_visited_elems = topo.max_core_id + 1; + const int pkg_visited_elems = topo.max_package_id + 1; + const int cci_num = topo.max_cpu_num + 1; + + ccstate_counter_info = calloc(cci_num, sizeof(*ccstate_counter_info)); + if (!ccstate_counter_info) + err(1, "calloc ccstate_counter_arch_info"); + ccstate_counter_info_size = cci_num; + + cores_visited = calloc(cores_visited_elems, sizeof(*cores_visited)); + if (!cores_visited) + err(1, "calloc cores_visited"); + + pkg_visited = calloc(pkg_visited_elems, sizeof(*pkg_visited)); + if (!pkg_visited) + err(1, "calloc pkg_visited"); + + /* Initialize cstate_counter_info_percpu */ + for (int cpu = 0; cpu < cci_num; ++cpu) { + ccstate_counter_info[cpu].fd_perf_core = -1; + ccstate_counter_info[cpu].fd_perf_pkg = -1; } - /* units on package 0, verify later other packages match */ - if (get_msr(base_cpu, MSR_RAPL_POWER_UNIT, &msr)) - return; + for (int cidx = 0; cidx < NUM_CSTATE_COUNTERS; ++cidx) { + has_counter = false; + memset(cores_visited, 0, cores_visited_elems * sizeof(*cores_visited)); + memset(pkg_visited, 0, pkg_visited_elems * sizeof(*pkg_visited)); - rapl_power_units = 1.0 / (1 << (msr & 0xF)); - if (model == INTEL_FAM6_ATOM_SILVERMONT) - rapl_energy_units = 1.0 * (1 << (msr >> 8 & 0x1F)) / 1000000; - else - rapl_energy_units = 1.0 / (1 << (msr >> 8 & 0x1F)); + const struct cstate_counter_arch_info *cai = &ccstate_counter_arch_infos[cidx]; - rapl_dram_energy_units = rapl_dram_energy_units_probe(model, rapl_energy_units); + for (int cpu = 0; cpu < cci_num; ++cpu) { - time_unit = msr >> 16 & 0xF; - if (time_unit == 0) - time_unit = 0xA; + struct cstate_counter_info_t *const cci = &ccstate_counter_info[cpu]; - rapl_time_units = 1.0 / (1 << (time_unit)); + if (cpu_is_not_allowed(cpu)) + continue; - tdp = get_tdp_intel(model); + const int core_id = cpus[cpu].physical_core_id; + const int pkg_id = cpus[cpu].physical_package_id; - rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp; - if (!quiet) - fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp); -} + assert(core_id < cores_visited_elems); + assert(pkg_id < pkg_visited_elems); -void rapl_probe_amd(unsigned int family, unsigned int model) -{ - unsigned long long msr; - unsigned int eax, ebx, ecx, edx; - unsigned int has_rapl = 0; - double tdp; + const bool per_thread = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD; + const bool per_core = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_CORE; - UNUSED(model); + if (!per_thread && cores_visited[core_id]) + continue; - if (max_extended_level >= 0x80000007) { - __cpuid(0x80000007, eax, ebx, ecx, edx); - /* RAPL (Fam 17h+) */ - has_rapl = edx & (1 << 14); - } + if (!per_core && pkg_visited[pkg_id]) + continue; - if (!has_rapl || family < 0x17) - return; + const bool counter_needed = BIC_IS_ENABLED(cai->bic_number) || + (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY)); + const bool counter_supported = (platform->supported_cstates & cai->feature_mask); - do_rapl = RAPL_AMD_F17H | RAPL_PER_CORE_ENERGY; - if (rapl_joules) { - BIC_PRESENT(BIC_Pkg_J); - BIC_PRESENT(BIC_Cor_J); - } else { - BIC_PRESENT(BIC_PkgWatt); - BIC_PRESENT(BIC_CorWatt); - } + if (counter_needed && counter_supported) { + /* Use perf API for this counter */ + if (add_cstate_perf_counter(cpu, cci, cai) != -1) { - if (get_msr(base_cpu, MSR_RAPL_PWR_UNIT, &msr)) - return; + cci->source[cai->rci_index] = COUNTER_SOURCE_PERF; - rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf)); - rapl_energy_units = ldexp(1.0, -(msr >> 8 & 0x1f)); - rapl_power_units = ldexp(1.0, -(msr & 0xf)); + /* User MSR for this counter */ + } else if (pkg_cstate_limit >= cai->pkg_cstate_limit && add_msr_counter(cpu, cai->msr) >= 0) { + cci->source[cai->rci_index] = COUNTER_SOURCE_MSR; + cci->msr[cai->rci_index] = cai->msr; + } + } - tdp = get_tdp_amd(family); + if (cci->source[cai->rci_index] != COUNTER_SOURCE_NONE) { + has_counter = true; + cores_visited[core_id] = true; + pkg_visited[pkg_id] = true; + } + } - rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp; - if (!quiet) - fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp); + /* If any CPU has access to the counter, make it present */ + if (has_counter) + BIC_PRESENT(cai->bic_number); + } + + free(cores_visited); + free(pkg_visited); } -/* - * rapl_probe() - * - * sets do_rapl, rapl_power_units, rapl_energy_units, rapl_time_units - */ -void rapl_probe(unsigned int family, unsigned int model) +void cstate_perf_init(void) { - if (genuine_intel) - rapl_probe_intel(family, model); - if (authentic_amd || hygon_genuine) - rapl_probe_amd(family, model); + /* + * If we don't have a C1 residency MSR, we calculate it "in software", + * but we need APERF, MPERF too. + */ + const bool soft_c1 = !platform->has_msr_core_c1_res && has_amperf_access() + && platform->supported_cstates & CC1; + + if (soft_c1) + BIC_PRESENT(BIC_CPU_c1); + + cstate_perf_init_(soft_c1); } -void perf_limit_reasons_probe(unsigned int family, unsigned int model) +void probe_cstates(void) { - if (!genuine_intel) - return; + probe_cst_limit(); - if (family != 6) - return; + if (platform->has_msr_module_c6_res_ms) + BIC_PRESENT(BIC_Mod_c6); - switch (model) { - case INTEL_FAM6_HASWELL: /* HSW */ - case INTEL_FAM6_HASWELL_L: /* HSW */ - case INTEL_FAM6_HASWELL_G: /* HSW */ - do_gfx_perf_limit_reasons = 1; - /* FALLTHRU */ - case INTEL_FAM6_HASWELL_X: /* HSX */ - do_core_perf_limit_reasons = 1; - do_ring_perf_limit_reasons = 1; - default: - return; + if (platform->has_ext_cst_msrs && !no_msr) { + BIC_PRESENT(BIC_Totl_c0); + BIC_PRESENT(BIC_Any_c0); + BIC_PRESENT(BIC_GFX_c0); + BIC_PRESENT(BIC_CPUGFX); } + + if (quiet) + return; + + dump_power_ctl(); + dump_cst_cfg(); + decode_c6_demotion_policy_msr(); + print_dev_latency(); + dump_sysfs_cstate_config(); + print_irtl(); } -void automatic_cstate_conversion_probe(unsigned int family, unsigned int model) +void probe_lpi(void) { - if (family != 6) - return; + if (!access("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", R_OK)) + BIC_PRESENT(BIC_CPU_LPI); + else + BIC_NOT_PRESENT(BIC_CPU_LPI); - switch (model) { - case INTEL_FAM6_BROADWELL_X: - case INTEL_FAM6_SKYLAKE_X: - has_automatic_cstate_conversion = 1; + if (!access(sys_lpi_file_sysfs, R_OK)) { + sys_lpi_file = sys_lpi_file_sysfs; + BIC_PRESENT(BIC_SYS_LPI); + } else if (!access(sys_lpi_file_debugfs, R_OK)) { + sys_lpi_file = sys_lpi_file_debugfs; + BIC_PRESENT(BIC_SYS_LPI); + } else { + sys_lpi_file_sysfs = NULL; + BIC_NOT_PRESENT(BIC_SYS_LPI); } + } -void prewake_cstate_probe(unsigned int family, unsigned int model) +void probe_pstates(void) { - if (is_icx(family, model) || is_spr(family, model)) - dis_cstate_prewake = 1; + probe_bclk(); + + if (quiet) + return; + + dump_platform_info(); + dump_turbo_ratio_info(); + dump_sysfs_pstate_config(); + decode_misc_pwr_mgmt_msr(); + + for_all_cpus(print_hwp, ODD_COUNTERS); + for_all_cpus(print_epb, ODD_COUNTERS); + for_all_cpus(print_perf_limit, ODD_COUNTERS); } -int print_thermal(struct thread_data *t, struct core_data *c, struct pkg_data *p) +void process_cpuid() { - unsigned long long msr; - unsigned int dts, dts2; - int cpu; + unsigned int eax, ebx, ecx, edx; + unsigned int fms, family, model, stepping, ecx_flags, edx_flags; + unsigned long long ucode_patch = 0; + bool ucode_patch_valid = false; - UNUSED(c); - UNUSED(p); + eax = ebx = ecx = edx = 0; - if (!(do_dts || do_ptm)) - return 0; + __cpuid(0, max_level, ebx, ecx, edx); - cpu = t->cpu_id; + if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69) + genuine_intel = 1; + else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65) + authentic_amd = 1; + else if (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e) + hygon_genuine = 1; - /* DTS is per-core, no need to print for each thread */ - if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE)) - return 0; + if (!quiet) + fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n", (char *)&ebx, (char *)&edx, (char *)&ecx, max_level); - if (cpu_migrate(cpu)) { - fprintf(outf, "print_thermal: Could not migrate to CPU %d\n", cpu); - return -1; + __cpuid(1, fms, ebx, ecx, edx); + family = (fms >> 8) & 0xf; + model = (fms >> 4) & 0xf; + stepping = fms & 0xf; + if (family == 0xf) + family += (fms >> 20) & 0xff; + if (family >= 6) + model += ((fms >> 16) & 0xf) << 4; + ecx_flags = ecx; + edx_flags = edx; + + if (!no_msr) { + if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) { + warnx("get_msr(UCODE)"); + } else { + ucode_patch_valid = true; + if (!authentic_amd && !hygon_genuine) + ucode_patch >>= 32; + } } - if (do_ptm && (t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) { - if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr)) - return 0; + /* + * check max extended function levels of CPUID. + * This is needed to check for invariant TSC. + * This check is valid for both Intel and AMD. + */ + ebx = ecx = edx = 0; + __cpuid(0x80000000, max_extended_level, ebx, ecx, edx); - dts = (msr >> 16) & 0x7F; - fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_STATUS: 0x%08llx (%d C)\n", cpu, msr, tj_max - dts); + if (!quiet) { + fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", family, model, stepping, family, model, stepping); + if (ucode_patch_valid) + fprintf(outf, " microcode 0x%x", (unsigned int)ucode_patch); + fputc('\n', outf); - if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &msr)) - return 0; + fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level); + fprintf(outf, "CPUID(1): %s %s %s %s %s %s %s %s %s %s\n", + ecx_flags & (1 << 0) ? "SSE3" : "-", + ecx_flags & (1 << 3) ? "MONITOR" : "-", + ecx_flags & (1 << 6) ? "SMX" : "-", + ecx_flags & (1 << 7) ? "EIST" : "-", + ecx_flags & (1 << 8) ? "TM2" : "-", + edx_flags & (1 << 4) ? "TSC" : "-", + edx_flags & (1 << 5) ? "MSR" : "-", + edx_flags & (1 << 22) ? "ACPI-TM" : "-", edx_flags & (1 << 28) ? "HT" : "-", edx_flags & (1 << 29) ? "TM" : "-"); + } - dts = (msr >> 16) & 0x7F; - dts2 = (msr >> 8) & 0x7F; - fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", - cpu, msr, tj_max - dts, tj_max - dts2); + probe_platform_features(family, model); + + if (!(edx_flags & (1 << 5))) + errx(1, "CPUID: no MSR"); + + if (max_extended_level >= 0x80000007) { + + /* + * Non-Stop TSC is advertised by CPUID.EAX=0x80000007: EDX.bit8 + * this check is valid for both Intel and AMD + */ + __cpuid(0x80000007, eax, ebx, ecx, edx); + has_invariant_tsc = edx & (1 << 8); } - if (do_dts && debug) { - unsigned int resolution; + /* + * APERF/MPERF is advertised by CPUID.EAX=0x6: ECX.bit0 + * this check is valid for both Intel and AMD + */ - if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr)) - return 0; + __cpuid(0x6, eax, ebx, ecx, edx); + cpuid_has_aperf_mperf = ecx & (1 << 0); + do_dts = eax & (1 << 0); + if (do_dts) + BIC_PRESENT(BIC_CoreTmp); + has_turbo = eax & (1 << 1); + do_ptm = eax & (1 << 6); + if (do_ptm) + BIC_PRESENT(BIC_PkgTmp); + has_hwp = eax & (1 << 7); + has_hwp_notify = eax & (1 << 8); + has_hwp_activity_window = eax & (1 << 9); + has_hwp_epp = eax & (1 << 10); + has_hwp_pkg = eax & (1 << 11); + has_epb = ecx & (1 << 3); - dts = (msr >> 16) & 0x7F; - resolution = (msr >> 27) & 0xF; - fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", - cpu, msr, tj_max - dts, resolution); + if (!quiet) + fprintf(outf, "CPUID(6): %sAPERF, %sTURBO, %sDTS, %sPTM, %sHWP, " + "%sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n", + cpuid_has_aperf_mperf ? "" : "No-", + has_turbo ? "" : "No-", + do_dts ? "" : "No-", + do_ptm ? "" : "No-", + has_hwp ? "" : "No-", + has_hwp_notify ? "" : "No-", + has_hwp_activity_window ? "" : "No-", has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-"); - if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr)) - return 0; + if (!quiet) + decode_misc_enable_msr(); - dts = (msr >> 16) & 0x7F; - dts2 = (msr >> 8) & 0x7F; - fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", - cpu, msr, tj_max - dts, tj_max - dts2); + if (max_level >= 0x7 && !quiet) { + int has_sgx; + + ecx = 0; + + __cpuid_count(0x7, 0, eax, ebx, ecx, edx); + + has_sgx = ebx & (1 << 2); + + is_hybrid = edx & (1 << 15); + + fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-"); + + if (has_sgx) + decode_feature_control_msr(); } - return 0; -} + if (max_level >= 0x15) { + unsigned int eax_crystal; + unsigned int ebx_tsc; -void print_power_limit_msr(int cpu, unsigned long long msr, char *label) -{ - fprintf(outf, "cpu%d: %s: %sabled (%0.3f Watts, %f sec, clamp %sabled)\n", - cpu, label, - ((msr >> 15) & 1) ? "EN" : "DIS", - ((msr >> 0) & 0x7FFF) * rapl_power_units, - (1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units, - (((msr >> 16) & 1) ? "EN" : "DIS")); + /* + * CPUID 15H TSC/Crystal ratio, possibly Crystal Hz + */ + eax_crystal = ebx_tsc = crystal_hz = edx = 0; + __cpuid(0x15, eax_crystal, ebx_tsc, crystal_hz, edx); + + if (ebx_tsc != 0) { + if (!quiet && (ebx != 0)) + fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n", eax_crystal, ebx_tsc, crystal_hz); + + if (crystal_hz == 0) + crystal_hz = platform->crystal_freq; + + if (crystal_hz) { + tsc_hz = (unsigned long long)crystal_hz *ebx_tsc / eax_crystal; + if (!quiet) + fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n", + tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal); + } + } + } + if (max_level >= 0x16) { + unsigned int base_mhz, max_mhz, bus_mhz, edx; - return; -} + /* + * CPUID 16H Base MHz, Max MHz, Bus MHz + */ + base_mhz = max_mhz = bus_mhz = edx = 0; -int print_rapl(struct thread_data *t, struct core_data *c, struct pkg_data *p) -{ - unsigned long long msr; - const char *msr_name; - int cpu; + __cpuid(0x16, base_mhz, max_mhz, bus_mhz, edx); - UNUSED(c); - UNUSED(p); + bclk = bus_mhz; - if (!do_rapl) - return 0; + base_hz = base_mhz * 1000000; + has_base_hz = 1; - /* RAPL counters are per package, so print only for 1st thread/package */ - if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE) || !(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) - return 0; + if (platform->enable_tsc_tweak) + tsc_tweak = base_hz / tsc_hz; - cpu = t->cpu_id; - if (cpu_migrate(cpu)) { - fprintf(outf, "print_rapl: Could not migrate to CPU %d\n", cpu); - return -1; + if (!quiet) + fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n", base_mhz, max_mhz, bus_mhz); } - if (do_rapl & RAPL_AMD_F17H) { - msr_name = "MSR_RAPL_PWR_UNIT"; - if (get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr)) - return -1; - } else { - msr_name = "MSR_RAPL_POWER_UNIT"; - if (get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr)) - return -1; - } + if (cpuid_has_aperf_mperf) + aperf_mperf_multiplier = platform->need_perf_multiplier ? 1024 : 1; - fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, - rapl_power_units, rapl_energy_units, rapl_time_units); + BIC_PRESENT(BIC_IRQ); + BIC_PRESENT(BIC_NMI); + BIC_PRESENT(BIC_TSC_MHz); +} - if (do_rapl & RAPL_PKG_POWER_INFO) { +static void counter_info_init(void) +{ + for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i) { + struct cstate_counter_arch_info *const cai = &ccstate_counter_arch_infos[i]; - if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr)) - return -5; + if (platform->has_msr_knl_core_c6_residency && cai->msr == MSR_CORE_C6_RESIDENCY) + cai->msr = MSR_KNL_CORE_C6_RESIDENCY; - fprintf(outf, "cpu%d: MSR_PKG_POWER_INFO: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n", - cpu, msr, - ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); + if (!platform->has_msr_core_c1_res && cai->msr == MSR_CORE_C1_RES) + cai->msr = 0; + if (platform->has_msr_atom_pkg_c6_residency && cai->msr == MSR_PKG_C6_RESIDENCY) + cai->msr = MSR_ATOM_PKG_C6_RESIDENCY; } - if (do_rapl & RAPL_PKG) { - - if (get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr)) - return -9; - fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 63) & 1 ? "" : "UN"); + for (int i = 0; i < NUM_MSR_COUNTERS; ++i) { + msr_counter_arch_infos[i].present = false; + msr_counter_arch_infos[i].needed = false; + } +} - print_power_limit_msr(cpu, msr, "PKG Limit #1"); - fprintf(outf, "cpu%d: PKG Limit #2: %sabled (%0.3f Watts, %f* sec, clamp %sabled)\n", - cpu, - ((msr >> 47) & 1) ? "EN" : "DIS", - ((msr >> 32) & 0x7FFF) * rapl_power_units, - (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, - ((msr >> 48) & 1) ? "EN" : "DIS"); +void probe_pm_features(void) +{ + probe_pstates(); - if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr)) - return -9; + probe_cstates(); - fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr); - fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n", - cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN"); - } + probe_lpi(); - if (do_rapl & RAPL_DRAM_POWER_INFO) { - if (get_msr(cpu, MSR_DRAM_POWER_INFO, &msr)) - return -6; + probe_intel_uncore_frequency(); - fprintf(outf, "cpu%d: MSR_DRAM_POWER_INFO,: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n", - cpu, msr, - ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); - } - if (do_rapl & RAPL_DRAM) { - if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr)) - return -9; - fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 31) & 1 ? "" : "UN"); + probe_graphics(); - print_power_limit_msr(cpu, msr, "DRAM Limit"); - } - if (do_rapl & RAPL_CORE_POLICY) { - if (get_msr(cpu, MSR_PP0_POLICY, &msr)) - return -7; + probe_rapl(); - fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF); - } - if (do_rapl & RAPL_CORES_POWER_LIMIT) { - if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr)) - return -9; - fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 31) & 1 ? "" : "UN"); - print_power_limit_msr(cpu, msr, "Cores Limit"); - } - if (do_rapl & RAPL_GFX) { - if (get_msr(cpu, MSR_PP1_POLICY, &msr)) - return -8; + probe_thermal(); - fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF); + if (platform->has_nhm_msrs && !no_msr) + BIC_PRESENT(BIC_SMI); - if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr)) - return -9; - fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 31) & 1 ? "" : "UN"); - print_power_limit_msr(cpu, msr, "GFX Limit"); - } - return 0; + if (!quiet) + decode_misc_feature_control(); } -/* - * SNB adds support for additional MSRs: +/* perf_llc_probe * - * MSR_PKG_C7_RESIDENCY 0x000003fa - * MSR_CORE_C7_RESIDENCY 0x000003fe - * MSR_PKG_C2_RESIDENCY 0x0000060d + * return 1 on success, else 0 */ - -int has_snb_msrs(unsigned int family, unsigned int model) +int has_perf_llc_access(void) { - if (!genuine_intel) - return 0; + int fd; - if (family != 6) + if (no_perf) return 0; - switch (model) { - case INTEL_FAM6_SANDYBRIDGE: - case INTEL_FAM6_SANDYBRIDGE_X: - case INTEL_FAM6_IVYBRIDGE: /* IVB */ - case INTEL_FAM6_IVYBRIDGE_X: /* IVB Xeon */ - case INTEL_FAM6_HASWELL: /* HSW */ - case INTEL_FAM6_HASWELL_X: /* HSW */ - case INTEL_FAM6_HASWELL_L: /* HSW */ - case INTEL_FAM6_HASWELL_G: /* HSW */ - case INTEL_FAM6_BROADWELL: /* BDW */ - case INTEL_FAM6_BROADWELL_G: /* BDW */ - case INTEL_FAM6_BROADWELL_X: /* BDX */ - case INTEL_FAM6_SKYLAKE_L: /* SKL */ - case INTEL_FAM6_CANNONLAKE_L: /* CNL */ - case INTEL_FAM6_SKYLAKE_X: /* SKX */ - case INTEL_FAM6_ICELAKE_X: /* ICX */ - case INTEL_FAM6_SAPPHIRERAPIDS_X: /* SPR */ - case INTEL_FAM6_ATOM_GOLDMONT: /* BXT */ - case INTEL_FAM6_ATOM_GOLDMONT_PLUS: - case INTEL_FAM6_ATOM_GOLDMONT_D: /* DNV */ - case INTEL_FAM6_ATOM_TREMONT: /* EHL */ - case INTEL_FAM6_ATOM_TREMONT_D: /* JVL */ - return 1; - } - return 0; + fd = open_perf_counter(base_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP); + if (fd != -1) + close(fd); + + if (fd == -1) + warnx("Failed to access %s. Some of the counters may not be available\n" + "\tRun as root to enable them or use %s to disable the access explicitly", "perf LLC counters", "'--hide LLC' or '--no-perf'"); + + return (fd != -1); } -/* - * HSW ULT added support for C8/C9/C10 MSRs: - * - * MSR_PKG_C8_RESIDENCY 0x00000630 - * MSR_PKG_C9_RESIDENCY 0x00000631 - * MSR_PKG_C10_RESIDENCY 0x00000632 - * - * MSR_PKGC8_IRTL 0x00000633 - * MSR_PKGC9_IRTL 0x00000634 - * MSR_PKGC10_IRTL 0x00000635 - * - */ -int has_c8910_msrs(unsigned int family, unsigned int model) +void perf_llc_init(void) { - if (!genuine_intel) - return 0; + int cpu; + int retval; - if (family != 6) - return 0; + if (no_perf) + return; + if (!(BIC_IS_ENABLED(BIC_LLC_RPS) && BIC_IS_ENABLED(BIC_LLC_HIT))) + return; - switch (model) { - case INTEL_FAM6_HASWELL_L: /* HSW */ - case INTEL_FAM6_BROADWELL: /* BDW */ - case INTEL_FAM6_SKYLAKE_L: /* SKL */ - case INTEL_FAM6_CANNONLAKE_L: /* CNL */ - case INTEL_FAM6_ATOM_GOLDMONT: /* BXT */ - case INTEL_FAM6_ATOM_GOLDMONT_PLUS: - case INTEL_FAM6_ATOM_TREMONT: /* EHL */ - return 1; + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { + + if (cpu_is_not_allowed(cpu)) + continue; + + assert(fd_llc_percpu != 0); + fd_llc_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP); + if (fd_llc_percpu[cpu] == -1) { + warnx("%s: perf REFS: failed to open counter on cpu%d", __func__, cpu); + free_fd_llc_percpu(); + return; + } + assert(fd_llc_percpu != 0); + retval = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, fd_llc_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + warnx("%s: perf MISS: failed to open counter on cpu%d", __func__, cpu); + free_fd_llc_percpu(); + return; + } } - return 0; + BIC_PRESENT(BIC_LLC_RPS); + BIC_PRESENT(BIC_LLC_HIT); } /* - * SKL adds support for additional MSRS: - * - * MSR_PKG_WEIGHTED_CORE_C0_RES 0x00000658 - * MSR_PKG_ANY_CORE_C0_RES 0x00000659 - * MSR_PKG_ANY_GFXE_C0_RES 0x0000065A - * MSR_PKG_BOTH_CORE_GFXE_C0_RES 0x0000065B + * in /dev/cpu/ return success for names that are numbers + * ie. filter out ".", "..", "microcode". */ -int has_skl_msrs(unsigned int family, unsigned int model) +int dir_filter(const struct dirent *dirp) { - if (!genuine_intel) - return 0; - - if (family != 6) - return 0; - - switch (model) { - case INTEL_FAM6_SKYLAKE_L: /* SKL */ - case INTEL_FAM6_CANNONLAKE_L: /* CNL */ + if (isdigit(dirp->d_name[0])) return 1; - } - return 0; + else + return 0; } -int is_slm(unsigned int family, unsigned int model) -{ - if (!genuine_intel) - return 0; +char *possible_file = "/sys/devices/system/cpu/possible"; +char possible_buf[1024]; - if (family != 6) - return 0; +int initialize_cpu_possible_set(void) +{ + FILE *fp; - switch (model) { - case INTEL_FAM6_ATOM_SILVERMONT: /* BYT */ - case INTEL_FAM6_ATOM_SILVERMONT_D: /* AVN */ - return 1; + fp = fopen(possible_file, "r"); + if (!fp) { + warn("open %s", possible_file); + return -1; + } + if (fread(possible_buf, sizeof(char), 1024, fp) == 0) { + warn("read %s", possible_file); + goto err; + } + if (parse_cpu_str(possible_buf, cpu_possible_set, cpu_possible_setsize)) { + warnx("%s: cpu str malformat %s\n", possible_file, cpu_effective_str); + goto err; } return 0; + +err: + fclose(fp); + return -1; } -int is_knl(unsigned int family, unsigned int model) +void topology_probe(bool startup) { - if (!genuine_intel) - return 0; + int i; + int max_core_id = 0; + int max_package_id = 0; + int max_siblings = 0; - if (family != 6) - return 0; + /* Initialize num_cpus, max_cpu_num */ + set_max_cpu_num(); + topo.num_cpus = 0; + for_all_proc_cpus(count_cpus); + if (!summary_only) + BIC_PRESENT(BIC_CPU); - switch (model) { - case INTEL_FAM6_XEON_PHI_KNL: /* KNL */ - return 1; - } - return 0; -} + if (debug > 1) + fprintf(outf, "num_cpus %d max_cpu_num %d\n", topo.num_cpus, topo.max_cpu_num); -int is_cnl(unsigned int family, unsigned int model) -{ - if (!genuine_intel) - return 0; + cpus = calloc(1, (topo.max_cpu_num + 1) * sizeof(struct cpu_topology)); + if (cpus == NULL) + err(1, "calloc cpus"); - if (family != 6) - return 0; + /* + * Allocate and initialize cpu_present_set + */ + cpu_present_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (cpu_present_set == NULL) + err(3, "CPU_ALLOC"); + cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); + CPU_ZERO_S(cpu_present_setsize, cpu_present_set); + for_all_proc_cpus(mark_cpu_present); - switch (model) { - case INTEL_FAM6_CANNONLAKE_L: /* CNL */ - return 1; - } + /* + * Allocate and initialize cpu_possible_set + */ + cpu_possible_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (cpu_possible_set == NULL) + err(3, "CPU_ALLOC"); + cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); + CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set); + initialize_cpu_possible_set(); - return 0; -} + /* + * Allocate and initialize cpu_effective_set + */ + cpu_effective_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (cpu_effective_set == NULL) + err(3, "CPU_ALLOC"); + cpu_effective_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); + CPU_ZERO_S(cpu_effective_setsize, cpu_effective_set); + update_effective_set(startup); -unsigned int get_aperf_mperf_multiplier(unsigned int family, unsigned int model) -{ - if (is_knl(family, model)) - return 1024; - return 1; -} + /* + * Allocate and initialize cpu_allowed_set + */ + cpu_allowed_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (cpu_allowed_set == NULL) + err(3, "CPU_ALLOC"); + cpu_allowed_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); + CPU_ZERO_S(cpu_allowed_setsize, cpu_allowed_set); -#define SLM_BCLK_FREQS 5 -double slm_freq_table[SLM_BCLK_FREQS] = { 83.3, 100.0, 133.3, 116.7, 80.0 }; + /* + * Validate and update cpu_allowed_set. + * + * Make sure all cpus in cpu_subset are also in cpu_present_set during startup. + * Give a warning when cpus in cpu_subset become unavailable at runtime. + * Give a warning when cpus are not effective because of cgroup setting. + * + * cpu_allowed_set is the intersection of cpu_present_set/cpu_effective_set/cpu_subset. + */ + for (i = 0; i < CPU_SUBSET_MAXCPUS; ++i) { + if (cpu_subset && !CPU_ISSET_S(i, cpu_subset_size, cpu_subset)) + continue; -double slm_bclk(void) -{ - unsigned long long msr = 3; - unsigned int i; - double freq; + if (!CPU_ISSET_S(i, cpu_present_setsize, cpu_present_set)) { + if (cpu_subset) { + /* cpus in cpu_subset must be in cpu_present_set during startup */ + if (startup) + err(1, "cpu%d not present", i); + else + fprintf(stderr, "cpu%d not present\n", i); + } + continue; + } - if (get_msr(base_cpu, MSR_FSB_FREQ, &msr)) - fprintf(outf, "SLM BCLK: unknown\n"); + if (CPU_COUNT_S(cpu_effective_setsize, cpu_effective_set)) { + if (!CPU_ISSET_S(i, cpu_effective_setsize, cpu_effective_set)) { + fprintf(stderr, "cpu%d not effective\n", i); + continue; + } + } - i = msr & 0xf; - if (i >= SLM_BCLK_FREQS) { - fprintf(outf, "SLM BCLK[%d] invalid\n", i); - i = 3; + CPU_SET_S(i, cpu_allowed_setsize, cpu_allowed_set); } - freq = slm_freq_table[i]; - - if (!quiet) - fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq); - return freq; -} - -double discover_bclk(unsigned int family, unsigned int model) -{ - if (has_snb_msrs(family, model) || is_knl(family, model)) - return 100.00; - else if (is_slm(family, model)) - return slm_bclk(); - else - return 133.33; -} + if (!CPU_COUNT_S(cpu_allowed_setsize, cpu_allowed_set)) + err(-ENODEV, "No valid cpus found"); + sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set); -int get_cpu_type(struct thread_data *t, struct core_data *c, struct pkg_data *p) -{ - unsigned int eax, ebx, ecx, edx; + /* + * Allocate and initialize cpu_affinity_set + */ + cpu_affinity_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (cpu_affinity_set == NULL) + err(3, "CPU_ALLOC"); + cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); + CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set); - UNUSED(c); - UNUSED(p); + for_all_proc_cpus(init_thread_id); - if (!genuine_intel) - return 0; + for_all_proc_cpus(set_cpu_hybrid_type); - if (cpu_migrate(t->cpu_id)) { - fprintf(outf, "Could not migrate to CPU %d\n", t->cpu_id); - return -1; - } + /* + * For online cpus + * find max_core_id, max_package_id + */ + for (i = 0; i <= topo.max_cpu_num; ++i) { + int siblings; - if (max_level < 0x1a) - return 0; + if (cpu_is_not_present(i)) { + if (debug > 1) + fprintf(outf, "cpu%d NOT PRESENT\n", i); + continue; + } - __cpuid(0x1a, eax, ebx, ecx, edx); - eax = (eax >> 24) & 0xFF; - if (eax == 0x20) - t->is_atom = true; - return 0; -} + cpus[i].logical_cpu_id = i; -/* - * MSR_IA32_TEMPERATURE_TARGET indicates the temperature where - * the Thermal Control Circuit (TCC) activates. - * This is usually equal to tjMax. - * - * Older processors do not have this MSR, so there we guess, - * but also allow cmdline over-ride with -T. - * - * Several MSR temperature values are in units of degrees-C - * below this value, including the Digital Thermal Sensor (DTS), - * Package Thermal Management Sensor (PTM), and thermal event thresholds. - */ -int set_temperature_target(struct thread_data *t, struct core_data *c, struct pkg_data *p) -{ - unsigned long long msr; - unsigned int tcc_default, tcc_offset; - int cpu; + /* get package information */ + cpus[i].physical_package_id = get_physical_package_id(i); + if (cpus[i].physical_package_id > max_package_id) + max_package_id = cpus[i].physical_package_id; - UNUSED(c); - UNUSED(p); + /* get die information */ + cpus[i].die_id = get_die_id(i); + if (cpus[i].die_id > topo.max_die_id) + topo.max_die_id = cpus[i].die_id; - /* tj_max is used only for dts or ptm */ - if (!(do_dts || do_ptm)) - return 0; + /* get l3 information */ + cpus[i].l3_id = get_l3_id(i); + if (cpus[i].l3_id > topo.max_l3_id) + topo.max_l3_id = cpus[i].l3_id; - /* this is a per-package concept */ - if (!(t->flags & CPU_IS_FIRST_THREAD_IN_CORE) || !(t->flags & CPU_IS_FIRST_CORE_IN_PACKAGE)) - return 0; + /* get numa node information */ + cpus[i].physical_node_id = get_physical_node_id(&cpus[i]); + if (cpus[i].physical_node_id > topo.max_node_num) + topo.max_node_num = cpus[i].physical_node_id; - cpu = t->cpu_id; - if (cpu_migrate(cpu)) { - fprintf(outf, "Could not migrate to CPU %d\n", cpu); - return -1; - } + /* get core information */ + cpus[i].physical_core_id = get_core_id(i); + if (cpus[i].physical_core_id > max_core_id) + max_core_id = cpus[i].physical_core_id; - if (tj_max_override != 0) { - tj_max = tj_max_override; - fprintf(outf, "cpu%d: Using cmdline TCC Target (%d C)\n", cpu, tj_max); - return 0; + /* get thread information */ + siblings = get_thread_siblings(&cpus[i]); + if (siblings > max_siblings) + max_siblings = siblings; + if (cpus[i].thread_id == 0) + topo.num_cores++; } + topo.max_core_id = max_core_id; + topo.max_package_id = max_package_id; - /* Temperature Target MSR is Nehalem and newer only */ - if (!do_nhm_platform_info) - goto guess; + topo.cores_per_node = max_core_id + 1; + if (debug > 1) + fprintf(outf, "max_core_id %d, sizing for %d cores per package\n", max_core_id, topo.cores_per_node); + if (!summary_only) + BIC_PRESENT(BIC_Core); - if (get_msr(base_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr)) - goto guess; + topo.num_die = topo.max_die_id + 1; + if (debug > 1) + fprintf(outf, "max_die_id %d, sizing for %d die\n", topo.max_die_id, topo.num_die); + if (!summary_only && topo.num_die > 1) + BIC_PRESENT(BIC_Die); - tcc_default = (msr >> 16) & 0xFF; + if (!summary_only && topo.max_l3_id > 0) + BIC_PRESENT(BIC_L3); - if (!quiet) { - switch (tcc_offset_bits) { - case 4: - tcc_offset = (msr >> 24) & 0xF; - fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C) (%d default - %d offset)\n", - cpu, msr, tcc_default - tcc_offset, tcc_default, tcc_offset); - break; - case 6: - tcc_offset = (msr >> 24) & 0x3F; - fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C) (%d default - %d offset)\n", - cpu, msr, tcc_default - tcc_offset, tcc_default, tcc_offset); - break; - default: - fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C)\n", cpu, msr, tcc_default); - break; - } - } + topo.num_packages = max_package_id + 1; + if (debug > 1) + fprintf(outf, "max_package_id %d, sizing for %d packages\n", max_package_id, topo.num_packages); + if (!summary_only && topo.num_packages > 1) + BIC_PRESENT(BIC_Package); - if (!tcc_default) - goto guess; + set_node_data(); + if (debug > 1) + fprintf(outf, "nodes_per_pkg %d\n", topo.nodes_per_pkg); + if (!summary_only && topo.nodes_per_pkg > 1) + BIC_PRESENT(BIC_Node); - tj_max = tcc_default; + topo.threads_per_core = max_siblings; + if (debug > 1) + fprintf(outf, "max_siblings %d\n", max_siblings); - return 0; + if (debug < 1) + return; -guess: - tj_max = TJMAX_DEFAULT; - fprintf(outf, "cpu%d: Guessing tjMax %d C, Please use -T to specify\n", cpu, tj_max); + for (i = 0; i <= topo.max_cpu_num; ++i) { + if (cpu_is_not_present(i)) + continue; + fprintf(outf, + "cpu %d pkg %d die %d l3 %d node %d lnode %d core %d thread %d\n", + i, cpus[i].physical_package_id, cpus[i].die_id, cpus[i].l3_id, + cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].physical_core_id, cpus[i].thread_id); + } - return 0; } -void decode_feature_control_msr(void) +void allocate_counters(struct thread_data **t, struct core_data **c, struct pkg_data **p) { - unsigned long long msr; + int i; + int num_cores = topo.cores_per_node * topo.nodes_per_pkg * topo.num_packages; + int num_threads = topo.threads_per_core * num_cores; - if (!get_msr(base_cpu, MSR_IA32_FEAT_CTL, &msr)) - fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n", - base_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : ""); -} + *t = calloc(num_threads, sizeof(struct thread_data)); + if (*t == NULL) + goto error; -void decode_misc_enable_msr(void) -{ - unsigned long long msr; + for (i = 0; i < num_threads; i++) + (*t)[i].cpu_id = -1; - if (!genuine_intel) - return; + *c = calloc(num_cores, sizeof(struct core_data)); + if (*c == NULL) + goto error; - if (!get_msr(base_cpu, MSR_IA32_MISC_ENABLE, &msr)) - fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n", - base_cpu, msr, - msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-", - msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-", - msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-", - msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", - msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : ""); -} + for (i = 0; i < num_cores; i++) { + (*c)[i].core_id = -1; + (*c)[i].base_cpu = -1; + } -void decode_misc_feature_control(void) -{ - unsigned long long msr; + *p = calloc(topo.num_packages, sizeof(struct pkg_data)); + if (*p == NULL) + goto error; - if (!has_misc_feature_control) - return; + for (i = 0; i < topo.num_packages; i++) { + (*p)[i].package_id = i; + (*p)[i].base_cpu = -1; + } - if (!get_msr(base_cpu, MSR_MISC_FEATURE_CONTROL, &msr)) - fprintf(outf, - "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n", - base_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", - msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : ""); + return; +error: + err(1, "calloc counters"); } /* - * Decode MSR_MISC_PWR_MGMT + * init_counter() * - * Decode the bits according to the Nehalem documentation - * bit[0] seems to continue to have same meaning going forward - * bit[1] less so... + * set FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE */ -void decode_misc_pwr_mgmt_msr(void) +void init_counter(struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base, int cpu_id) { - unsigned long long msr; + int pkg_id = cpus[cpu_id].physical_package_id; + int node_id = cpus[cpu_id].logical_node_id; + int core_id = cpus[cpu_id].physical_core_id; + int thread_id = cpus[cpu_id].thread_id; + struct thread_data *t; + struct core_data *c; - if (!do_nhm_platform_info) - return; + /* Workaround for systems where physical_node_id==-1 + * and logical_node_id==(-1 - topo.num_cpus) + */ + if (node_id < 0) + node_id = 0; - if (no_MSR_MISC_PWR_MGMT) - return; + t = GET_THREAD(thread_base, thread_id, core_id, node_id, pkg_id); + c = GET_CORE(core_base, core_id, node_id, pkg_id); - if (!get_msr(base_cpu, MSR_MISC_PWR_MGMT, &msr)) - fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n", - base_cpu, msr, - msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS"); + t->cpu_id = cpu_id; + if (!cpu_is_not_allowed(cpu_id)) { + + if (c->base_cpu < 0) + c->base_cpu = t->cpu_id; + if (pkg_base[pkg_id].base_cpu < 0) + pkg_base[pkg_id].base_cpu = t->cpu_id; + } + + c->core_id = core_id; + pkg_base[pkg_id].package_id = pkg_id; } -/* - * Decode MSR_CC6_DEMOTION_POLICY_CONFIG, MSR_MC6_DEMOTION_POLICY_CONFIG - * - * This MSRs are present on Silvermont processors, - * Intel Atom processor E3000 series (Baytrail), and friends. - */ -void decode_c6_demotion_policy_msr(void) +int initialize_counters(int cpu_id) { - unsigned long long msr; - - if (!get_msr(base_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr)) - fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", - base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); + init_counter(EVEN_COUNTERS, cpu_id); + init_counter(ODD_COUNTERS, cpu_id); + return 0; +} - if (!get_msr(base_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr)) - fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", - base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); +void allocate_output_buffer() +{ + output_buffer = calloc(1, (1 + topo.num_cpus) * 2048); + outp = output_buffer; + if (outp == NULL) + err(-1, "calloc output buffer"); } -/* - * When models are the same, for the purpose of turbostat, reuse - */ -unsigned int intel_model_duplicates(unsigned int model) -{ - - switch (model) { - case INTEL_FAM6_NEHALEM_EP: /* Core i7, Xeon 5500 series - Bloomfield, Gainstown NHM-EP */ - case INTEL_FAM6_NEHALEM: /* Core i7 and i5 Processor - Clarksfield, Lynnfield, Jasper Forest */ - case 0x1F: /* Core i7 and i5 Processor - Nehalem */ - case INTEL_FAM6_WESTMERE: /* Westmere Client - Clarkdale, Arrandale */ - case INTEL_FAM6_WESTMERE_EP: /* Westmere EP - Gulftown */ - return INTEL_FAM6_NEHALEM; - - case INTEL_FAM6_NEHALEM_EX: /* Nehalem-EX Xeon - Beckton */ - case INTEL_FAM6_WESTMERE_EX: /* Westmere-EX Xeon - Eagleton */ - return INTEL_FAM6_NEHALEM_EX; - - case INTEL_FAM6_XEON_PHI_KNM: - return INTEL_FAM6_XEON_PHI_KNL; - - case INTEL_FAM6_BROADWELL_X: - case INTEL_FAM6_BROADWELL_D: /* BDX-DE */ - return INTEL_FAM6_BROADWELL_X; - - case INTEL_FAM6_SKYLAKE_L: - case INTEL_FAM6_SKYLAKE: - case INTEL_FAM6_KABYLAKE_L: - case INTEL_FAM6_KABYLAKE: - case INTEL_FAM6_COMETLAKE_L: - case INTEL_FAM6_COMETLAKE: - return INTEL_FAM6_SKYLAKE_L; - - case INTEL_FAM6_ICELAKE_L: - case INTEL_FAM6_ICELAKE_NNPI: - case INTEL_FAM6_TIGERLAKE_L: - case INTEL_FAM6_TIGERLAKE: - case INTEL_FAM6_ROCKETLAKE: - case INTEL_FAM6_LAKEFIELD: - case INTEL_FAM6_ALDERLAKE: - case INTEL_FAM6_ALDERLAKE_L: - case INTEL_FAM6_ATOM_GRACEMONT: - case INTEL_FAM6_RAPTORLAKE: - case INTEL_FAM6_RAPTORLAKE_P: - case INTEL_FAM6_RAPTORLAKE_S: - case INTEL_FAM6_METEORLAKE: - case INTEL_FAM6_METEORLAKE_L: - return INTEL_FAM6_CANNONLAKE_L; - - case INTEL_FAM6_ATOM_TREMONT_L: - return INTEL_FAM6_ATOM_TREMONT; - - case INTEL_FAM6_ICELAKE_D: - return INTEL_FAM6_ICELAKE_X; - - case INTEL_FAM6_EMERALDRAPIDS_X: - return INTEL_FAM6_SAPPHIRERAPIDS_X; - } - return model; +void allocate_fd_percpu(void) +{ + fd_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); + if (fd_percpu == NULL) + err(-1, "calloc fd_percpu"); } -void print_dev_latency(void) +void allocate_irq_buffers(void) { - char *path = "/dev/cpu_dma_latency"; - int fd; - int value; - int retval; + irq_column_2_cpu = calloc(topo.num_cpus, sizeof(int)); + if (irq_column_2_cpu == NULL) + err(-1, "calloc %d", topo.num_cpus); - fd = open(path, O_RDONLY); - if (fd < 0) { - if (debug) - warnx("Read %s failed", path); - return; - } + irqs_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int)); + if (irqs_per_cpu == NULL) + err(-1, "calloc %d IRQ", topo.max_cpu_num + 1); + + nmi_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int)); + if (nmi_per_cpu == NULL) + err(-1, "calloc %d NMI", topo.max_cpu_num + 1); +} - retval = read(fd, (void *)&value, sizeof(int)); - if (retval != sizeof(int)) { - warn("read failed %s", path); - close(fd); - return; - } - fprintf(outf, "/dev/cpu_dma_latency: %d usec (%s)\n", value, value == 2000000000 ? "default" : "constrained"); +int update_topo(PER_THREAD_PARAMS) +{ + topo.allowed_cpus++; + if ((int)t->cpu_id == c->base_cpu) + topo.allowed_cores++; + if ((int)t->cpu_id == p->base_cpu) + topo.allowed_packages++; - close(fd); + return 0; } -/* - * Linux-perf manages the HW instructions-retired counter - * by enabling when requested, and hiding rollover - */ -void linux_perf_init(void) +void topology_update(void) { - if (!BIC_IS_ENABLED(BIC_IPC)) - return; + topo.allowed_cpus = 0; + topo.allowed_cores = 0; + topo.allowed_packages = 0; + for_all_cpus(update_topo, ODD_COUNTERS); +} - if (access("/proc/sys/kernel/perf_event_paranoid", F_OK)) - return; +void setup_all_buffers(bool startup) +{ + topology_probe(startup); + allocate_irq_buffers(); + allocate_fd_percpu(); + allocate_counters(&thread_even, &core_even, &package_even); + allocate_counters(&thread_odd, &core_odd, &package_odd); + allocate_output_buffer(); + for_all_proc_cpus(initialize_counters); + topology_update(); +} - fd_instr_count_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); - if (fd_instr_count_percpu == NULL) - err(-1, "calloc fd_instr_count_percpu"); +void set_base_cpu(void) +{ + int i; - BIC_PRESENT(BIC_IPC); + for (i = 0; i < topo.max_cpu_num + 1; ++i) { + if (cpu_is_not_allowed(i)) + continue; + base_cpu = i; + if (debug > 1) + fprintf(outf, "base_cpu = %d\n", base_cpu); + return; + } + err(-ENODEV, "No valid cpus found"); } -void process_cpuid() +bool has_added_counters(void) { - unsigned int eax, ebx, ecx, edx; - unsigned int fms, family, model, stepping, ecx_flags, edx_flags; - unsigned long long ucode_patch = 0; - bool ucode_patch_valid = false; + /* + * It only makes sense to call this after the command line is parsed, + * otherwise sys structure is not populated. + */ - eax = ebx = ecx = edx = 0; + return sys.added_core_counters | sys.added_thread_counters | sys.added_package_counters; +} - __cpuid(0, max_level, ebx, ecx, edx); +void check_msr_access(void) +{ + check_msr_driver(); + check_msr_permission(); - if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69) - genuine_intel = 1; - else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65) - authentic_amd = 1; - else if (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e) - hygon_genuine = 1; + if (no_msr) + bic_disable_msr_access(); +} - if (!quiet) - fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n", - (char *)&ebx, (char *)&edx, (char *)&ecx, max_level); +void check_perf_access(void) +{ + if (BIC_IS_ENABLED(BIC_IPC)) + if (!has_perf_instr_count_access()) + no_perf = 1; - __cpuid(1, fms, ebx, ecx, edx); - family = (fms >> 8) & 0xf; - model = (fms >> 4) & 0xf; - stepping = fms & 0xf; - if (family == 0xf) - family += (fms >> 20) & 0xff; - if (family >= 6) - model += ((fms >> 16) & 0xf) << 4; - ecx_flags = ecx; - edx_flags = edx; + if (BIC_IS_ENABLED(BIC_LLC_RPS) || BIC_IS_ENABLED(BIC_LLC_HIT)) + if (!has_perf_llc_access()) + no_perf = 1; - if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) - warnx("get_msr(UCODE)"); - else - ucode_patch_valid = true; + if (no_perf) + bic_disable_perf_access(); +} +bool perf_has_hybrid_devices(void) +{ /* - * check max extended function levels of CPUID. - * This is needed to check for invariant TSC. - * This check is valid for both Intel and AMD. + * 0: unknown + * 1: has separate perf device for p and e core + * -1: doesn't have separate perf device for p and e core */ - ebx = ecx = edx = 0; - __cpuid(0x80000000, max_extended_level, ebx, ecx, edx); + static int cached; - if (!quiet) { - fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", - family, model, stepping, family, model, stepping); - if (ucode_patch_valid) - fprintf(outf, " microcode 0x%x", (unsigned int)((ucode_patch >> 32) & 0xFFFFFFFF)); - fputc('\n', outf); + if (cached > 0) + return true; - fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level); - fprintf(outf, "CPUID(1): %s %s %s %s %s %s %s %s %s %s\n", - ecx_flags & (1 << 0) ? "SSE3" : "-", - ecx_flags & (1 << 3) ? "MONITOR" : "-", - ecx_flags & (1 << 6) ? "SMX" : "-", - ecx_flags & (1 << 7) ? "EIST" : "-", - ecx_flags & (1 << 8) ? "TM2" : "-", - edx_flags & (1 << 4) ? "TSC" : "-", - edx_flags & (1 << 5) ? "MSR" : "-", - edx_flags & (1 << 22) ? "ACPI-TM" : "-", - edx_flags & (1 << 28) ? "HT" : "-", edx_flags & (1 << 29) ? "TM" : "-"); + if (cached < 0) + return false; + + if (access("/sys/bus/event_source/devices/cpu_core", F_OK)) { + cached = -1; + return false; } - if (genuine_intel) { - model_orig = model; - model = intel_model_duplicates(model); + + if (access("/sys/bus/event_source/devices/cpu_atom", F_OK)) { + cached = -1; + return false; } - if (!(edx_flags & (1 << 5))) - errx(1, "CPUID: no MSR"); + cached = 1; + return true; +} - if (max_extended_level >= 0x80000007) { +int added_perf_counters_init_(struct perf_counter_info *pinfo) +{ + size_t num_domains = 0; + unsigned int next_domain; + bool *domain_visited; + unsigned int perf_type, perf_config; + double perf_scale; + int fd_perf; - /* - * Non-Stop TSC is advertised by CPUID.EAX=0x80000007: EDX.bit8 - * this check is valid for both Intel and AMD - */ - __cpuid(0x80000007, eax, ebx, ecx, edx); - has_invariant_tsc = edx & (1 << 8); - } + if (!pinfo) + return 0; - /* - * APERF/MPERF is advertised by CPUID.EAX=0x6: ECX.bit0 - * this check is valid for both Intel and AMD - */ + const size_t max_num_domains = MAX(topo.max_cpu_num + 1, MAX(topo.max_core_id + 1, topo.max_package_id + 1)); - __cpuid(0x6, eax, ebx, ecx, edx); - has_aperf = ecx & (1 << 0); - if (has_aperf) { - BIC_PRESENT(BIC_Avg_MHz); - BIC_PRESENT(BIC_Busy); - BIC_PRESENT(BIC_Bzy_MHz); - } - do_dts = eax & (1 << 0); - if (do_dts) - BIC_PRESENT(BIC_CoreTmp); - has_turbo = eax & (1 << 1); - do_ptm = eax & (1 << 6); - if (do_ptm) - BIC_PRESENT(BIC_PkgTmp); - has_hwp = eax & (1 << 7); - has_hwp_notify = eax & (1 << 8); - has_hwp_activity_window = eax & (1 << 9); - has_hwp_epp = eax & (1 << 10); - has_hwp_pkg = eax & (1 << 11); - has_epb = ecx & (1 << 3); + domain_visited = calloc(max_num_domains, sizeof(*domain_visited)); - if (!quiet) - fprintf(outf, "CPUID(6): %sAPERF, %sTURBO, %sDTS, %sPTM, %sHWP, " - "%sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n", - has_aperf ? "" : "No-", - has_turbo ? "" : "No-", - do_dts ? "" : "No-", - do_ptm ? "" : "No-", - has_hwp ? "" : "No-", - has_hwp_notify ? "" : "No-", - has_hwp_activity_window ? "" : "No-", - has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-"); + while (pinfo) { + switch (pinfo->scope) { + case SCOPE_CPU: + num_domains = topo.max_cpu_num + 1; + break; - if (!quiet) - decode_misc_enable_msr(); + case SCOPE_CORE: + num_domains = topo.max_core_id + 1; + break; - if (max_level >= 0x7 && !quiet) { - int has_sgx; + case SCOPE_PACKAGE: + num_domains = topo.max_package_id + 1; + break; + } - ecx = 0; + /* Allocate buffer for file descriptor for each domain. */ + pinfo->fd_perf_per_domain = calloc(num_domains, sizeof(*pinfo->fd_perf_per_domain)); + if (!pinfo->fd_perf_per_domain) + errx(1, "%s: alloc %s", __func__, "fd_perf_per_domain"); - __cpuid_count(0x7, 0, eax, ebx, ecx, edx); + for (size_t i = 0; i < num_domains; ++i) + pinfo->fd_perf_per_domain[i] = -1; - has_sgx = ebx & (1 << 2); + pinfo->num_domains = num_domains; + pinfo->scale = 1.0; - is_hybrid = edx & (1 << 15); + memset(domain_visited, 0, max_num_domains * sizeof(*domain_visited)); - fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-"); + for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) { - if (has_sgx) - decode_feature_control_msr(); - } + next_domain = cpu_to_domain(pinfo, cpu); - if (max_level >= 0x15) { - unsigned int eax_crystal; - unsigned int ebx_tsc; + assert(next_domain < num_domains); - /* - * CPUID 15H TSC/Crystal ratio, possibly Crystal Hz - */ - eax_crystal = ebx_tsc = crystal_hz = edx = 0; - __cpuid(0x15, eax_crystal, ebx_tsc, crystal_hz, edx); + if (cpu_is_not_allowed(cpu)) + continue; - if (ebx_tsc != 0) { + if (domain_visited[next_domain]) + continue; - if (!quiet && (ebx != 0)) - fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n", - eax_crystal, ebx_tsc, crystal_hz); + /* + * Intel hybrid platforms expose different perf devices for P and E cores. + * Instead of one, "/sys/bus/event_source/devices/cpu" device, there are + * "/sys/bus/event_source/devices/{cpu_core,cpu_atom}". + * + * This makes it more complicated to the user, because most of the counters + * are available on both and have to be handled manually, otherwise. + * + * Code below, allow user to use the old "cpu" name, which is translated accordingly. + */ + const char *perf_device = pinfo->device; - if (crystal_hz == 0) - switch (model) { - case INTEL_FAM6_SKYLAKE_L: /* SKL */ - crystal_hz = 24000000; /* 24.0 MHz */ + if (strcmp(perf_device, "cpu") == 0 && perf_has_hybrid_devices()) { + switch (cpus[cpu].type) { + case INTEL_PCORE_TYPE: + perf_device = "cpu_core"; break; - case INTEL_FAM6_ATOM_GOLDMONT_D: /* DNV */ - crystal_hz = 25000000; /* 25.0 MHz */ + + case INTEL_ECORE_TYPE: + perf_device = "cpu_atom"; break; - case INTEL_FAM6_ATOM_GOLDMONT: /* BXT */ - case INTEL_FAM6_ATOM_GOLDMONT_PLUS: - crystal_hz = 19200000; /* 19.2 MHz */ + + default: /* Don't change, we will probably fail and report a problem soon. */ break; - default: - crystal_hz = 0; } - - if (crystal_hz) { - tsc_hz = (unsigned long long)crystal_hz *ebx_tsc / eax_crystal; - if (!quiet) - fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n", - tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal); } - } - } - if (max_level >= 0x16) { - unsigned int base_mhz, max_mhz, bus_mhz, edx; - - /* - * CPUID 16H Base MHz, Max MHz, Bus MHz - */ - base_mhz = max_mhz = bus_mhz = edx = 0; - - __cpuid(0x16, base_mhz, max_mhz, bus_mhz, edx); - if (!quiet) - fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n", - base_mhz, max_mhz, bus_mhz); - } - - if (has_aperf) - aperf_mperf_multiplier = get_aperf_mperf_multiplier(family, model); - - BIC_PRESENT(BIC_IRQ); - BIC_PRESENT(BIC_TSC_MHz); - if (probe_nhm_msrs(family, model)) { - do_nhm_platform_info = 1; - BIC_PRESENT(BIC_CPU_c1); - BIC_PRESENT(BIC_CPU_c3); - BIC_PRESENT(BIC_CPU_c6); - BIC_PRESENT(BIC_SMI); - } - do_snb_cstates = has_snb_msrs(family, model); - - if (do_snb_cstates) - BIC_PRESENT(BIC_CPU_c7); - - do_irtl_snb = has_snb_msrs(family, model); - if (do_snb_cstates && (pkg_cstate_limit >= PCL__2)) - BIC_PRESENT(BIC_Pkgpc2); - if (pkg_cstate_limit >= PCL__3) - BIC_PRESENT(BIC_Pkgpc3); - if (pkg_cstate_limit >= PCL__6) - BIC_PRESENT(BIC_Pkgpc6); - if (do_snb_cstates && (pkg_cstate_limit >= PCL__7)) - BIC_PRESENT(BIC_Pkgpc7); - if (has_slv_msrs(family, model)) { - BIC_NOT_PRESENT(BIC_Pkgpc2); - BIC_NOT_PRESENT(BIC_Pkgpc3); - BIC_PRESENT(BIC_Pkgpc6); - BIC_NOT_PRESENT(BIC_Pkgpc7); - BIC_PRESENT(BIC_Mod_c6); - use_c1_residency_msr = 1; - } - if (is_jvl(family, model)) { - BIC_NOT_PRESENT(BIC_CPU_c3); - BIC_NOT_PRESENT(BIC_CPU_c7); - BIC_NOT_PRESENT(BIC_Pkgpc2); - BIC_NOT_PRESENT(BIC_Pkgpc3); - BIC_NOT_PRESENT(BIC_Pkgpc6); - BIC_NOT_PRESENT(BIC_Pkgpc7); - } - if (is_dnv(family, model)) { - BIC_PRESENT(BIC_CPU_c1); - BIC_NOT_PRESENT(BIC_CPU_c3); - BIC_NOT_PRESENT(BIC_Pkgpc3); - BIC_NOT_PRESENT(BIC_CPU_c7); - BIC_NOT_PRESENT(BIC_Pkgpc7); - use_c1_residency_msr = 1; - } - if (is_skx(family, model) || is_icx(family, model) || is_spr(family, model)) { - BIC_NOT_PRESENT(BIC_CPU_c3); - BIC_NOT_PRESENT(BIC_Pkgpc3); - BIC_NOT_PRESENT(BIC_CPU_c7); - BIC_NOT_PRESENT(BIC_Pkgpc7); - } - if (is_bdx(family, model)) { - BIC_NOT_PRESENT(BIC_CPU_c7); - BIC_NOT_PRESENT(BIC_Pkgpc7); - } - if (has_c8910_msrs(family, model)) { - if (pkg_cstate_limit >= PCL__8) - BIC_PRESENT(BIC_Pkgpc8); - if (pkg_cstate_limit >= PCL__9) - BIC_PRESENT(BIC_Pkgpc9); - if (pkg_cstate_limit >= PCL_10) - BIC_PRESENT(BIC_Pkgpc10); - } - do_irtl_hsw = has_c8910_msrs(family, model); - if (has_skl_msrs(family, model)) { - BIC_PRESENT(BIC_Totl_c0); - BIC_PRESENT(BIC_Any_c0); - BIC_PRESENT(BIC_GFX_c0); - BIC_PRESENT(BIC_CPUGFX); - } - do_slm_cstates = is_slm(family, model); - do_knl_cstates = is_knl(family, model); + perf_type = read_perf_type(perf_device); + if (perf_type == (unsigned int)-1) { + warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "type"); + continue; + } - if (do_slm_cstates || do_knl_cstates || is_cnl(family, model) || is_ehl(family, model)) - BIC_NOT_PRESENT(BIC_CPU_c3); + perf_config = read_perf_config(perf_device, pinfo->event); + if (perf_config == (unsigned int)-1) { + warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "config"); + continue; + } - if (!quiet) - decode_misc_pwr_mgmt_msr(); + /* Scale is not required, some counters just don't have it. */ + perf_scale = read_perf_scale(perf_device, pinfo->event); + if (perf_scale == 0.0) + perf_scale = 1.0; - if (!quiet && has_slv_msrs(family, model)) - decode_c6_demotion_policy_msr(); + fd_perf = open_perf_counter(cpu, perf_type, perf_config, -1, 0); + if (fd_perf == -1) { + warnx("%s: perf/%s/%s: failed to open counter on cpu%d", __func__, perf_device, pinfo->event, cpu); + continue; + } - rapl_probe(family, model); - perf_limit_reasons_probe(family, model); - automatic_cstate_conversion_probe(family, model); - prewake_cstate_probe(family, model); + domain_visited[next_domain] = 1; + pinfo->fd_perf_per_domain[next_domain] = fd_perf; + pinfo->scale = perf_scale; - check_tcc_offset(model_orig); + if (debug) + fprintf(stderr, "Add perf/%s/%s cpu%d: %d\n", perf_device, pinfo->event, cpu, pinfo->fd_perf_per_domain[next_domain]); + } - if (!quiet) - dump_cstate_pstate_config_info(family, model); - intel_uncore_frequency_probe(); + pinfo = pinfo->next; + } - if (!quiet) - print_dev_latency(); - if (!quiet) - dump_sysfs_cstate_config(); - if (!quiet) - dump_sysfs_pstate_config(); + free(domain_visited); - if (has_skl_msrs(family, model) || is_ehl(family, model)) - calculate_tsc_tweak(); + return 0; +} - if (!access("/sys/class/drm/card0/power/rc6_residency_ms", R_OK)) - BIC_PRESENT(BIC_GFX_rc6); +void added_perf_counters_init(void) +{ + if (added_perf_counters_init_(sys.perf_tp)) + errx(1, "%s: %s", __func__, "thread"); - if (!access("/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", R_OK)) - BIC_PRESENT(BIC_GFXMHz); + if (added_perf_counters_init_(sys.perf_cp)) + errx(1, "%s: %s", __func__, "core"); - if (!access("/sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz", R_OK)) - BIC_PRESENT(BIC_GFXACTMHz); + if (added_perf_counters_init_(sys.perf_pp)) + errx(1, "%s: %s", __func__, "package"); +} - if (!access("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", R_OK)) - BIC_PRESENT(BIC_CPU_LPI); - else - BIC_NOT_PRESENT(BIC_CPU_LPI); +int parse_telem_info_file(int fd_dir, const char *info_filename, const char *format, unsigned long *output) +{ + int fd_telem_info; + FILE *file_telem_info; + unsigned long value; - if (!access("/sys/devices/system/cpu/cpu0/thermal_throttle/core_throttle_count", R_OK)) - BIC_PRESENT(BIC_CORE_THROT_CNT); - else - BIC_NOT_PRESENT(BIC_CORE_THROT_CNT); + fd_telem_info = openat(fd_dir, info_filename, O_RDONLY); + if (fd_telem_info == -1) + return -1; - if (!access(sys_lpi_file_sysfs, R_OK)) { - sys_lpi_file = sys_lpi_file_sysfs; - BIC_PRESENT(BIC_SYS_LPI); - } else if (!access(sys_lpi_file_debugfs, R_OK)) { - sys_lpi_file = sys_lpi_file_debugfs; - BIC_PRESENT(BIC_SYS_LPI); - } else { - sys_lpi_file_sysfs = NULL; - BIC_NOT_PRESENT(BIC_SYS_LPI); + file_telem_info = fdopen(fd_telem_info, "r"); + if (file_telem_info == NULL) { + close(fd_telem_info); + return -1; } - if (!quiet) - decode_misc_feature_control(); - - return; -} - -/* - * in /dev/cpu/ return success for names that are numbers - * ie. filter out ".", "..", "microcode". - */ -int dir_filter(const struct dirent *dirp) -{ - if (isdigit(dirp->d_name[0])) - return 1; - else - return 0; -} + if (fscanf(file_telem_info, format, &value) != 1) { + fclose(file_telem_info); + return -1; + } -void topology_probe() -{ - int i; - int max_core_id = 0; - int max_package_id = 0; - int max_siblings = 0; + fclose(file_telem_info); - /* Initialize num_cpus, max_cpu_num */ - set_max_cpu_num(); - topo.num_cpus = 0; - for_all_proc_cpus(count_cpus); - if (!summary_only && topo.num_cpus > 1) - BIC_PRESENT(BIC_CPU); + *output = value; - if (debug > 1) - fprintf(outf, "num_cpus %d max_cpu_num %d\n", topo.num_cpus, topo.max_cpu_num); + return 0; +} - cpus = calloc(1, (topo.max_cpu_num + 1) * sizeof(struct cpu_topology)); - if (cpus == NULL) - err(1, "calloc cpus"); +struct pmt_mmio *pmt_mmio_open(unsigned int target_guid) +{ + struct pmt_diriter_t pmt_iter; + const struct dirent *entry; + struct stat st; + int fd_telem_dir, fd_pmt; + unsigned long guid, size, offset; + size_t mmap_size; + void *mmio; + struct pmt_mmio *head = NULL, *last = NULL; + struct pmt_mmio *new_pmt = NULL; - /* - * Allocate and initialize cpu_present_set - */ - cpu_present_set = CPU_ALLOC((topo.max_cpu_num + 1)); - if (cpu_present_set == NULL) - err(3, "CPU_ALLOC"); - cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); - CPU_ZERO_S(cpu_present_setsize, cpu_present_set); - for_all_proc_cpus(mark_cpu_present); + if (stat(SYSFS_TELEM_PATH, &st) == -1) + return NULL; - /* - * Validate that all cpus in cpu_subset are also in cpu_present_set - */ - for (i = 0; i < CPU_SUBSET_MAXCPUS; ++i) { - if (CPU_ISSET_S(i, cpu_subset_size, cpu_subset)) - if (!CPU_ISSET_S(i, cpu_present_setsize, cpu_present_set)) - err(1, "cpu%d not present", i); + pmt_diriter_init(&pmt_iter); + entry = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH); + if (!entry) { + pmt_diriter_remove(&pmt_iter); + return NULL; } - /* - * Allocate and initialize cpu_affinity_set - */ - cpu_affinity_set = CPU_ALLOC((topo.max_cpu_num + 1)); - if (cpu_affinity_set == NULL) - err(3, "CPU_ALLOC"); - cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); - CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set); + for (; entry != NULL; entry = pmt_diriter_next(&pmt_iter)) { + if (fstatat(dirfd(pmt_iter.dir), entry->d_name, &st, 0) == -1) + break; - for_all_proc_cpus(init_thread_id); + if (!S_ISDIR(st.st_mode)) + continue; - /* - * For online cpus - * find max_core_id, max_package_id - */ - for (i = 0; i <= topo.max_cpu_num; ++i) { - int siblings; + fd_telem_dir = openat(dirfd(pmt_iter.dir), entry->d_name, O_RDONLY); + if (fd_telem_dir == -1) + break; - if (cpu_is_not_present(i)) { - if (debug > 1) - fprintf(outf, "cpu%d NOT PRESENT\n", i); - continue; + if (parse_telem_info_file(fd_telem_dir, "guid", "%lx", &guid)) { + close(fd_telem_dir); + break; } - cpus[i].logical_cpu_id = i; + if (parse_telem_info_file(fd_telem_dir, "size", "%lu", &size)) { + close(fd_telem_dir); + break; + } - /* get package information */ - cpus[i].physical_package_id = get_physical_package_id(i); - if (cpus[i].physical_package_id > max_package_id) - max_package_id = cpus[i].physical_package_id; + if (guid != target_guid) { + close(fd_telem_dir); + continue; + } - /* get die information */ - cpus[i].die_id = get_die_id(i); - if (cpus[i].die_id > topo.max_die_id) - topo.max_die_id = cpus[i].die_id; + if (parse_telem_info_file(fd_telem_dir, "offset", "%lu", &offset)) { + close(fd_telem_dir); + break; + } - /* get numa node information */ - cpus[i].physical_node_id = get_physical_node_id(&cpus[i]); - if (cpus[i].physical_node_id > topo.max_node_num) - topo.max_node_num = cpus[i].physical_node_id; + assert(offset == 0); - /* get core information */ - cpus[i].physical_core_id = get_core_id(i); - if (cpus[i].physical_core_id > max_core_id) - max_core_id = cpus[i].physical_core_id; + fd_pmt = openat(fd_telem_dir, "telem", O_RDONLY); + if (fd_pmt == -1) + goto loop_cleanup_and_break; - /* get thread information */ - siblings = get_thread_siblings(&cpus[i]); - if (siblings > max_siblings) - max_siblings = siblings; - if (cpus[i].thread_id == 0) - topo.num_cores++; - } + mmap_size = ROUND_UP_TO_PAGE_SIZE(size); + mmio = mmap(0, mmap_size, PROT_READ, MAP_SHARED, fd_pmt, 0); + if (mmio != MAP_FAILED) { + if (debug) + fprintf(stderr, "%s: 0x%lx mmaped at: %p\n", __func__, guid, mmio); - topo.cores_per_node = max_core_id + 1; - if (debug > 1) - fprintf(outf, "max_core_id %d, sizing for %d cores per package\n", max_core_id, topo.cores_per_node); - if (!summary_only && topo.cores_per_node > 1) - BIC_PRESENT(BIC_Core); + new_pmt = calloc(1, sizeof(*new_pmt)); - topo.num_die = topo.max_die_id + 1; - if (debug > 1) - fprintf(outf, "max_die_id %d, sizing for %d die\n", topo.max_die_id, topo.num_die); - if (!summary_only && topo.num_die > 1) - BIC_PRESENT(BIC_Die); + if (!new_pmt) { + fprintf(stderr, "%s: Failed to allocate pmt_mmio\n", __func__); + exit(1); + } - topo.num_packages = max_package_id + 1; - if (debug > 1) - fprintf(outf, "max_package_id %d, sizing for %d packages\n", max_package_id, topo.num_packages); - if (!summary_only && topo.num_packages > 1) - BIC_PRESENT(BIC_Package); + /* + * Create linked list of mmaped regions, + * but preserve the ordering from sysfs. + * Ordering is important for the user to + * use the seq=%u parameter when adding a counter. + */ + new_pmt->guid = guid; + new_pmt->mmio_base = mmio; + new_pmt->pmt_offset = offset; + new_pmt->size = size; + new_pmt->next = pmt_mmios; + + if (last) + last->next = new_pmt; + else + head = new_pmt; - set_node_data(); - if (debug > 1) - fprintf(outf, "nodes_per_pkg %d\n", topo.nodes_per_pkg); - if (!summary_only && topo.nodes_per_pkg > 1) - BIC_PRESENT(BIC_Node); + last = new_pmt; + } - topo.threads_per_core = max_siblings; - if (debug > 1) - fprintf(outf, "max_siblings %d\n", max_siblings); +loop_cleanup_and_break: + close(fd_pmt); + close(fd_telem_dir); + } - if (debug < 1) - return; + pmt_diriter_remove(&pmt_iter); - for (i = 0; i <= topo.max_cpu_num; ++i) { - if (cpu_is_not_present(i)) - continue; - fprintf(outf, - "cpu %d pkg %d die %d node %d lnode %d core %d thread %d\n", - i, cpus[i].physical_package_id, cpus[i].die_id, - cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].physical_core_id, cpus[i].thread_id); - } + /* + * If we found something, stick just + * created linked list to the front. + */ + if (head) + pmt_mmios = head; + return head; } -void allocate_counters(struct thread_data **t, struct core_data **c, struct pkg_data **p) +struct pmt_mmio *pmt_mmio_find(unsigned int guid) { - int i; - int num_cores = topo.cores_per_node * topo.nodes_per_pkg * topo.num_packages; - int num_threads = topo.threads_per_core * num_cores; + struct pmt_mmio *pmmio = pmt_mmios; - *t = calloc(num_threads, sizeof(struct thread_data)); - if (*t == NULL) - goto error; + while (pmmio) { + if (pmmio->guid == guid) + return pmmio; - for (i = 0; i < num_threads; i++) - (*t)[i].cpu_id = -1; + pmmio = pmmio->next; + } - *c = calloc(num_cores, sizeof(struct core_data)); - if (*c == NULL) - goto error; + return NULL; +} - for (i = 0; i < num_cores; i++) - (*c)[i].core_id = -1; +void *pmt_get_counter_pointer(struct pmt_mmio *pmmio, unsigned long counter_offset) +{ + char *ret; - *p = calloc(topo.num_packages, sizeof(struct pkg_data)); - if (*p == NULL) - goto error; + /* Get base of mmaped PMT file. */ + ret = (char *)pmmio->mmio_base; - for (i = 0; i < topo.num_packages; i++) - (*p)[i].package_id = i; + /* + * Apply PMT MMIO offset to obtain beginning of the mmaped telemetry data. + * It's not guaranteed that the mmaped memory begins with the telemetry data + * - we might have to apply the offset first. + */ + ret += pmmio->pmt_offset; - return; -error: - err(1, "calloc counters"); + /* Apply the counter offset to get the address to the mmaped counter. */ + ret += counter_offset; + + return ret; } -/* - * init_counter() - * - * set FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE - */ -void init_counter(struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base, int cpu_id) +struct pmt_mmio *pmt_add_guid(unsigned int guid, unsigned int seq) { - int pkg_id = cpus[cpu_id].physical_package_id; - int node_id = cpus[cpu_id].logical_node_id; - int core_id = cpus[cpu_id].physical_core_id; - int thread_id = cpus[cpu_id].thread_id; - struct thread_data *t; - struct core_data *c; - struct pkg_data *p; + struct pmt_mmio *ret; - /* Workaround for systems where physical_node_id==-1 - * and logical_node_id==(-1 - topo.num_cpus) - */ - if (node_id < 0) - node_id = 0; - - t = GET_THREAD(thread_base, thread_id, core_id, node_id, pkg_id); - c = GET_CORE(core_base, core_id, node_id, pkg_id); - p = GET_PKG(pkg_base, pkg_id); + ret = pmt_mmio_find(guid); + if (!ret) + ret = pmt_mmio_open(guid); - t->cpu_id = cpu_id; - if (thread_id == 0) { - t->flags |= CPU_IS_FIRST_THREAD_IN_CORE; - if (cpu_is_first_core_in_package(cpu_id)) - t->flags |= CPU_IS_FIRST_CORE_IN_PACKAGE; + while (ret && seq) { + ret = ret->next; + --seq; } - c->core_id = core_id; - p->package_id = pkg_id; + return ret; } -int initialize_counters(int cpu_id) -{ - init_counter(EVEN_COUNTERS, cpu_id); - init_counter(ODD_COUNTERS, cpu_id); - return 0; -} +enum pmt_open_mode { + PMT_OPEN_TRY, /* Open failure is not an error. */ + PMT_OPEN_REQUIRED, /* Open failure is a fatal error. */ +}; -void allocate_output_buffer() +struct pmt_counter *pmt_find_counter(struct pmt_counter *pcounter, const char *name) { - output_buffer = calloc(1, (1 + topo.num_cpus) * 2048); - outp = output_buffer; - if (outp == NULL) - err(-1, "calloc output buffer"); + while (pcounter) { + if (strcmp(pcounter->name, name) == 0) + break; + + pcounter = pcounter->next; + } + + return pcounter; } -void allocate_fd_percpu(void) +struct pmt_counter **pmt_get_scope_root(enum counter_scope scope) { - fd_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); - if (fd_percpu == NULL) - err(-1, "calloc fd_percpu"); + switch (scope) { + case SCOPE_CPU: + return &sys.pmt_tp; + case SCOPE_CORE: + return &sys.pmt_cp; + case SCOPE_PACKAGE: + return &sys.pmt_pp; + } + + __builtin_unreachable(); } -void allocate_irq_buffers(void) +void pmt_counter_add_domain(struct pmt_counter *pcounter, unsigned long *pmmio, unsigned int domain_id) { - irq_column_2_cpu = calloc(topo.num_cpus, sizeof(int)); - if (irq_column_2_cpu == NULL) - err(-1, "calloc %d", topo.num_cpus); + /* Make sure the new domain fits. */ + if (domain_id >= pcounter->num_domains) + pmt_counter_resize(pcounter, domain_id + 1); - irqs_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int)); - if (irqs_per_cpu == NULL) - err(-1, "calloc %d", topo.max_cpu_num + 1); + assert(pcounter->domains); + assert(domain_id < pcounter->num_domains); + + pcounter->domains[domain_id].pcounter = pmmio; } -void setup_all_buffers(void) +int pmt_add_counter(unsigned int guid, unsigned int seq, const char *name, enum pmt_datatype type, + unsigned int lsb, unsigned int msb, unsigned int offset, enum counter_scope scope, + enum counter_format format, unsigned int domain_id, enum pmt_open_mode mode) { - topology_probe(); - allocate_irq_buffers(); - allocate_fd_percpu(); - allocate_counters(&thread_even, &core_even, &package_even); - allocate_counters(&thread_odd, &core_odd, &package_odd); - allocate_output_buffer(); - for_all_proc_cpus(initialize_counters); + struct pmt_mmio *mmio; + struct pmt_counter *pcounter; + struct pmt_counter **const pmt_root = pmt_get_scope_root(scope); + bool new_counter = false; + int conflict = 0; + + if (lsb > msb) { + fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "lsb <= msb", name); + exit(1); + } + + if (msb >= 64) { + fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "msb < 64", name); + exit(1); + } + + mmio = pmt_add_guid(guid, seq); + if (!mmio) { + if (mode != PMT_OPEN_TRY) { + fprintf(stderr, "%s: failed to map PMT MMIO for guid %x, seq %u\n", __func__, guid, seq); + exit(1); + } + + return 1; + } + + if (offset >= mmio->size) { + if (mode != PMT_OPEN_TRY) { + fprintf(stderr, "%s: offset %u outside of PMT MMIO size %u\n", __func__, offset, mmio->size); + exit(1); + } + + return 1; + } + + pcounter = pmt_find_counter(*pmt_root, name); + if (!pcounter) { + pcounter = calloc(1, sizeof(*pcounter)); + new_counter = true; + } + + if (new_counter) { + strncpy(pcounter->name, name, ARRAY_SIZE(pcounter->name) - 1); + pcounter->type = type; + pcounter->scope = scope; + pcounter->lsb = lsb; + pcounter->msb = msb; + pcounter->format = format; + } else { + conflict += pcounter->type != type; + conflict += pcounter->scope != scope; + conflict += pcounter->lsb != lsb; + conflict += pcounter->msb != msb; + conflict += pcounter->format != format; + } + + if (conflict) { + fprintf(stderr, "%s: conflicting parameters for the PMT counter with the same name %s\n", __func__, name); + exit(1); + } + + pmt_counter_add_domain(pcounter, pmt_get_counter_pointer(mmio, offset), domain_id); + + if (new_counter) { + pcounter->next = *pmt_root; + *pmt_root = pcounter; + } + + return 0; } -void set_base_cpu(void) +void pmt_init(void) { - base_cpu = sched_getcpu(); - if (base_cpu < 0) - err(-ENODEV, "No valid cpus found"); + int cpu_num; + unsigned long seq, offset, mod_num; + + if (BIC_IS_ENABLED(BIC_Diec6)) { + pmt_add_counter(PMT_MTL_DC6_GUID, PMT_MTL_DC6_SEQ, "Die%c6", PMT_TYPE_XTAL_TIME, + PMT_COUNTER_MTL_DC6_LSB, PMT_COUNTER_MTL_DC6_MSB, PMT_COUNTER_MTL_DC6_OFFSET, + SCOPE_PACKAGE, FORMAT_DELTA, 0, PMT_OPEN_TRY); + } + + if (BIC_IS_ENABLED(BIC_CPU_c1e)) { + seq = 0; + offset = PMT_COUNTER_CWF_MC1E_OFFSET_BASE; + mod_num = 0; /* Relative module number for current PMT file. */ + + /* Open the counter for each CPU. */ + for (cpu_num = 0; cpu_num < topo.max_cpu_num;) { + + if (cpu_is_not_allowed(cpu_num)) + goto next_loop_iter; + + /* + * Set the scope to CPU, even though CWF report the counter per module. + * CPUs inside the same module will read from the same location, instead of reporting zeros. + * + * CWF with newer firmware might require a PMT_TYPE_XTAL_TIME intead of PMT_TYPE_TCORE_CLOCK. + */ + pmt_add_counter(PMT_CWF_MC1E_GUID, seq, "CPU%c1e", PMT_TYPE_TCORE_CLOCK, + PMT_COUNTER_CWF_MC1E_LSB, PMT_COUNTER_CWF_MC1E_MSB, offset, SCOPE_CPU, FORMAT_DELTA, cpu_num, PMT_OPEN_TRY); + + /* + * Rather complex logic for each time we go to the next loop iteration, + * so keep it as a label. + */ +next_loop_iter: + /* + * Advance the cpu number and check if we should also advance offset to + * the next counter inside the PMT file. + * + * On Clearwater Forest platform, the counter is reported per module, + * so open the same counter for all of the CPUs inside the module. + * That way, reported table show the correct value for all of the CPUs inside the module, + * instead of zeros. + */ + ++cpu_num; + if (cpu_num % PMT_COUNTER_CWF_CPUS_PER_MODULE == 0) { + offset += PMT_COUNTER_CWF_MC1E_OFFSET_INCREMENT; + ++mod_num; + } - if (debug > 1) - fprintf(outf, "base_cpu = %d\n", base_cpu); + /* + * There are PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE in each PMT file. + * + * If that number is reached, seq must be incremented to advance to the next file in a sequence. + * Offset inside that file and a module counter has to be reset. + */ + if (mod_num == PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE) { + ++seq; + offset = PMT_COUNTER_CWF_MC1E_OFFSET_BASE; + mod_num = 0; + } + } + } } void turbostat_init() { - setup_all_buffers(); + setup_all_buffers(true); set_base_cpu(); - check_dev_msr(); - check_permissions(); + check_msr_access(); + check_perf_access(); process_cpuid(); + counter_info_init(); + probe_pm_features(); + msr_perf_init(); linux_perf_init(); - - if (!quiet) - for_all_cpus(print_hwp, ODD_COUNTERS); - - if (!quiet) - for_all_cpus(print_epb, ODD_COUNTERS); - - if (!quiet) - for_all_cpus(print_perf_limit, ODD_COUNTERS); - - if (!quiet) - for_all_cpus(print_rapl, ODD_COUNTERS); - - for_all_cpus(set_temperature_target, ODD_COUNTERS); + rapl_perf_init(); + cstate_perf_init(); + perf_llc_init(); + added_perf_counters_init(); + pmt_init(); for_all_cpus(get_cpu_type, ODD_COUNTERS); for_all_cpus(get_cpu_type, EVEN_COUNTERS); - if (!quiet) - for_all_cpus(print_thermal, ODD_COUNTERS); + if (BIC_IS_ENABLED(BIC_IPC) && has_aperf_access && get_instr_count_fd(base_cpu) != -1) + BIC_PRESENT(BIC_IPC); + + /* + * If TSC tweak is needed, but couldn't get it, + * disable more BICs, since it can't be reported accurately. + */ + if (platform->enable_tsc_tweak && !has_base_hz) { + CLR_BIC(BIC_Busy, &bic_enabled); + CLR_BIC(BIC_Bzy_MHz, &bic_enabled); + } +} - if (!quiet && do_irtl_snb) - print_irtl(); +void affinitize_child(void) +{ + /* Prefer cpu_possible_set, if available */ + if (sched_setaffinity(0, cpu_possible_setsize, cpu_possible_set)) { + warn("sched_setaffinity cpu_possible_set"); - if (DO_BIC(BIC_IPC)) - (void)get_instr_count_fd(base_cpu); + /* Otherwise, allow child to run on same cpu set as turbostat */ + if (sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set)) + warn("sched_setaffinity cpu_allowed_set"); + } } int fork_it(char **argv) @@ -6171,13 +10091,12 @@ int fork_it(char **argv) first_counter_read = 0; if (status) exit(status); - /* clear affinity side-effect of get_counters() */ - sched_setaffinity(0, cpu_present_setsize, cpu_present_set); gettimeofday(&tv_even, (struct timezone *)NULL); child_pid = fork(); if (!child_pid) { /* child */ + affinitize_child(); execvp(argv[0], argv); err(errno, "exec %s", argv[0]); } else { @@ -6204,10 +10123,10 @@ int fork_it(char **argv) timersub(&tv_odd, &tv_even, &tv_delta); if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS)) fprintf(outf, "%s: Counter reset detected\n", progname); - else { - compute_average(EVEN_COUNTERS); - format_all_counters(EVEN_COUNTERS); - } + delta_platform(&platform_counters_odd, &platform_counters_even); + + compute_average(EVEN_COUNTERS); + format_all_counters(EVEN_COUNTERS); fprintf(outf, "%.6f sec\n", tv_delta.tv_sec + tv_delta.tv_usec / 1000000.0); @@ -6236,7 +10155,7 @@ int get_and_dump_counters(void) void print_version() { - fprintf(outf, "turbostat version 2023.03.17 - Len Brown \n"); + fprintf(outf, "turbostat version 2025.12.02 - Len Brown \n"); } #define COMMAND_LINE_SIZE 2048 @@ -6262,166 +10181,640 @@ void print_bootcmd(void) fclose(fp); } +struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name) +{ + struct msr_counter *mp; + + for (mp = head; mp; mp = mp->next) { + if (debug) + fprintf(stderr, "%s: %s %s\n", __func__, name, mp->name); + if (!strcmp(name, mp->name)) + return mp; + } + return NULL; +} + int add_counter(unsigned int msr_num, char *path, char *name, - unsigned int width, enum counter_scope scope, - enum counter_type type, enum counter_format format, int flags) + unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int id) { struct msr_counter *msrp; - msrp = calloc(1, sizeof(struct msr_counter)); + if (no_msr && msr_num) + errx(1, "Requested MSR counter 0x%x, but in --no-msr mode", msr_num); + + if (debug) + fprintf(stderr, "%s(msr%d, %s, %s, width%d, scope%d, type%d, format%d, flags%x, id%d)\n", + __func__, msr_num, path, name, width, scope, type, format, flags, id); + + switch (scope) { + + case SCOPE_CPU: + msrp = find_msrp_by_name(sys.tp, name); + if (msrp) { + if (debug) + fprintf(stderr, "%s: %s FOUND\n", __func__, name); + break; + } + if (sys.added_thread_counters++ >= MAX_ADDED_THREAD_COUNTERS) { + warnx("ignoring thread counter %s", name); + return -1; + } + break; + case SCOPE_CORE: + msrp = find_msrp_by_name(sys.cp, name); + if (msrp) { + if (debug) + fprintf(stderr, "%s: %s FOUND\n", __func__, name); + break; + } + if (sys.added_core_counters++ >= MAX_ADDED_CORE_COUNTERS) { + warnx("ignoring core counter %s", name); + return -1; + } + break; + case SCOPE_PACKAGE: + msrp = find_msrp_by_name(sys.pp, name); + if (msrp) { + if (debug) + fprintf(stderr, "%s: %s FOUND\n", __func__, name); + break; + } + if (sys.added_package_counters++ >= MAX_ADDED_PACKAGE_COUNTERS) { + warnx("ignoring package counter %s", name); + return -1; + } + break; + default: + warnx("ignoring counter %s with unknown scope", name); + return -1; + } + if (msrp == NULL) { - perror("calloc"); + msrp = calloc(1, sizeof(struct msr_counter)); + if (msrp == NULL) + err(-1, "calloc msr_counter"); + + msrp->msr_num = msr_num; + strncpy(msrp->name, name, NAME_BYTES - 1); + msrp->width = width; + msrp->type = type; + msrp->format = format; + msrp->flags = flags; + + switch (scope) { + case SCOPE_CPU: + msrp->next = sys.tp; + sys.tp = msrp; + break; + case SCOPE_CORE: + msrp->next = sys.cp; + sys.cp = msrp; + break; + case SCOPE_PACKAGE: + msrp->next = sys.pp; + sys.pp = msrp; + break; + } + } + + if (path) { + struct sysfs_path *sp; + + sp = calloc(1, sizeof(struct sysfs_path)); + if (sp == NULL) { + perror("calloc"); + exit(1); + } + strncpy(sp->path, path, PATH_BYTES - 1); + sp->id = id; + sp->next = msrp->sp; + msrp->sp = sp; + } + + return 0; +} + +/* + * Initialize the fields used for identifying and opening the counter. + * + * Defer the initialization of any runtime buffers for actually reading + * the counters for when we initialize all perf counters, so we can later + * easily call re_initialize(). + */ +struct perf_counter_info *make_perf_counter_info(const char *perf_device, + const char *perf_event, + const char *name, + unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format) +{ + struct perf_counter_info *pinfo; + + pinfo = calloc(1, sizeof(*pinfo)); + if (!pinfo) + errx(1, "%s: Failed to allocate %s/%s\n", __func__, perf_device, perf_event); + + strncpy(pinfo->device, perf_device, ARRAY_SIZE(pinfo->device) - 1); + strncpy(pinfo->event, perf_event, ARRAY_SIZE(pinfo->event) - 1); + + strncpy(pinfo->name, name, ARRAY_SIZE(pinfo->name) - 1); + pinfo->width = width; + pinfo->scope = scope; + pinfo->type = type; + pinfo->format = format; + + return pinfo; +} + +int add_perf_counter(const char *perf_device, const char *perf_event, const char *name_buffer, unsigned int width, + enum counter_scope scope, enum counter_type type, enum counter_format format) +{ + struct perf_counter_info *pinfo; + + switch (scope) { + case SCOPE_CPU: + if (sys.added_thread_perf_counters >= MAX_ADDED_THREAD_COUNTERS) { + warnx("ignoring thread counter perf/%s/%s", perf_device, perf_event); + return -1; + } + break; + + case SCOPE_CORE: + if (sys.added_core_perf_counters >= MAX_ADDED_CORE_COUNTERS) { + warnx("ignoring core counter perf/%s/%s", perf_device, perf_event); + return -1; + } + break; + + case SCOPE_PACKAGE: + if (sys.added_package_perf_counters >= MAX_ADDED_PACKAGE_COUNTERS) { + warnx("ignoring package counter perf/%s/%s", perf_device, perf_event); + return -1; + } + break; + } + + pinfo = make_perf_counter_info(perf_device, perf_event, name_buffer, width, scope, type, format); + + if (!pinfo) + return -1; + + switch (scope) { + case SCOPE_CPU: + pinfo->next = sys.perf_tp; + sys.perf_tp = pinfo; + ++sys.added_thread_perf_counters; + break; + + case SCOPE_CORE: + pinfo->next = sys.perf_cp; + sys.perf_cp = pinfo; + ++sys.added_core_perf_counters; + break; + + case SCOPE_PACKAGE: + pinfo->next = sys.perf_pp; + sys.perf_pp = pinfo; + ++sys.added_package_perf_counters; + break; + } + + // FIXME: we might not have debug here yet + if (debug) + fprintf(stderr, "%s: %s/%s, name: %s, scope%d\n", __func__, pinfo->device, pinfo->event, pinfo->name, pinfo->scope); + + return 0; +} + +void parse_add_command_msr(char *add_command) +{ + int msr_num = 0; + char *path = NULL; + char perf_device[PERF_DEV_NAME_BYTES] = ""; + char perf_event[PERF_EVT_NAME_BYTES] = ""; + char name_buffer[PERF_NAME_BYTES] = ""; + int width = 64; + int fail = 0; + enum counter_scope scope = SCOPE_CPU; + enum counter_type type = COUNTER_CYCLES; + enum counter_format format = FORMAT_DELTA; + + while (add_command) { + + if (sscanf(add_command, "msr0x%x", &msr_num) == 1) + goto next; + + if (sscanf(add_command, "msr%d", &msr_num) == 1) + goto next; + + BUILD_BUG_ON(ARRAY_SIZE(perf_device) <= 31); + BUILD_BUG_ON(ARRAY_SIZE(perf_event) <= 31); + if (sscanf(add_command, "perf/%31[^/]/%31[^,]", &perf_device[0], &perf_event[0]) == 2) + goto next; + + if (*add_command == '/') { + path = add_command; + goto next; + } + + if (sscanf(add_command, "u%d", &width) == 1) { + if ((width == 32) || (width == 64)) + goto next; + width = 64; + } + if (!strncmp(add_command, "cpu", strlen("cpu"))) { + scope = SCOPE_CPU; + goto next; + } + if (!strncmp(add_command, "core", strlen("core"))) { + scope = SCOPE_CORE; + goto next; + } + if (!strncmp(add_command, "package", strlen("package"))) { + scope = SCOPE_PACKAGE; + goto next; + } + if (!strncmp(add_command, "cycles", strlen("cycles"))) { + type = COUNTER_CYCLES; + goto next; + } + if (!strncmp(add_command, "seconds", strlen("seconds"))) { + type = COUNTER_SECONDS; + goto next; + } + if (!strncmp(add_command, "usec", strlen("usec"))) { + type = COUNTER_USEC; + goto next; + } + if (!strncmp(add_command, "raw", strlen("raw"))) { + format = FORMAT_RAW; + goto next; + } + if (!strncmp(add_command, "average", strlen("average"))) { + format = FORMAT_AVERAGE; + goto next; + } + if (!strncmp(add_command, "delta", strlen("delta"))) { + format = FORMAT_DELTA; + goto next; + } + if (!strncmp(add_command, "percent", strlen("percent"))) { + format = FORMAT_PERCENT; + goto next; + } + + BUILD_BUG_ON(ARRAY_SIZE(name_buffer) <= 18); + if (sscanf(add_command, "%18s,%*s", name_buffer) == 1) { + char *eos; + + eos = strchr(name_buffer, ','); + if (eos) + *eos = '\0'; + goto next; + } + +next: + add_command = strchr(add_command, ','); + if (add_command) { + *add_command = '\0'; + add_command++; + } + + } + if ((msr_num == 0) && (path == NULL) && (perf_device[0] == '\0' || perf_event[0] == '\0')) { + fprintf(stderr, "--add: (msrDDD | msr0xXXX | /path_to_counter | perf/device/event) required\n"); + fail++; + } + + /* Test for non-empty perf_device and perf_event */ + const bool is_perf_counter = perf_device[0] && perf_event[0]; + + /* generate default column header */ + if (*name_buffer == '\0') { + if (is_perf_counter) { + snprintf(name_buffer, ARRAY_SIZE(name_buffer), "perf/%s", perf_event); + } else { + if (width == 32) + sprintf(name_buffer, "M0x%x%s", msr_num, format == FORMAT_PERCENT ? "%" : ""); + else + sprintf(name_buffer, "M0X%x%s", msr_num, format == FORMAT_PERCENT ? "%" : ""); + } + } + + if (is_perf_counter) { + if (add_perf_counter(perf_device, perf_event, name_buffer, width, scope, type, format)) + fail++; + } else { + if (add_counter(msr_num, path, name_buffer, width, scope, type, format, 0, 0)) + fail++; + } + + if (fail) { + help(); + exit(1); + } +} + +bool starts_with(const char *str, const char *prefix) +{ + return strncmp(prefix, str, strlen(prefix)) == 0; +} + +int pmt_parse_from_path(const char *target_path, unsigned int *out_guid, unsigned int *out_seq) +{ + struct pmt_diriter_t pmt_iter; + const struct dirent *dirname; + struct stat stat, target_stat; + int fd_telem_dir = -1; + int fd_target_dir; + unsigned int seq = 0; + unsigned long guid, target_guid; + int ret = -1; + + fd_target_dir = open(target_path, O_RDONLY | O_DIRECTORY); + if (fd_target_dir == -1) { + return -1; + } + + if (fstat(fd_target_dir, &target_stat) == -1) { + fprintf(stderr, "%s: Failed to stat the target: %s", __func__, strerror(errno)); + exit(1); + } + + if (parse_telem_info_file(fd_target_dir, "guid", "%lx", &target_guid)) { + fprintf(stderr, "%s: Failed to parse the target guid file: %s", __func__, strerror(errno)); exit(1); } - msrp->msr_num = msr_num; - strncpy(msrp->name, name, NAME_BYTES - 1); - if (path) - strncpy(msrp->path, path, PATH_BYTES - 1); - msrp->width = width; - msrp->type = type; - msrp->format = format; - msrp->flags = flags; + close(fd_target_dir); + + pmt_diriter_init(&pmt_iter); + + for (dirname = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH); dirname != NULL; dirname = pmt_diriter_next(&pmt_iter)) { + + fd_telem_dir = openat(dirfd(pmt_iter.dir), dirname->d_name, O_RDONLY | O_DIRECTORY); + if (fd_telem_dir == -1) + continue; + + if (parse_telem_info_file(fd_telem_dir, "guid", "%lx", &guid)) { + fprintf(stderr, "%s: Failed to parse the guid file: %s", __func__, strerror(errno)); + continue; + } + + if (fstat(fd_telem_dir, &stat) == -1) { + fprintf(stderr, "%s: Failed to stat %s directory: %s", __func__, dirname->d_name, strerror(errno)); + continue; + } + + /* + * If reached the same directory as target, exit the loop. + * Seq has the correct value now. + */ + if (stat.st_dev == target_stat.st_dev && stat.st_ino == target_stat.st_ino) { + ret = 0; + break; + } + + /* + * If reached directory with the same guid, + * but it's not the target directory yet, + * increment seq and continue the search. + */ + if (guid == target_guid) + ++seq; - switch (scope) { + close(fd_telem_dir); + fd_telem_dir = -1; + } - case SCOPE_CPU: - msrp->next = sys.tp; - sys.tp = msrp; - sys.added_thread_counters++; - if (sys.added_thread_counters > MAX_ADDED_THREAD_COUNTERS) { - fprintf(stderr, "exceeded max %d added thread counters\n", MAX_ADDED_COUNTERS); - exit(-1); - } - break; + pmt_diriter_remove(&pmt_iter); - case SCOPE_CORE: - msrp->next = sys.cp; - sys.cp = msrp; - sys.added_core_counters++; - if (sys.added_core_counters > MAX_ADDED_COUNTERS) { - fprintf(stderr, "exceeded max %d added core counters\n", MAX_ADDED_COUNTERS); - exit(-1); - } - break; + if (fd_telem_dir != -1) + close(fd_telem_dir); - case SCOPE_PACKAGE: - msrp->next = sys.pp; - sys.pp = msrp; - sys.added_package_counters++; - if (sys.added_package_counters > MAX_ADDED_COUNTERS) { - fprintf(stderr, "exceeded max %d added package counters\n", MAX_ADDED_COUNTERS); - exit(-1); - } - break; + if (!ret) { + *out_guid = target_guid; + *out_seq = seq; } - return 0; + return ret; } -void parse_add_command(char *add_command) +void parse_add_command_pmt(char *add_command) { - int msr_num = 0; - char *path = NULL; - char name_buffer[NAME_BYTES] = ""; - int width = 64; - int fail = 0; - enum counter_scope scope = SCOPE_CPU; - enum counter_type type = COUNTER_CYCLES; - enum counter_format format = FORMAT_DELTA; + char *name = NULL; + char *type_name = NULL; + char *format_name = NULL; + char *direct_path = NULL; + static const char direct_path_prefix[] = "path="; + unsigned int offset; + unsigned int lsb; + unsigned int msb; + unsigned int guid; + unsigned int seq = 0; /* By default, pick first file in a sequence with a given GUID. */ + unsigned int domain_id; + enum counter_scope scope = 0; + enum pmt_datatype type = PMT_TYPE_RAW; + enum counter_format format = FORMAT_RAW; + bool has_offset = false; + bool has_lsb = false; + bool has_msb = false; + bool has_format = true; /* Format has a default value. */ + bool has_guid = false; + bool has_scope = false; + bool has_type = true; /* Type has a default value. */ - while (add_command) { + /* Consume the "pmt," prefix. */ + add_command = strchr(add_command, ','); + if (!add_command) { + help(); + exit(1); + } + ++add_command; - if (sscanf(add_command, "msr0x%x", &msr_num) == 1) + while (add_command) { + if (starts_with(add_command, "name=")) { + name = add_command + strlen("name="); goto next; + } - if (sscanf(add_command, "msr%d", &msr_num) == 1) + if (starts_with(add_command, "type=")) { + type_name = add_command + strlen("type="); goto next; + } + + if (starts_with(add_command, "domain=")) { + const size_t prefix_len = strlen("domain="); + + if (sscanf(add_command + prefix_len, "cpu%u", &domain_id) == 1) { + scope = SCOPE_CPU; + has_scope = true; + } else if (sscanf(add_command + prefix_len, "core%u", &domain_id) == 1) { + scope = SCOPE_CORE; + has_scope = true; + } else if (sscanf(add_command + prefix_len, "package%u", &domain_id) == 1) { + scope = SCOPE_PACKAGE; + has_scope = true; + } + + if (!has_scope) { + printf("%s: invalid value for scope. Expected cpu%%u, core%%u or package%%u.\n", __func__); + exit(1); + } - if (*add_command == '/') { - path = add_command; goto next; } - if (sscanf(add_command, "u%d", &width) == 1) { - if ((width == 32) || (width == 64)) - goto next; - width = 64; - } - if (!strncmp(add_command, "cpu", strlen("cpu"))) { - scope = SCOPE_CPU; + if (starts_with(add_command, "format=")) { + format_name = add_command + strlen("format="); goto next; } - if (!strncmp(add_command, "core", strlen("core"))) { - scope = SCOPE_CORE; + + if (sscanf(add_command, "offset=%u", &offset) == 1) { + has_offset = true; goto next; } - if (!strncmp(add_command, "package", strlen("package"))) { - scope = SCOPE_PACKAGE; + + if (sscanf(add_command, "lsb=%u", &lsb) == 1) { + has_lsb = true; goto next; } - if (!strncmp(add_command, "cycles", strlen("cycles"))) { - type = COUNTER_CYCLES; + + if (sscanf(add_command, "msb=%u", &msb) == 1) { + has_msb = true; goto next; } - if (!strncmp(add_command, "seconds", strlen("seconds"))) { - type = COUNTER_SECONDS; + + if (sscanf(add_command, "guid=%x", &guid) == 1) { + has_guid = true; goto next; } - if (!strncmp(add_command, "usec", strlen("usec"))) { - type = COUNTER_USEC; + + if (sscanf(add_command, "seq=%x", &seq) == 1) + goto next; + + if (strncmp(add_command, direct_path_prefix, strlen(direct_path_prefix)) == 0) { + direct_path = add_command + strlen(direct_path_prefix); goto next; } - if (!strncmp(add_command, "raw", strlen("raw"))) { +next: + add_command = strchr(add_command, ','); + if (add_command) { + *add_command = '\0'; + add_command++; + } + } + + if (!name) { + printf("%s: missing %s\n", __func__, "name"); + exit(1); + } + + if (strlen(name) >= PMT_COUNTER_NAME_SIZE_BYTES) { + printf("%s: name has to be at most %d characters long\n", __func__, PMT_COUNTER_NAME_SIZE_BYTES); + exit(1); + } + + if (format_name) { + has_format = false; + + if (strcmp("raw", format_name) == 0) { format = FORMAT_RAW; - goto next; + has_format = true; } - if (!strncmp(add_command, "delta", strlen("delta"))) { + + if (strcmp("average", format_name) == 0) { + format = FORMAT_AVERAGE; + has_format = true; + } + + if (strcmp("delta", format_name) == 0) { format = FORMAT_DELTA; - goto next; + has_format = true; } - if (!strncmp(add_command, "percent", strlen("percent"))) { - format = FORMAT_PERCENT; - goto next; + + if (!has_format) { + fprintf(stderr, "%s: Invalid format %s. Expected raw, average or delta\n", __func__, format_name); + exit(1); } + } - if (sscanf(add_command, "%18s,%*s", name_buffer) == 1) { /* 18 < NAME_BYTES */ - char *eos; + if (type_name) { + has_type = false; - eos = strchr(name_buffer, ','); - if (eos) - *eos = '\0'; - goto next; + if (strcmp("raw", type_name) == 0) { + type = PMT_TYPE_RAW; + has_type = true; } -next: - add_command = strchr(add_command, ','); - if (add_command) { - *add_command = '\0'; - add_command++; + if (strcmp("txtal_time", type_name) == 0) { + type = PMT_TYPE_XTAL_TIME; + has_type = true; } + if (strcmp("tcore_clock", type_name) == 0) { + type = PMT_TYPE_TCORE_CLOCK; + has_type = true; + } + + if (!has_type) { + printf("%s: invalid %s: %s\n", __func__, "type", type_name); + exit(1); + } } - if ((msr_num == 0) && (path == NULL)) { - fprintf(stderr, "--add: (msrDDD | msr0xXXX | /path_to_counter ) required\n"); - fail++; + + if (!has_offset) { + printf("%s : missing %s\n", __func__, "offset"); + exit(1); } - /* generate default column header */ - if (*name_buffer == '\0') { - if (width == 32) - sprintf(name_buffer, "M0x%x%s", msr_num, format == FORMAT_PERCENT ? "%" : ""); - else - sprintf(name_buffer, "M0X%x%s", msr_num, format == FORMAT_PERCENT ? "%" : ""); + if (!has_lsb) { + printf("%s: missing %s\n", __func__, "lsb"); + exit(1); } - if (add_counter(msr_num, path, name_buffer, width, scope, type, format, 0)) - fail++; + if (!has_msb) { + printf("%s: missing %s\n", __func__, "msb"); + exit(1); + } - if (fail) { - help(); + if (direct_path && has_guid) { + printf("%s: path and guid+seq parameters are mutually exclusive\n" + "notice: passed guid=0x%x and path=%s\n", __func__, guid, direct_path); + exit(1); + } + + if (direct_path) { + if (pmt_parse_from_path(direct_path, &guid, &seq)) { + printf("%s: failed to parse PMT file from %s\n", __func__, direct_path); + exit(1); + } + + /* GUID was just infered from the direct path. */ + has_guid = true; + } + + if (!has_guid) { + printf("%s: missing %s\n", __func__, "guid or path"); + exit(1); + } + + if (!has_scope) { + printf("%s: missing %s\n", __func__, "scope"); + exit(1); + } + + if (lsb > msb) { + printf("%s: lsb > msb doesn't make sense\n", __func__); exit(1); } + + pmt_add_counter(guid, seq, name, type, lsb, msb, offset, scope, format, domain_id, PMT_OPEN_REQUIRED); +} + +void parse_add_command(char *add_command) +{ + if (strncmp(add_command, "pmt", strlen("pmt")) == 0) + return parse_add_command_pmt(add_command); + return parse_add_command_msr(add_command); } int is_deferred_add(char *name) @@ -6429,8 +10822,10 @@ int is_deferred_add(char *name) int i; for (i = 0; i < deferred_add_index; ++i) - if (!strcmp(name, deferred_add_names[i])) + if (!strcmp(name, deferred_add_names[i])) { + deferred_add_consumed |= (1 << i); return 1; + } return 0; } @@ -6439,17 +10834,41 @@ int is_deferred_skip(char *name) int i; for (i = 0; i < deferred_skip_index; ++i) - if (!strcmp(name, deferred_skip_names[i])) + if (!strcmp(name, deferred_skip_names[i])) { + deferred_skip_consumed |= (1 << i); return 1; + } return 0; } -void probe_sysfs(void) +void verify_deferred_consumed(void) +{ + int i; + int fail = 0; + + for (i = 0; i < deferred_add_index; ++i) { + if (!(deferred_add_consumed & (1 << i))) { + warnx("Counter '%s' can not be added.", deferred_add_names[i]); + fail++; + } + } + for (i = 0; i < deferred_skip_index; ++i) { + if (!(deferred_skip_consumed & (1 << i))) { + warnx("Counter '%s' can not be skipped.", deferred_skip_names[i]); + fail++; + } + } + if (fail) + exit(-EINVAL); +} + +void probe_cpuidle_residency(void) { char path[64]; char name_buf[16]; FILE *input; int state; + int min_state = 1024, max_state = 0; char *sp; for (state = 10; state >= 0; --state) { @@ -6474,14 +10893,40 @@ void probe_sysfs(void) sprintf(path, "cpuidle/state%d/time", state); - if (!DO_BIC(BIC_sysfs) && !is_deferred_add(name_buf)) + if (!DO_BIC(BIC_pct_idle) && !is_deferred_add(name_buf)) continue; if (is_deferred_skip(name_buf)) continue; - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU); + add_counter(0, path, name_buf, 32, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU, 0); + + if (state > max_state) + max_state = state; + if (state < min_state) + min_state = state; } +} + +static bool cpuidle_counter_wanted(char *name) +{ + if (is_deferred_skip(name)) + return false; + + return DO_BIC(BIC_cpuidle) || is_deferred_add(name); +} + +void probe_cpuidle_counts(void) +{ + char path[64]; + char name_buf[16]; + FILE *input; + int state; + int min_state = 1024, max_state = 0; + char *sp; + + if (!DO_BIC(BIC_cpuidle) && !deferred_add_index) + return; for (state = 10; state >= 0; --state) { @@ -6491,26 +10936,51 @@ void probe_sysfs(void) continue; if (!fgets(name_buf, sizeof(name_buf), input)) err(1, "%s: failed to read file", path); - /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */ - sp = strchr(name_buf, '-'); - if (!sp) - sp = strchrnul(name_buf, '\n'); - *sp = '\0'; fclose(input); remove_underbar(name_buf); - sprintf(path, "cpuidle/state%d/usage", state); + /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */ + sp = strchr(name_buf, '-'); + if (!sp) + sp = strchrnul(name_buf, '\n'); - if (!DO_BIC(BIC_sysfs) && !is_deferred_add(name_buf)) - continue; + /* + * The 'below' sysfs file always contains 0 for the deepest state (largest index), + * do not add it. + */ + if (state != max_state) { + /* + * Add 'C1+' for C1, and so on. The 'below' sysfs file always contains 0 for + * the last state, so do not add it. + */ + *sp = '+'; + *(sp + 1) = '\0'; + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/below", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } + } - if (is_deferred_skip(name_buf)) - continue; + *sp = '\0'; + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/usage", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU); + /* + * The 'above' sysfs file always contains 0 for the shallowest state (smallest + * index), do not add it. + */ + if (state != min_state) { + *sp = '-'; + *(sp + 1) = '\0'; + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/above", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } + } } - } /* @@ -6519,9 +10989,6 @@ void probe_sysfs(void) */ void parse_cpu_command(char *optarg) { - unsigned int start, end; - char *next; - if (!strcmp(optarg, "core")) { if (cpu_subset) goto error; @@ -6544,52 +11011,8 @@ void parse_cpu_command(char *optarg) CPU_ZERO_S(cpu_subset_size, cpu_subset); - next = optarg; - - while (next && *next) { - - if (*next == '-') /* no negative cpu numbers */ - goto error; - - start = strtoul(next, &next, 10); - - if (start >= CPU_SUBSET_MAXCPUS) - goto error; - CPU_SET_S(start, cpu_subset_size, cpu_subset); - - if (*next == '\0') - break; - - if (*next == ',') { - next += 1; - continue; - } - - if (*next == '-') { - next += 1; /* start range */ - } else if (*next == '.') { - next += 1; - if (*next == '.') - next += 1; /* start range */ - else - goto error; - } - - end = strtoul(next, &next, 10); - if (end <= start) - goto error; - - while (++start <= end) { - if (start >= CPU_SUBSET_MAXCPUS) - goto error; - CPU_SET_S(start, cpu_subset_size, cpu_subset); - } - - if (*next == ',') - next += 1; - else if (*next != '\0') - goto error; - } + if (parse_cpu_str(optarg, cpu_subset, cpu_subset_size)) + goto error; return; @@ -6609,6 +11032,7 @@ void cmdline(int argc, char **argv) { "Dump", no_argument, 0, 'D' }, { "debug", no_argument, 0, 'd' }, /* internal, not documented */ { "enable", required_argument, 0, 'e' }, + { "force", no_argument, 0, 'f' }, { "interval", required_argument, 0, 'i' }, { "IPC", no_argument, 0, 'I' }, { "num_iterations", required_argument, 0, 'n' }, @@ -6619,6 +11043,8 @@ void cmdline(int argc, char **argv) { "list", no_argument, 0, 'l' }, { "out", required_argument, 0, 'o' }, { "quiet", no_argument, 0, 'q' }, + { "no-msr", no_argument, 0, 'M' }, + { "no-perf", no_argument, 0, 'P' }, { "show", required_argument, 0, 's' }, { "Summary", no_argument, 0, 'S' }, { "TCC", required_argument, 0, 'T' }, @@ -6628,7 +11054,25 @@ void cmdline(int argc, char **argv) progname = argv[0]; - while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:o:qST:v", long_options, &option_index)) != -1) { + /* + * Parse some options early, because they may make other options invalid, + * like adding the MSR counter with --add and at the same time using --no-msr. + */ + while ((opt = getopt_long_only(argc, argv, "+MPn:", long_options, &option_index)) != -1) { + switch (opt) { + case 'M': + no_msr = 1; + break; + case 'P': + no_perf = 1; + break; + default: + break; + } + } + optind = 0; + + while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:N:o:qMPST:v", long_options, &option_index)) != -1) { switch (opt) { case 'a': parse_add_command(optarg); @@ -6638,21 +11082,37 @@ void cmdline(int argc, char **argv) break; case 'D': dump_only++; + /* + * Force the no_perf early to prevent using it as a source. + * User asks for raw values, but perf returns them relative + * to the opening of the file descriptor. + */ + no_perf = 1; break; case 'e': - /* --enable specified counter */ - bic_enabled = bic_enabled | bic_lookup(optarg, SHOW_LIST); + /* --enable specified counter, without clearning existing list */ + bic_lookup(&bic_enabled, optarg, SHOW_LIST); + break; + case 'f': + force_load++; break; case 'd': debug++; - ENABLE_BIC(BIC_DISABLED_BY_DEFAULT); + bic_set_all(&bic_enabled); break; case 'H': /* * --hide: do not show those specified * multiple invocations simply clear more bits in enabled mask */ - bic_enabled &= ~bic_lookup(optarg, HIDE_LIST); + { + cpu_set_t bic_group_hide; + + BIC_INIT(&bic_group_hide); + + bic_lookup(&bic_group_hide, optarg, HIDE_LIST); + bic_clear_bits(&bic_enabled, &bic_group_hide); + } break; case 'h': default: @@ -6676,7 +11136,7 @@ void cmdline(int argc, char **argv) rapl_joules++; break; case 'l': - ENABLE_BIC(BIC_DISABLED_BY_DEFAULT); + bic_set_all(&bic_enabled); list_header_only++; quiet++; break; @@ -6686,6 +11146,10 @@ void cmdline(int argc, char **argv) case 'q': quiet = 1; break; + case 'M': + case 'P': + /* Parsed earlier */ + break; case 'n': num_iterations = strtod(optarg, NULL); @@ -6709,9 +11173,8 @@ void cmdline(int argc, char **argv) * subsequent invocations can add to it. */ if (shown == 0) - bic_enabled = bic_lookup(optarg, SHOW_LIST); - else - bic_enabled |= bic_lookup(optarg, SHOW_LIST); + BIC_INIT(&bic_enabled); + bic_lookup(&bic_enabled, optarg, SHOW_LIST); shown = 1; break; case 'S': @@ -6746,6 +11209,21 @@ void set_rlimit(void) int main(int argc, char **argv) { + int fd, ret; + + bic_groups_init(); + + fd = open("/sys/fs/cgroup/cgroup.procs", O_WRONLY); + if (fd < 0) + goto skip_cgroup_setting; + + ret = write(fd, "0\n", 2); + if (ret == -1) + perror("Can't update cgroup\n"); + + close(fd); + +skip_cgroup_setting: outf = stderr; cmdline(argc, argv); @@ -6754,14 +11232,18 @@ int main(int argc, char **argv) print_bootcmd(); } - probe_sysfs(); + probe_cpuidle_residency(); + probe_cpuidle_counts(); + + verify_deferred_consumed(); if (!getuid()) set_rlimit(); turbostat_init(); - msr_sum_record(); + if (!no_msr) + msr_sum_record(); /* dump counters and exit */ if (dump_only) diff --git a/tools/testing/selftests/turbostat/defcolumns.py b/tools/testing/selftests/turbostat/defcolumns.py new file mode 100755 index 0000000000000..d9b042097da7a --- /dev/null +++ b/tools/testing/selftests/turbostat/defcolumns.py @@ -0,0 +1,60 @@ +#!/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +import subprocess +from shutil import which + +turbostat = which('turbostat') +if turbostat is None: + print('Could not find turbostat binary') + exit(1) + +timeout = which('timeout') +if timeout is None: + print('Could not find timeout binary') + exit(1) + +proc_turbostat = subprocess.run([turbostat, '--list'], capture_output = True) +if proc_turbostat.returncode != 0: + print(f'turbostat failed with {proc_turbostat.returncode}') + exit(1) + +# +# By default --list reports also "usec" and "Time_Of_Day_Seconds" columns +# which are only visible when running with --debug. +# +expected_columns_debug = proc_turbostat.stdout.replace(b',', b'\t').strip() +expected_columns = expected_columns_debug.replace(b'usec\t', b'').replace(b'Time_Of_Day_Seconds\t', b'').replace(b'X2APIC\t', b'').replace(b'APIC\t', b'') + +# +# Run turbostat with no options for 10 seconds and send SIGINT +# +timeout_argv = [timeout, '--preserve-status', '-s', 'SIGINT', '-k', '3', '1s'] +turbostat_argv = [turbostat, '-i', '0.250'] + +print(f'Running turbostat with {turbostat_argv=}... ', end = '', flush = True) +proc_turbostat = subprocess.run(timeout_argv + turbostat_argv, capture_output = True) +if proc_turbostat.returncode != 0: + print(f'turbostat failed with {proc_turbostat.returncode}') + exit(1) +actual_columns = proc_turbostat.stdout.split(b'\n')[0] +if expected_columns != actual_columns: + print(f'turbostat column check failed\n{expected_columns=}\n{actual_columns=}') + exit(1) +print('OK') + +# +# Same, but with --debug +# +turbostat_argv.append('--debug') + +print(f'Running turbostat with {turbostat_argv=}... ', end = '', flush = True) +proc_turbostat = subprocess.run(timeout_argv + turbostat_argv, capture_output = True) +if proc_turbostat.returncode != 0: + print(f'turbostat failed with {proc_turbostat.returncode}') + exit(1) +actual_columns = proc_turbostat.stdout.split(b'\n')[0] +if expected_columns_debug != actual_columns: + print(f'turbostat column check failed\n{expected_columns_debug=}\n{actual_columns=}') + exit(1) +print('OK')