From 92a0d0f710803d21a2d502baee03bd1c56860642 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:05:00 +0800 Subject: [PATCH 001/188] cpufreq: amd-pstate: Update amd-pstate preferred core ranking dynamically commit e571a5e2068ef57945fcd5d0fb950f8f96da6dc8 upstream. Preferred core rankings can be changed dynamically by the platform based on the workload and platform conditions and accounting for thermals and aging. When this occurs, cpu priority need to be set. Tested-by: Oleksandr Natalenko Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Reviewed-by: Huang Rui Reviewed-by: Perry Yuan Signed-off-by: Meng Li Signed-off-by: Rafael J. Wysocki Signed-off-by: Abhishek Rajput Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 52 ++++++++++++++++++++++++++++++++++++ include/linux/amd-pstate.h | 6 +++++ 2 files changed, 58 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index cc55cc1d5f390..9401f2ac03d68 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -354,6 +354,7 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); + WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1)); WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1)); return 0; } @@ -378,6 +379,7 @@ static int cppc_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->lowest_nonlinear_perf, cppc_perf.lowest_nonlinear_perf); WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); + WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf); if (cppc_state == AMD_PSTATE_ACTIVE) @@ -816,6 +818,40 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) schedule_work(&sched_prefcore_work); } +static void amd_pstate_update_limits(unsigned int cpu) +{ + struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + struct amd_cpudata *cpudata = policy->driver_data; + u32 prev_high = 0, cur_high = 0; + int ret; + bool highest_perf_changed = false; + + mutex_lock(&amd_pstate_driver_lock); + if ((!amd_pstate_prefcore) || (!cpudata->hw_prefcore)) + goto free_cpufreq_put; + + ret = amd_pstate_get_highest_perf(cpu, &cur_high); + if (ret) + goto free_cpufreq_put; + + prev_high = READ_ONCE(cpudata->prefcore_ranking); + if (prev_high != cur_high) { + highest_perf_changed = true; + WRITE_ONCE(cpudata->prefcore_ranking, cur_high); + + if (cur_high < CPPC_MAX_PERF) + sched_set_itmt_core_prio((int)cur_high, cpu); + } + +free_cpufreq_put: + cpufreq_cpu_put(policy); + + if (!highest_perf_changed) + cpufreq_update_policy(cpu); + + mutex_unlock(&amd_pstate_driver_lock); +} + static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; @@ -989,6 +1025,17 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, return sysfs_emit(buf, "%u\n", perf); } +static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, + char *buf) +{ + u32 perf; + struct amd_cpudata *cpudata = policy->driver_data; + + perf = READ_ONCE(cpudata->prefcore_ranking); + + return sysfs_emit(buf, "%u\n", perf); +} + static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, char *buf) { @@ -1217,6 +1264,7 @@ cpufreq_freq_attr_ro(amd_pstate_max_freq); cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); cpufreq_freq_attr_ro(amd_pstate_highest_perf); +cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); cpufreq_freq_attr_rw(energy_performance_preference); cpufreq_freq_attr_ro(energy_performance_available_preferences); @@ -1227,6 +1275,7 @@ static struct freq_attr *amd_pstate_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_prefcore_ranking, &amd_pstate_hw_prefcore, NULL, }; @@ -1235,6 +1284,7 @@ static struct freq_attr *amd_pstate_epp_attr[] = { &amd_pstate_max_freq, &amd_pstate_lowest_nonlinear_freq, &amd_pstate_highest_perf, + &amd_pstate_prefcore_ranking, &amd_pstate_hw_prefcore, &energy_performance_preference, &energy_performance_available_preferences, @@ -1591,6 +1641,7 @@ static struct cpufreq_driver amd_pstate_driver = { .suspend = amd_pstate_cpu_suspend, .resume = amd_pstate_cpu_resume, .set_boost = amd_pstate_set_boost, + .update_limits = amd_pstate_update_limits, .name = "amd-pstate", .attr = amd_pstate_attr, }; @@ -1605,6 +1656,7 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .online = amd_pstate_epp_cpu_online, .suspend = amd_pstate_epp_suspend, .resume = amd_pstate_epp_resume, + .update_limits = amd_pstate_update_limits, .name = "amd-pstate-epp", .attr = amd_pstate_epp_attr, }; diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index 68fc1bd8d851e..d21838835abda 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -39,11 +39,16 @@ struct amd_aperf_mperf { * @cppc_req_cached: cached performance request hints * @highest_perf: the maximum performance an individual processor may reach, * assuming ideal conditions + * For platforms that do not support the preferred core feature, the + * highest_pef may be configured with 166 or 255, to avoid max frequency + * calculated wrongly. we take the fixed value as the highest_perf. * @nominal_perf: the maximum sustained performance level of the processor, * assuming ideal operating conditions * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power * savings are achieved * @lowest_perf: the absolute lowest performance level of the processor + * @prefcore_ranking: the preferred core ranking, the higher value indicates a higher + * priority. * @max_freq: the frequency that mapped to highest_perf * @min_freq: the frequency that mapped to lowest_perf * @nominal_freq: the frequency that mapped to nominal_perf @@ -73,6 +78,7 @@ struct amd_cpudata { u32 nominal_perf; u32 lowest_nonlinear_perf; u32 lowest_perf; + u32 prefcore_ranking; u32 min_limit_perf; u32 max_limit_perf; u32 min_limit_freq; From 050d0d05aa0296af3bfc2fb1ed43631a4f0145a3 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 25 Apr 2024 16:07:51 +0800 Subject: [PATCH 002/188] cpufreq: amd-pstate: Document *_limit_* fields in struct amd_cpudata commit b37ef7210e51b1e996ca03b03227d93f7470784b upstream. The four fields of struct cpudata namely min_limit_perf, max_limit_perf, min_limit_freq, max_limit_freq introduced in the commit febab20caeba("cpufreq/amd-pstate: Fix scaling_min_freq and scaling_max_freq update") are currently undocumented Add comments describing these fields Acked-by: Huang Rui Fixes: febab20caeba("cpufreq/amd-pstate: Fix scaling_min_freq and scaling_max_freq update") Reviewed-by: Li Meng Tested-by: Dhananjay Ugwekar Signed-off-by: Gautham R. Shenoy Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki Signed-off-by: Abhishek Rajput Signed-off-by: mohanasv2 --- include/linux/amd-pstate.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index d21838835abda..83fb3fc647fce 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -49,6 +49,10 @@ struct amd_aperf_mperf { * @lowest_perf: the absolute lowest performance level of the processor * @prefcore_ranking: the preferred core ranking, the higher value indicates a higher * priority. + * @min_limit_perf: Cached value of the performance corresponding to policy->min + * @max_limit_perf: Cached value of the performance corresponding to policy->max + * @min_limit_freq: Cached value of policy->min + * @max_limit_freq: Cached value of policy->max * @max_freq: the frequency that mapped to highest_perf * @min_freq: the frequency that mapped to lowest_perf * @nominal_freq: the frequency that mapped to nominal_perf From 967359baeaaa90a446cc6dae2dce3552cec2f1db Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 25 Apr 2024 16:07:52 +0800 Subject: [PATCH 003/188] cpufreq: amd-pstate: Document the units for freq variables in amd_cpudata commit 4fcfd1954ad305e331b6b4b62de2874fbae61394 upstream. The min_limit_freq, max_limit_freq, min_freq, max_freq, nominal_freq and the lowest_nominal_freq members of struct cpudata store the frequency value in khz to be consistent with the cpufreq core. Update the comment to document this. Reviewed-by: Li Meng Tested-by: Dhananjay Ugwekar Signed-off-by: Gautham R. Shenoy Signed-off-by: Perry Yuan Acked-by: Huang Rui Signed-off-by: Rafael J. Wysocki Signed-off-by: Abhishek Rajput Signed-off-by: mohanasv2 --- include/linux/amd-pstate.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index 83fb3fc647fce..ec0b0fa3e9bb4 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -51,15 +51,15 @@ struct amd_aperf_mperf { * priority. * @min_limit_perf: Cached value of the performance corresponding to policy->min * @max_limit_perf: Cached value of the performance corresponding to policy->max - * @min_limit_freq: Cached value of policy->min - * @max_limit_freq: Cached value of policy->max - * @max_freq: the frequency that mapped to highest_perf - * @min_freq: the frequency that mapped to lowest_perf - * @nominal_freq: the frequency that mapped to nominal_perf - * @lowest_nonlinear_freq: the frequency that mapped to lowest_nonlinear_perf + * @min_limit_freq: Cached value of policy->min (in khz) + * @max_limit_freq: Cached value of policy->max (in khz) + * @max_freq: the frequency (in khz) that mapped to highest_perf + * @min_freq: the frequency (in khz) that mapped to lowest_perf + * @nominal_freq: the frequency (in khz) that mapped to nominal_perf + * @lowest_nonlinear_freq: the frequency (in khz) that mapped to lowest_nonlinear_perf * @cur: Difference of Aperf/Mperf/tsc count between last and current sample * @prev: Last Aperf/Mperf/tsc count value read from register - * @freq: current cpu frequency value + * @freq: current cpu frequency value (in khz) * @boost_supported: check whether the Processor or SBIOS supports boost mode * @hw_prefcore: check whether HW supports preferred core featue. * Only when hw_prefcore and early prefcore param are true, From ba0d96f88ebbd6a197cb64d1aa9251c01a7a00e5 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Thu, 25 Apr 2024 16:07:53 +0800 Subject: [PATCH 004/188] cpufreq: amd-pstate: Unify computation of {max,min,nominal,lowest_nonlinear}_freq commit 5547c0ebfc2efdab6ee93a7fd4d9c411ad87013e upstream. Currently the amd_get_{min, max, nominal, lowest_nonlinear}_freq() helpers computes the values of min_freq, max_freq, nominal_freq and lowest_nominal_freq respectively afresh from cppc_get_perf_caps(). This is not necessary as there are fields in cpudata to cache these values. To simplify this, add a single helper function named amd_pstate_init_freq() which computes all these frequencies at once, and caches it in cpudata. Use the cached values everywhere else in the code. Acked-by: Huang Rui Reviewed-by: Li Meng Tested-by: Dhananjay Ugwekar Co-developed-by: Gautham R. Shenoy Signed-off-by: Gautham R. Shenoy Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki Signed-off-by: Abhishek Rajput Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 126 ++++++++++++++++------------------- 1 file changed, 59 insertions(+), 67 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 9401f2ac03d68..f55769566a766 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -630,74 +630,22 @@ static void amd_pstate_adjust_perf(unsigned int cpu, static int amd_get_min_freq(struct amd_cpudata *cpudata) { - struct cppc_perf_caps cppc_perf; - - int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); - if (ret) - return ret; - - /* Switch to khz */ - return cppc_perf.lowest_freq * 1000; + return READ_ONCE(cpudata->min_freq); } static int amd_get_max_freq(struct amd_cpudata *cpudata) { - struct cppc_perf_caps cppc_perf; - u32 max_perf, max_freq, nominal_freq, nominal_perf; - u64 boost_ratio; - - int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); - if (ret) - return ret; - - nominal_freq = cppc_perf.nominal_freq; - nominal_perf = READ_ONCE(cpudata->nominal_perf); - max_perf = READ_ONCE(cpudata->highest_perf); - - boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, - nominal_perf); - - max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; - - /* Switch to khz */ - return max_freq * 1000; + return READ_ONCE(cpudata->max_freq); } static int amd_get_nominal_freq(struct amd_cpudata *cpudata) { - struct cppc_perf_caps cppc_perf; - - int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); - if (ret) - return ret; - - /* Switch to khz */ - return cppc_perf.nominal_freq * 1000; + return READ_ONCE(cpudata->nominal_freq); } static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) { - struct cppc_perf_caps cppc_perf; - u32 lowest_nonlinear_freq, lowest_nonlinear_perf, - nominal_freq, nominal_perf; - u64 lowest_nonlinear_ratio; - - int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); - if (ret) - return ret; - - nominal_freq = cppc_perf.nominal_freq; - nominal_perf = READ_ONCE(cpudata->nominal_perf); - - lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; - - lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, - nominal_perf); - - lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; - - /* Switch to khz */ - return lowest_nonlinear_freq * 1000; + return READ_ONCE(cpudata->lowest_nonlinear_freq); } static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) @@ -852,6 +800,53 @@ static void amd_pstate_update_limits(unsigned int cpu) mutex_unlock(&amd_pstate_driver_lock); } +/** + * amd_pstate_init_freq: Initialize the max_freq, min_freq, + * nominal_freq and lowest_nonlinear_freq for + * the @cpudata object. + * + * Requires: highest_perf, lowest_perf, nominal_perf and + * lowest_nonlinear_perf members of @cpudata to be + * initialized. + * + * Returns 0 on success, non-zero value on failure. + */ +static int amd_pstate_init_freq(struct amd_cpudata *cpudata) +{ + int ret; + u32 min_freq; + u32 highest_perf, max_freq; + u32 nominal_perf, nominal_freq; + u32 lowest_nonlinear_perf, lowest_nonlinear_freq; + u32 boost_ratio, lowest_nonlinear_ratio; + struct cppc_perf_caps cppc_perf; + + + ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); + if (ret) + return ret; + + min_freq = cppc_perf.lowest_freq * 1000; + nominal_freq = cppc_perf.nominal_freq; + nominal_perf = READ_ONCE(cpudata->nominal_perf); + + highest_perf = READ_ONCE(cpudata->highest_perf); + boost_ratio = div_u64(highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); + max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT) * 1000; + + lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); + lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, + nominal_perf); + lowest_nonlinear_freq = (nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT) * 1000; + + WRITE_ONCE(cpudata->min_freq, min_freq); + WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); + WRITE_ONCE(cpudata->nominal_freq, nominal_freq); + WRITE_ONCE(cpudata->max_freq, max_freq); + + return 0; +} + static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; @@ -879,6 +874,10 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) if (ret) goto free_cpudata1; + ret = amd_pstate_init_freq(cpudata); + if (ret) + goto free_cpudata1; + min_freq = amd_get_min_freq(cpudata); max_freq = amd_get_max_freq(cpudata); nominal_freq = amd_get_nominal_freq(cpudata); @@ -920,13 +919,8 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) goto free_cpudata2; } - /* Initial processor data capability frequencies */ - cpudata->max_freq = max_freq; - cpudata->min_freq = min_freq; cpudata->max_limit_freq = max_freq; cpudata->min_limit_freq = min_freq; - cpudata->nominal_freq = nominal_freq; - cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; policy->driver_data = cpudata; @@ -1351,6 +1345,10 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (ret) goto free_cpudata1; + ret = amd_pstate_init_freq(cpudata); + if (ret) + goto free_cpudata1; + min_freq = amd_get_min_freq(cpudata); max_freq = amd_get_max_freq(cpudata); nominal_freq = amd_get_nominal_freq(cpudata); @@ -1367,12 +1365,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) /* It will be updated by governor */ policy->cur = policy->cpuinfo.min_freq; - /* Initial processor data capability frequencies */ - cpudata->max_freq = max_freq; - cpudata->min_freq = min_freq; - cpudata->nominal_freq = nominal_freq; - cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; - policy->driver_data = cpudata; cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); From eeec5c5ea2259fd98f2fa80bcfb46997cd75f404 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 25 Apr 2024 16:07:54 +0800 Subject: [PATCH 005/188] cpufreq: amd-pstate: Remove amd_get_{min,max,nominal,lowest_nonlinear}_freq() commit 3cbbe8871a2fb8f454e740f3e04ff2e29b573abe upstream. amd_get_{min,max,nominal,lowest_nonlinear}_freq() functions merely return cpudata->{min,max,nominal,lowest_nonlinear}_freq values. There is no loss in readability in replacing their invocations by accesses to the corresponding members of cpudata. Do so and remove these helper functions. Acked-by: Huang Rui Reviewed-by: Li Meng Tested-by: Dhananjay Ugwekar Signed-off-by: Gautham R. Shenoy Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki Signed-off-by: Abhishek Rajput Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 40 +++++++++--------------------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index f55769566a766..09655f0e62f1a 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -628,26 +628,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, cpufreq_cpu_put(policy); } -static int amd_get_min_freq(struct amd_cpudata *cpudata) -{ - return READ_ONCE(cpudata->min_freq); -} - -static int amd_get_max_freq(struct amd_cpudata *cpudata) -{ - return READ_ONCE(cpudata->max_freq); -} - -static int amd_get_nominal_freq(struct amd_cpudata *cpudata) -{ - return READ_ONCE(cpudata->nominal_freq); -} - -static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) -{ - return READ_ONCE(cpudata->lowest_nonlinear_freq); -} - static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) { struct amd_cpudata *cpudata = policy->driver_data; @@ -878,10 +858,10 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) if (ret) goto free_cpudata1; - min_freq = amd_get_min_freq(cpudata); - max_freq = amd_get_max_freq(cpudata); - nominal_freq = amd_get_nominal_freq(cpudata); - lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); + min_freq = READ_ONCE(cpudata->min_freq); + max_freq = READ_ONCE(cpudata->max_freq); + nominal_freq = READ_ONCE(cpudata->nominal_freq); + lowest_nonlinear_freq = READ_ONCE(cpudata->lowest_nonlinear_freq); if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", @@ -984,7 +964,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, int max_freq; struct amd_cpudata *cpudata = policy->driver_data; - max_freq = amd_get_max_freq(cpudata); + max_freq = READ_ONCE(cpudata->max_freq); if (max_freq < 0) return max_freq; @@ -997,7 +977,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli int freq; struct amd_cpudata *cpudata = policy->driver_data; - freq = amd_get_lowest_nonlinear_freq(cpudata); + freq = READ_ONCE(cpudata->lowest_nonlinear_freq); if (freq < 0) return freq; @@ -1349,10 +1329,10 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (ret) goto free_cpudata1; - min_freq = amd_get_min_freq(cpudata); - max_freq = amd_get_max_freq(cpudata); - nominal_freq = amd_get_nominal_freq(cpudata); - lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); + min_freq = READ_ONCE(cpudata->min_freq); + max_freq = READ_ONCE(cpudata->max_freq); + nominal_freq = READ_ONCE(cpudata->nominal_freq); + lowest_nonlinear_freq = READ_ONCE(cpudata->lowest_nonlinear_freq); if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", min_freq, max_freq); From b260a6cba1c97852600c9fea2d4f2fb88421e519 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Thu, 25 Apr 2024 16:07:55 +0800 Subject: [PATCH 006/188] cpufreq: amd-pstate: Bail out if min/max/nominal_freq is 0 commit 2ddb8a3946d4d02115b2dca53c5493ff00536002 upstream. The amd-pstate driver cannot work when the min_freq, nominal_freq or the max_freq is zero. When this happens it is prudent to error out early on rather than waiting failing at the time of the governor initialization. Acked-by: Huang Rui Reviewed-by: Gautham R. Shenoy Tested-by: Dhananjay Ugwekar Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki Signed-off-by: Abhishek Rajput Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 09655f0e62f1a..0b846e501e8fb 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -863,9 +863,11 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) nominal_freq = READ_ONCE(cpudata->nominal_freq); lowest_nonlinear_freq = READ_ONCE(cpudata->lowest_nonlinear_freq); - if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { - dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", - min_freq, max_freq); + if (min_freq <= 0 || max_freq <= 0 || + nominal_freq <= 0 || min_freq > max_freq) { + dev_err(dev, + "min_freq(%d) or max_freq(%d) or nominal_freq (%d) value is incorrect, check _CPC in ACPI tables\n", + min_freq, max_freq, nominal_freq); ret = -EINVAL; goto free_cpudata1; } @@ -1333,9 +1335,11 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) max_freq = READ_ONCE(cpudata->max_freq); nominal_freq = READ_ONCE(cpudata->nominal_freq); lowest_nonlinear_freq = READ_ONCE(cpudata->lowest_nonlinear_freq); - if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { - dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", - min_freq, max_freq); + if (min_freq <= 0 || max_freq <= 0 || + nominal_freq <= 0 || min_freq > max_freq) { + dev_err(dev, + "min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect, check _CPC in ACPI tables\n", + min_freq, max_freq, nominal_freq); ret = -EINVAL; goto free_cpudata1; } From 006c666ed280355b9dfbf864f46d7d9e4e869b64 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Thu, 25 Apr 2024 16:07:56 +0800 Subject: [PATCH 007/188] cpufreq: amd-pstate: get transition delay and latency value from ACPI tables commit 069a2bb8c48c43176f2f0e6cae5efe2f39f6bdf2 upstream. Make pstate driver initially retrieve the P-state transition delay and latency values from the BIOS ACPI tables which has more reasonable delay and latency values according to the platform design and requirements. Previously there values were hardcoded at specific value which may have conflicted with platform and it might not reflect the most accurate or optimized setting for the processor. [054h 0084 8] Preserve Mask : FFFFFFFF00000000 [05Ch 0092 8] Write Mask : 0000000000000001 [064h 0100 4] Command Latency : 00000FA0 [068h 0104 4] Maximum Access Rate : 0000EA60 [06Ch 0108 2] Minimum Turnaround Time : 0000 Reviewed-by: Gautham R. Shenoy Reviewed-by: Mario Limonciello Tested-by: Dhananjay Ugwekar Signed-off-by: Perry Yuan Acked-by: Huang Rui Signed-off-by: Rafael J. Wysocki Signed-off-by: Abhishek Rajput Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 0b846e501e8fb..2283e9bf33c4a 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -780,6 +780,36 @@ static void amd_pstate_update_limits(unsigned int cpu) mutex_unlock(&amd_pstate_driver_lock); } +/** + * Get pstate transition delay time from ACPI tables that firmware set + * instead of using hardcode value directly. + */ +static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) +{ + u32 transition_delay_ns; + + transition_delay_ns = cppc_get_transition_latency(cpu); + if (transition_delay_ns == CPUFREQ_ETERNAL) + return AMD_PSTATE_TRANSITION_DELAY; + + return transition_delay_ns / NSEC_PER_USEC; +} + +/** + * Get pstate transition latency value from ACPI tables that firmware + * set instead of using hardcode value directly. + */ +static u32 amd_pstate_get_transition_latency(unsigned int cpu) +{ + u32 transition_latency; + + transition_latency = cppc_get_transition_latency(cpu); + if (transition_latency == CPUFREQ_ETERNAL) + return AMD_PSTATE_TRANSITION_LATENCY; + + return transition_latency; +} + /** * amd_pstate_init_freq: Initialize the max_freq, min_freq, * nominal_freq and lowest_nonlinear_freq for @@ -872,8 +902,8 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) goto free_cpudata1; } - policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; - policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; + policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); + policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); policy->min = min_freq; policy->max = max_freq; From 10c017046434631ed00cb3f40019da2819b260ec Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Thu, 25 Apr 2024 16:07:58 +0800 Subject: [PATCH 008/188] cpufreq: amd-pstate: Add quirk for the pstate CPPC capabilities missing commit eb8b6c36820214df96e7e86d8614d93f6b028f28 upstream. Add quirks table to get CPPC capabilities issue fixed by providing correct perf or frequency values while driver loading. If CPPC capabilities are not defined in the ACPI tables or wrongly defined by platform firmware, it needs to use quick to get those issues fixed with correct workaround values to make pstate driver can be loaded even though there are CPPC capabilities errors. The workaround will match the broken BIOS which lack of CPPC capabilities nominal_freq and lowest_freq definition in the ACPI table. $ cat /sys/devices/system/cpu/cpu0/acpi_cppc/lowest_freq 0 $ cat /sys/devices/system/cpu/cpu0/acpi_cppc/nominal_freq 0 Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Tested-by: Dhananjay Ugwekar Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki Signed-off-by: Abhishek Rajput Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 53 ++++++++++++++++++++++++++++++++++-- include/linux/amd-pstate.h | 6 ++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 2283e9bf33c4a..a5329d92cf290 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -68,6 +68,7 @@ static struct cpufreq_driver amd_pstate_epp_driver; static int cppc_state = AMD_PSTATE_UNDEFINED; static bool cppc_enabled; static bool amd_pstate_prefcore = true; +static struct quirk_entry *quirks; /* * AMD Energy Preference Performance (EPP) @@ -112,6 +113,41 @@ static unsigned int epp_values[] = { typedef int (*cppc_mode_transition_fn)(int); +static struct quirk_entry quirk_amd_7k62 = { + .nominal_freq = 2600, + .lowest_freq = 550, +}; + +static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) +{ + /** + * match the broken bios for family 17h processor support CPPC V2 + * broken BIOS lack of nominal_freq and lowest_freq capabilities + * definition in ACPI tables + */ + if (boot_cpu_has(X86_FEATURE_ZEN2)) { + quirks = dmi->driver_data; + pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); + return 1; + } + + return 0; +} + +static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { + { + .callback = dmi_matched_7k62_bios_bug, + .ident = "AMD EPYC 7K62", + .matches = { + DMI_MATCH(DMI_BIOS_VERSION, "5.14"), + DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), + }, + .driver_data = &quirk_amd_7k62, + }, + {} +}; +MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); + static inline int get_mode_idx_from_str(const char *str, size_t size) { int i; @@ -836,8 +872,16 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) if (ret) return ret; - min_freq = cppc_perf.lowest_freq * 1000; - nominal_freq = cppc_perf.nominal_freq; + if (quirks && quirks->lowest_freq) + min_freq = quirks->lowest_freq * 1000; + else + min_freq = cppc_perf.lowest_freq * 1000; + + if (quirks && quirks->nominal_freq) + nominal_freq = quirks->nominal_freq ; + else + nominal_freq = cppc_perf.nominal_freq; + nominal_perf = READ_ONCE(cpudata->nominal_perf); highest_perf = READ_ONCE(cpudata->highest_perf); @@ -1703,6 +1747,11 @@ static int __init amd_pstate_init(void) if (cpufreq_get_current_driver()) return -EEXIST; + quirks = NULL; + + /* check if this machine need CPPC quirks */ + dmi_check_system(amd_pstate_quirks_table); + switch (cppc_state) { case AMD_PSTATE_UNDEFINED: /* Disable on the following configs by default: diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h index ec0b0fa3e9bb4..d58fc022ec466 100644 --- a/include/linux/amd-pstate.h +++ b/include/linux/amd-pstate.h @@ -128,4 +128,10 @@ static const char * const amd_pstate_mode_string[] = { [AMD_PSTATE_GUIDED] = "guided", NULL, }; + +struct quirk_entry { + u32 nominal_freq; + u32 lowest_freq; +}; + #endif /* _LINUX_AMD_PSTATE_H */ From f49c49f4100509962ccd9b3a630ea9cfe126414b Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 30 Apr 2024 15:48:56 +0800 Subject: [PATCH 009/188] cpufreq: amd-pstate: fix code format problems commit 5131a3ca3518d726cb535543441a5f195b8b0299 upstream. get some code format problems fixed in the amd-pstate driver. Changes Made: - Fixed incorrect comment format in the functions. - Removed unnecessary blank line. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202404271148.HK9yHBlB-lkp@intel.com/ Signed-off-by: Perry Yuan Reviewed-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki Signed-off-by: Abhishek Rajput Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index a5329d92cf290..6231a59f45b9e 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -816,7 +816,7 @@ static void amd_pstate_update_limits(unsigned int cpu) mutex_unlock(&amd_pstate_driver_lock); } -/** +/* * Get pstate transition delay time from ACPI tables that firmware set * instead of using hardcode value directly. */ @@ -831,7 +831,7 @@ static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) return transition_delay_ns / NSEC_PER_USEC; } -/** +/* * Get pstate transition latency value from ACPI tables that firmware * set instead of using hardcode value directly. */ @@ -846,7 +846,7 @@ static u32 amd_pstate_get_transition_latency(unsigned int cpu) return transition_latency; } -/** +/* * amd_pstate_init_freq: Initialize the max_freq, min_freq, * nominal_freq and lowest_nonlinear_freq for * the @cpudata object. @@ -867,7 +867,6 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) u32 boost_ratio, lowest_nonlinear_ratio; struct cppc_perf_caps cppc_perf; - ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); if (ret) return ret; From 4b85d180150f219925a211fd0f3f321ea73447b7 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 30 Apr 2024 15:48:57 +0800 Subject: [PATCH 010/188] cpufreq: amd-pstate: remove unused variable lowest_nonlinear_freq commit 5c3fd1edaa8b4c093e877f6354b3745178015070 upstream. removed the unused variable `lowest_nonlinear_freq` for build warning. This variable was defined and assigned a value in the previous code, but it was not used in the subsequent code. Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202404271038.em6nJjzy-lkp@intel.com/ Signed-off-by: Perry Yuan Signed-off-by: Rafael J. Wysocki Signed-off-by: Abhishek Rajput Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 6231a59f45b9e..acc68c30a6c78 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -902,7 +902,7 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { - int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; + int min_freq, max_freq, nominal_freq, ret; struct device *dev; struct amd_cpudata *cpudata; @@ -934,7 +934,6 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) min_freq = READ_ONCE(cpudata->min_freq); max_freq = READ_ONCE(cpudata->max_freq); nominal_freq = READ_ONCE(cpudata->nominal_freq); - lowest_nonlinear_freq = READ_ONCE(cpudata->lowest_nonlinear_freq); if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { @@ -1373,7 +1372,7 @@ static bool amd_pstate_acpi_pm_profile_undefined(void) static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) { - int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; + int min_freq, max_freq, nominal_freq, ret; struct amd_cpudata *cpudata; struct device *dev; u64 value; @@ -1407,7 +1406,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) min_freq = READ_ONCE(cpudata->min_freq); max_freq = READ_ONCE(cpudata->max_freq); nominal_freq = READ_ONCE(cpudata->nominal_freq); - lowest_nonlinear_freq = READ_ONCE(cpudata->lowest_nonlinear_freq); if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { dev_err(dev, From 806ebfbfb2ce6891530ce3829f3757d29799376a Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 28 May 2024 14:09:23 +0200 Subject: [PATCH 011/188] cpufreq: amd-pstate: remove global header file commit 779b8a14afde110dd3502566be907289eba72447 upstream. When extra warnings are enabled, gcc points out a global variable definition in a header: In file included from drivers/cpufreq/amd-pstate-ut.c:29: include/linux/amd-pstate.h:123:27: error: 'amd_pstate_mode_string' defined but not used [-Werror=unused-const-variable=] 123 | static const char * const amd_pstate_mode_string[] = { | ^~~~~~~~~~~~~~~~~~~~~~ This header is only included from two files in the same directory, and one of them uses only a single definition from it, so clean it up by moving most of the contents into the driver that uses them, and making shared bits a local header file. Fixes: 36c5014e5460 ("cpufreq: amd-pstate: optimize driver working mode selection in amd_pstate_param()") Signed-off-by: Arnd Bergmann Acked-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- MAINTAINERS | 1 - drivers/cpufreq/amd-pstate-ut.c | 3 +- drivers/cpufreq/amd-pstate.c | 34 ++++++++++++++++++- .../linux => drivers/cpufreq}/amd-pstate.h | 33 ------------------ 4 files changed, 35 insertions(+), 36 deletions(-) rename {include/linux => drivers/cpufreq}/amd-pstate.h (82%) diff --git a/MAINTAINERS b/MAINTAINERS index a26a500e886b5..2007de0d42b01 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1054,7 +1054,6 @@ L: linux-pm@vger.kernel.org S: Supported F: Documentation/admin-guide/pm/amd-pstate.rst F: drivers/cpufreq/amd-pstate* -F: include/linux/amd-pstate.h F: tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py AMD PTDMA DRIVER diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index f04ae67dda372..fc275d41d51e9 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -26,10 +26,11 @@ #include #include #include -#include #include +#include "amd-pstate.h" + /* * Abbreviations: * amd_pstate_ut: used as a shortform for AMD P-State unit test. diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index acc68c30a6c78..7bfc244d0d75b 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include @@ -46,6 +45,8 @@ #include #include #include + +#include "amd-pstate.h" #include "amd-pstate-trace.h" #define AMD_PSTATE_TRANSITION_LATENCY 20000 @@ -53,6 +54,37 @@ #define CPPC_HIGHEST_PERF_PERFORMANCE 196 #define CPPC_HIGHEST_PERF_DEFAULT 166 +#define AMD_CPPC_EPP_PERFORMANCE 0x00 +#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 +#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF +#define AMD_CPPC_EPP_POWERSAVE 0xFF + +/* + * enum amd_pstate_mode - driver working mode of amd pstate + */ +enum amd_pstate_mode { + AMD_PSTATE_UNDEFINED = 0, + AMD_PSTATE_DISABLE, + AMD_PSTATE_PASSIVE, + AMD_PSTATE_ACTIVE, + AMD_PSTATE_GUIDED, + AMD_PSTATE_MAX, +}; + +static const char * const amd_pstate_mode_string[] = { + [AMD_PSTATE_UNDEFINED] = "undefined", + [AMD_PSTATE_DISABLE] = "disable", + [AMD_PSTATE_PASSIVE] = "passive", + [AMD_PSTATE_ACTIVE] = "active", + [AMD_PSTATE_GUIDED] = "guided", + NULL, +}; + +struct quirk_entry { + u32 nominal_freq; + u32 lowest_freq; +}; + /* * TODO: We need more time to fine tune processors with shared memory solution * with community together. diff --git a/include/linux/amd-pstate.h b/drivers/cpufreq/amd-pstate.h similarity index 82% rename from include/linux/amd-pstate.h rename to drivers/cpufreq/amd-pstate.h index d58fc022ec466..e6a28e7f4dbf1 100644 --- a/include/linux/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -1,7 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * linux/include/linux/amd-pstate.h - * * Copyright (C) 2022 Advanced Micro Devices, Inc. * * Author: Meng Li @@ -12,11 +10,6 @@ #include -#define AMD_CPPC_EPP_PERFORMANCE 0x00 -#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80 -#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF -#define AMD_CPPC_EPP_POWERSAVE 0xFF - /********************************************************************* * AMD P-state INTERFACE * *********************************************************************/ @@ -108,30 +101,4 @@ struct amd_cpudata { bool suspended; }; -/* - * enum amd_pstate_mode - driver working mode of amd pstate - */ -enum amd_pstate_mode { - AMD_PSTATE_UNDEFINED = 0, - AMD_PSTATE_DISABLE, - AMD_PSTATE_PASSIVE, - AMD_PSTATE_ACTIVE, - AMD_PSTATE_GUIDED, - AMD_PSTATE_MAX, -}; - -static const char * const amd_pstate_mode_string[] = { - [AMD_PSTATE_UNDEFINED] = "undefined", - [AMD_PSTATE_DISABLE] = "disable", - [AMD_PSTATE_PASSIVE] = "passive", - [AMD_PSTATE_ACTIVE] = "active", - [AMD_PSTATE_GUIDED] = "guided", - NULL, -}; - -struct quirk_entry { - u32 nominal_freq; - u32 lowest_freq; -}; - #endif /* _LINUX_AMD_PSTATE_H */ From fd7736b97afeb8db02a0efdf7c85c28b72b52d6b Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Wed, 12 Jun 2024 09:20:49 -0500 Subject: [PATCH 012/188] cpufreq: amd-pstate: Allow users to write 'default' EPP string commit fc6e0837264a8b2504b6160e63ec92eb012540f3 upstream. The EPP string for 'default' represents what the firmware had configured as the default EPP value but once a user changes EPP to another string they can't reset it back to 'default'. Cache the firmware EPP value and allow the user to write 'default' using this value. Reported-by: Artem S. Tashkinov Closes: https://bugzilla.kernel.org/show_bug.cgi?id=217931#c61 Reviewed-by: Perry Yuan Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 8 +++----- drivers/cpufreq/amd-pstate.h | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 7bfc244d0d75b..d9cbf86db4521 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -304,10 +304,8 @@ static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, int epp = -EINVAL; int ret; - if (!pref_index) { - pr_debug("EPP pref_index is invalid\n"); - return -EINVAL; - } + if (!pref_index) + epp = cpudata->epp_default; if (epp == -EINVAL) epp = epp_values[pref_index]; @@ -1454,7 +1452,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->driver_data = cpudata; - cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); + cpudata->epp_cached = cpudata->epp_default = amd_pstate_get_epp(cpudata, 0); policy->min = policy->cpuinfo.min_freq; policy->max = policy->cpuinfo.max_freq; diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index e6a28e7f4dbf1..f80b33fa5d43a 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -99,6 +99,7 @@ struct amd_cpudata { u32 policy; u64 cppc_cap1_cached; bool suspended; + s16 epp_default; }; #endif /* _LINUX_AMD_PSTATE_H */ From a4c6313947044c48b991a0dba681abd8f29cbd28 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Wed, 19 Jun 2024 23:40:12 +0800 Subject: [PATCH 013/188] cpufreq: amd-pstate: optimize the initial frequency values verification commit 8f8b42c1fcc939a73b547b172a9ffcb65ef4bf47 upstream. To enhance the debugging capability of the driver loading failure for broken CPPC ACPI tables, it can optimize the expression by moving the verification of `min_freq`, `nominal_freq`, and other dependency values to the `amd_pstate_init_freq()` function where they are initialized. If any of these values are incorrect, the `amd-pstate` driver will not be registered. By ensuring that these values are correct before they are used, it will facilitate the debugging process when encountering driver loading failures due to faulty CPPC ACPI tables from BIOS Signed-off-by: Perry Yuan Acked-by: Gautham R. Shenoy Acked-by: Mario Limonciello Link: https://lore.kernel.org/r/f9793f8451c1832e34cc9dc35f89c653b39cfe38.1718811234.git.perry.yuan@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index d9cbf86db4521..3c798574da5c2 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -927,6 +927,24 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->nominal_freq, nominal_freq); WRITE_ONCE(cpudata->max_freq, max_freq); + /** + * Below values need to be initialized correctly, otherwise driver will fail to load + * max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf + * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] + * Check _CPC in ACPI table objects if any values are incorrect + */ + if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { + pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", + min_freq, max_freq, nominal_freq * 1000); + return -EINVAL; + } + + if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq * 1000) { + pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", + lowest_nonlinear_freq, min_freq, nominal_freq * 1000); + return -EINVAL; + } + return 0; } @@ -965,15 +983,6 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) max_freq = READ_ONCE(cpudata->max_freq); nominal_freq = READ_ONCE(cpudata->nominal_freq); - if (min_freq <= 0 || max_freq <= 0 || - nominal_freq <= 0 || min_freq > max_freq) { - dev_err(dev, - "min_freq(%d) or max_freq(%d) or nominal_freq (%d) value is incorrect, check _CPC in ACPI tables\n", - min_freq, max_freq, nominal_freq); - ret = -EINVAL; - goto free_cpudata1; - } - policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); @@ -1436,14 +1445,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) min_freq = READ_ONCE(cpudata->min_freq); max_freq = READ_ONCE(cpudata->max_freq); nominal_freq = READ_ONCE(cpudata->nominal_freq); - if (min_freq <= 0 || max_freq <= 0 || - nominal_freq <= 0 || min_freq > max_freq) { - dev_err(dev, - "min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect, check _CPC in ACPI tables\n", - min_freq, max_freq, nominal_freq); - ret = -EINVAL; - goto free_cpudata1; - } policy->cpuinfo.min_freq = min_freq; policy->cpuinfo.max_freq = max_freq; From 7e3ee27240148a0833178721abea141f396554bf Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Wed, 19 Jun 2024 23:40:13 +0800 Subject: [PATCH 014/188] cpufreq: amd-pstate: remove unused variable nominal_freq commit 7bf7f22906671995b798b39f8c3cb62405ea03b2 upstream. removed the unused variable `nominal_freq` for build warning. This variable was defined and assigned a value in the previous code, but it was not used in the subsequent code. Closes: https://lore.kernel.org/oe-kbuild-all/202405080431.BPU6Yg9s-lkp@intel.com/ Reported-by: kernel test robot Signed-off-by: Perry Yuan Reviewed-by: Gautham R. Shenoy Acked-by: Mario Limonciello Link: https://lore.kernel.org/r/b7ef41557f71d40d098393ddb27f0fe1f23648ae.1718811234.git.perry.yuan@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 3c798574da5c2..226ddc7107412 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -950,7 +950,7 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { - int min_freq, max_freq, nominal_freq, ret; + int min_freq, max_freq, ret; struct device *dev; struct amd_cpudata *cpudata; @@ -981,7 +981,6 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) min_freq = READ_ONCE(cpudata->min_freq); max_freq = READ_ONCE(cpudata->max_freq); - nominal_freq = READ_ONCE(cpudata->nominal_freq); policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); @@ -1411,7 +1410,7 @@ static bool amd_pstate_acpi_pm_profile_undefined(void) static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) { - int min_freq, max_freq, nominal_freq, ret; + int min_freq, max_freq, ret; struct amd_cpudata *cpudata; struct device *dev; u64 value; @@ -1444,7 +1443,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) min_freq = READ_ONCE(cpudata->min_freq); max_freq = READ_ONCE(cpudata->max_freq); - nominal_freq = READ_ONCE(cpudata->nominal_freq); policy->cpuinfo.min_freq = min_freq; policy->cpuinfo.max_freq = max_freq; From 71a523a28eac99451621af3e0fa48f1e064c34b2 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Wed, 19 Jun 2024 23:40:14 +0800 Subject: [PATCH 015/188] cpufreq: amd-pstate: show CPPC debug message if CPPC is not supported commit cb817ec6673b7c2faed2141e7f7ae5d7052b2442 upstream. Add CPU ID checking in case the driver attempt to load on systems where CPPC functionality is unavailable. And the warning message will not be shown if CPPC is not supported. It will also print debug message if the CPU has no CPPC support that helps to debug the driver loading failure issue. Reported-by: Paul Menzel Closes: https://lore.kernel.org/linux-pm/CYYPR12MB8655D32EA18574C9497E888A9C122@CYYPR12MB8655.namprd12.prod.outlook.com/T/#t Signed-off-by: Perry Yuan Reviewed-by: Gautham R. Shenoy Acked-by: Mario Limonciello Link: https://lore.kernel.org/r/437dbd581a4119465581330081d9b1e289482ba2.1718811234.git.perry.yuan@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 226ddc7107412..23c19812c4bf9 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1756,6 +1756,20 @@ static int __init amd_pstate_set_driver(int mode_idx) return -EINVAL; } +/** + * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. + * show the debug message that helps to check if the CPU has CPPC support for loading issue. + */ +static bool amd_cppc_supported(void) +{ + if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) { + pr_debug_once("CPPC feature is not supported by the processor\n"); + return false; + } + + return true; +} + static int __init amd_pstate_init(void) { struct device *dev_root; @@ -1764,6 +1778,11 @@ static int __init amd_pstate_init(void) if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) return -ENODEV; + /* show debug message only if CPPC is not supported */ + if (!amd_cppc_supported()) + return -EOPNOTSUPP; + + /* show warning message when BIOS broken or ACPI disabled */ if (!acpi_cpc_valid()) { pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); return -ENODEV; From aa254b2566cb8396adae40435a2196af2deb2ddf Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Wed, 19 Jun 2024 23:40:15 +0800 Subject: [PATCH 016/188] cpufreq: amd-pstate: add debug message while CPPC is supported and disabled by SBIOS commit bff7d13c190ad98cf4f877189b022c75df4cb383 upstream. If CPPC feature is supported by the CPU however the CPUID flag bit is not set by SBIOS, the `amd_pstate` will be failed to load while system booting. So adding one more debug message to inform user to check the SBIOS setting, The change also can help maintainers to debug why amd_pstate driver failed to be loaded at system booting if the processor support CPPC. Closes: https://bugzilla.kernel.org/show_bug.cgi?id=218686 Signed-off-by: Perry Yuan Reviewed-by: Gautham R. Shenoy Acked-by: Mario Limonciello Link: https://lore.kernel.org/r/42c953616ac121bd1e5c329e83d015a02e6b32c7.1718811234.git.perry.yuan@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 23c19812c4bf9..679d4d4e753b7 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1762,11 +1762,37 @@ static int __init amd_pstate_set_driver(int mode_idx) */ static bool amd_cppc_supported(void) { + struct cpuinfo_x86 *c = &cpu_data(0); + bool warn = false; + if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) { pr_debug_once("CPPC feature is not supported by the processor\n"); return false; } + /* + * If the CPPC feature is disabled in the BIOS for processors that support MSR-based CPPC, + * the AMD Pstate driver may not function correctly. + * Check the CPPC flag and display a warning message if the platform supports CPPC. + * Note: below checking code will not abort the driver registeration process because of + * the code is added for debugging purposes. + */ + if (!cpu_feature_enabled(X86_FEATURE_CPPC)) { + if (cpu_feature_enabled(X86_FEATURE_ZEN1) || cpu_feature_enabled(X86_FEATURE_ZEN2)) { + if (c->x86_model > 0x60 && c->x86_model < 0xaf) + warn = true; + } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || cpu_feature_enabled(X86_FEATURE_ZEN4)) { + if ((c->x86_model > 0x10 && c->x86_model < 0x1F) || + (c->x86_model > 0x40 && c->x86_model < 0xaf)) + warn = true; + } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) { + warn = true; + } + } + + if (warn) + pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n" + "Please enable it if your BIOS has the CPPC option.\n"); return true; } From f9f9c0b0f04aa5a89333e943f381b52ff7e9133b Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Wed, 19 Jun 2024 23:40:17 +0800 Subject: [PATCH 017/188] cpufreq: amd-pstate: switch boot_cpu_has() to cpu_feature_enabled() commit c9fdaba8369ea6ca01728ed9c25c6ca1b90d3355 upstream. replace the usage of the deprecated boot_cpu_has() function with the modern cpu_feature_enabled() function. The switch to cpu_feature_enabled() ensures compatibility with the latest CPU feature detection mechanisms and improves code maintainability. Acked-by: Mario Limonciello Suggested-by: Borislav Petkov (AMD) Signed-off-by: Perry Yuan Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/f1567593ac5e1d38343067e9c681a8c4b0707038.1718811234.git.perry.yuan@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 679d4d4e753b7..93b34b9b82a28 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -157,7 +157,7 @@ static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) * broken BIOS lack of nominal_freq and lowest_freq capabilities * definition in ACPI tables */ - if (boot_cpu_has(X86_FEATURE_ZEN2)) { + if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { quirks = dmi->driver_data; pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); return 1; @@ -199,7 +199,7 @@ static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) u64 epp; int ret; - if (boot_cpu_has(X86_FEATURE_CPPC)) { + if (cpu_feature_enabled(X86_FEATURE_CPPC)) { if (!cppc_req_cached) { epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req_cached); @@ -272,7 +272,7 @@ static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) int ret; struct cppc_perf_ctrls perf_ctrls; - if (boot_cpu_has(X86_FEATURE_CPPC)) { + if (cpu_feature_enabled(X86_FEATURE_CPPC)) { u64 value = READ_ONCE(cpudata->cppc_req_cached); value &= ~GENMASK_ULL(31, 24); @@ -759,7 +759,7 @@ static int amd_pstate_get_highest_perf(int cpu, u32 *highest_perf) { int ret; - if (boot_cpu_has(X86_FEATURE_CPPC)) { + if (cpu_feature_enabled(X86_FEATURE_CPPC)) { u64 cap1; ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); @@ -994,7 +994,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) /* It will be updated by governor */ policy->cur = policy->cpuinfo.min_freq; - if (boot_cpu_has(X86_FEATURE_CPPC)) + if (cpu_feature_enabled(X86_FEATURE_CPPC)) policy->fast_switch_possible = true; ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], @@ -1237,7 +1237,7 @@ static int amd_pstate_change_mode_without_dvr_change(int mode) cppc_state = mode; - if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) + if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) return 0; for_each_online_cpu(cpu) { @@ -1466,7 +1466,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) else policy->policy = CPUFREQ_POLICY_POWERSAVE; - if (boot_cpu_has(X86_FEATURE_CPPC)) { + if (cpu_feature_enabled(X86_FEATURE_CPPC)) { ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); if (ret) return ret; @@ -1556,7 +1556,7 @@ static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy) epp = 0; /* Set initial EPP value */ - if (boot_cpu_has(X86_FEATURE_CPPC)) { + if (cpu_feature_enabled(X86_FEATURE_CPPC)) { value &= ~GENMASK_ULL(31, 24); value |= (u64)epp << 24; } @@ -1595,7 +1595,7 @@ static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) value = READ_ONCE(cpudata->cppc_req_cached); max_perf = READ_ONCE(cpudata->highest_perf); - if (boot_cpu_has(X86_FEATURE_CPPC)) { + if (cpu_feature_enabled(X86_FEATURE_CPPC)) { wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); } else { perf_ctrls.max_perf = max_perf; @@ -1629,7 +1629,7 @@ static void amd_pstate_epp_offline(struct cpufreq_policy *policy) value = READ_ONCE(cpudata->cppc_req_cached); mutex_lock(&amd_pstate_limits_lock); - if (boot_cpu_has(X86_FEATURE_CPPC)) { + if (cpu_feature_enabled(X86_FEATURE_CPPC)) { cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; /* Set max perf same as min perf */ @@ -1832,7 +1832,7 @@ static int __init amd_pstate_init(void) */ if (amd_pstate_acpi_pm_profile_undefined() || amd_pstate_acpi_pm_profile_server() || - !boot_cpu_has(X86_FEATURE_CPPC)) { + !cpu_feature_enabled(X86_FEATURE_CPPC)) { pr_info("driver load is disabled, boot with specific mode to enable this\n"); return -ENODEV; } @@ -1851,7 +1851,7 @@ static int __init amd_pstate_init(void) } /* capability check */ - if (boot_cpu_has(X86_FEATURE_CPPC)) { + if (cpu_feature_enabled(X86_FEATURE_CPPC)) { pr_debug("AMD CPPC MSR based functionality is supported\n"); if (cppc_state != AMD_PSTATE_ACTIVE) current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; From 7e318813657aeac058c06a9814588e4eaff141a5 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Wed, 19 Jun 2024 23:40:18 +0800 Subject: [PATCH 018/188] cpufreq: amd-pstate: enable shared memory type CPPC by default commit 918263938c41dac46935609477672442e8fa5d9e upstream. The amd-pstate-epp driver has been implemented and resolves the performance drop issue seen in passive mode for shared memory type CPPC systems. Users who enable the active mode driver will not experience a performance drop compared to the passive mode driver. Therefore, the EPP driver should be loaded by default for shared memory type CPPC system to get better performance. Signed-off-by: Perry Yuan Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/c705507cf3ee790e544251cfd897ed11e8e57712.1718811234.git.perry.yuan@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 93b34b9b82a28..000f9a818a5fe 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -85,15 +85,6 @@ struct quirk_entry { u32 lowest_freq; }; -/* - * TODO: We need more time to fine tune processors with shared memory solution - * with community together. - * - * There are some performance drops on the CPU benchmarks which reports from - * Suse. We are co-working with them to fine tune the shared memory solution. So - * we disable it by default to go acpi-cpufreq on these processors and add a - * module parameter to be able to enable it manually for debugging. - */ static struct cpufreq_driver *current_pstate_driver; static struct cpufreq_driver amd_pstate_driver; static struct cpufreq_driver amd_pstate_epp_driver; @@ -1828,11 +1819,9 @@ static int __init amd_pstate_init(void) /* Disable on the following configs by default: * 1. Undefined platforms * 2. Server platforms - * 3. Shared memory designs */ if (amd_pstate_acpi_pm_profile_undefined() || - amd_pstate_acpi_pm_profile_server() || - !cpu_feature_enabled(X86_FEATURE_CPPC)) { + amd_pstate_acpi_pm_profile_server()) { pr_info("driver load is disabled, boot with specific mode to enable this\n"); return -ENODEV; } From adc08122d30d18683dc3b31aa6715b2cccc5685c Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Wed, 19 Jun 2024 23:40:19 +0800 Subject: [PATCH 019/188] cpufreq: amd-pstate: auto-load pstate driver by default commit 4e4f600ee750facedf6a5dc97e8ae0b627ab4573 upstream. If the `amd-pstate` driver is not loaded automatically by default, it is because the kernel command line parameter has not been added. To resolve this issue, it is necessary to call the `amd_pstate_set_driver()` function to enable the desired mode (passive/active/guided) before registering the driver instance. This ensures that the driver is loaded correctly without relying on the kernel command line parameter. When there is no parameter added to command line, Kernel config will provide the default mode to load. Meanwhile, user can add driver mode in command line which will override the kernel config default option. Reported-by: Andrei Amuraritei Closes: https://bugzilla.kernel.org/show_bug.cgi?id=218705 Signed-off-by: Perry Yuan Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/83301c4cea4f92fb19e14b23f2bac7facfd8bdbb.1718811234.git.perry.yuan@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 000f9a818a5fe..01f5f7964dc44 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1814,8 +1814,13 @@ static int __init amd_pstate_init(void) /* check if this machine need CPPC quirks */ dmi_check_system(amd_pstate_quirks_table); - switch (cppc_state) { - case AMD_PSTATE_UNDEFINED: + /* + * determine the driver mode from the command line or kernel config. + * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED. + * command line options will override the kernel config settings. + */ + + if (cppc_state == AMD_PSTATE_UNDEFINED) { /* Disable on the following configs by default: * 1. Undefined platforms * 2. Server platforms @@ -1825,15 +1830,20 @@ static int __init amd_pstate_init(void) pr_info("driver load is disabled, boot with specific mode to enable this\n"); return -ENODEV; } - ret = amd_pstate_set_driver(CONFIG_X86_AMD_PSTATE_DEFAULT_MODE); - if (ret) - return ret; - break; + /* get driver mode from kernel config option [1:4] */ + cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; + } + + switch (cppc_state) { case AMD_PSTATE_DISABLE: + pr_info("driver load is disabled, boot with specific mode to enable this\n"); return -ENODEV; case AMD_PSTATE_PASSIVE: case AMD_PSTATE_ACTIVE: case AMD_PSTATE_GUIDED: + ret = amd_pstate_set_driver(cppc_state); + if (ret) + return ret; break; default: return -EINVAL; @@ -1854,7 +1864,7 @@ static int __init amd_pstate_init(void) /* enable amd pstate feature */ ret = amd_pstate_enable(true); if (ret) { - pr_err("failed to enable with return %d\n", ret); + pr_err("failed to enable driver mode(%d)\n", cppc_state); return ret; } From a89d8736dc814b248e1b03f5d432f61726e05194 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Tue, 27 Feb 2024 15:11:33 +0800 Subject: [PATCH 020/188] cpufreq/amd-pstate: fix setting policy current frequency value commit e8f555daacd3377bf691fdda2490c0b164e00085 upstream. When scaling min/max freq values were being setted, the value of policy->cur need to update. Signed-off-by: Meng Li Acked-by: Mario Limonciello Link: https://lore.kernel.org/r/20240227071133.3405003-1-li.meng@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 01f5f7964dc44..e2fd7d094fb11 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -513,6 +513,8 @@ static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags) { + unsigned long max_freq; + struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu); u64 prev = READ_ONCE(cpudata->cppc_req_cached); u64 value = prev; @@ -522,6 +524,9 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, cpudata->max_limit_perf); des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); + max_freq = READ_ONCE(cpudata->max_limit_freq); + policy->cur = div_u64(des_perf * max_freq, max_perf); + if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { min_perf = des_perf; des_perf = 0; @@ -643,9 +648,8 @@ static void amd_pstate_adjust_perf(unsigned int cpu, unsigned long capacity) { unsigned long max_perf, min_perf, des_perf, - cap_perf, lowest_nonlinear_perf, max_freq; + cap_perf, lowest_nonlinear_perf; struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - unsigned int target_freq; struct amd_cpudata *cpudata; if (!policy) @@ -659,7 +663,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, cap_perf = READ_ONCE(cpudata->highest_perf); lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); - max_freq = READ_ONCE(cpudata->max_freq); des_perf = cap_perf; if (target_perf < capacity) @@ -677,8 +680,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, max_perf = min_perf; des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); - target_freq = div_u64(des_perf * max_freq, max_perf); - policy->cur = target_freq; amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, policy->governor->flags); @@ -1570,6 +1571,12 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) amd_pstate_epp_update_limit(policy); + /* + * policy->cur is never updated with the amd_pstate_epp driver, but it + * is used as a stale frequency value. So, keep it within limits. + */ + policy->cur = policy->min; + return 0; } From c61650d47ea0792aa1cdf9bb4339b2ed31d782e7 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sun, 23 Jun 2024 15:09:18 -0500 Subject: [PATCH 021/188] cpufreq: amd-pstate: Don't create attributes when registration fails commit bc76f57574741cc8e7d04c53f199d8722ce09413 upstream. If driver registration fails then immediately return the failure instead of continuing to register attributes. This fixes issues of falling back from amd-pstate to other drivers when cpufreq init has failed for any reason. Reported-by: alex.s.cochran@proton.me Reviewed-by: Gautham R. Shenoy Reviewed-by: Perry Yuan Link: https://lore.kernel.org/r/20240623200918.52104-1-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index e2fd7d094fb11..057d429d01fc3 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1876,8 +1876,10 @@ static int __init amd_pstate_init(void) } ret = cpufreq_register_driver(current_pstate_driver); - if (ret) + if (ret) { pr_err("failed to register with return %d\n", ret); + goto disable_driver; + } dev_root = bus_get_dev_root(&cpu_subsys); if (dev_root) { @@ -1893,6 +1895,8 @@ static int __init amd_pstate_init(void) global_attr_free: cpufreq_unregister_driver(current_pstate_driver); +disable_driver: + amd_pstate_enable(false); return ret; } device_initcall(amd_pstate_init); From 62f49bb97b0edef420a222796bd64e98f8d11ce0 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Mon, 27 May 2024 10:41:28 +0530 Subject: [PATCH 022/188] cpufreq: amd-pstate: Fix the inconsistency in max frequency units commit e4731baaf29438508197d3a8a6d4f5a8c51663f8 upstream. The nominal frequency in cpudata is maintained in MHz whereas all other frequencies are in KHz. This means we have to convert nominal frequency value to KHz before we do any interaction with other frequency values. In amd_pstate_set_boost(), this conversion from MHz to KHz is missed, fix that. Tested on a AMD Zen4 EPYC server Before: $ cat /sys/devices/system/cpu/cpufreq/policy*/scaling_max_freq | uniq 2151 $ cat /sys/devices/system/cpu/cpufreq/policy*/cpuinfo_min_freq | uniq 400000 $ cat /sys/devices/system/cpu/cpufreq/policy*/scaling_cur_freq | uniq 2151 409422 After: $ cat /sys/devices/system/cpu/cpufreq/policy*/scaling_max_freq | uniq 2151000 $ cat /sys/devices/system/cpu/cpufreq/policy*/cpuinfo_min_freq | uniq 400000 $ cat /sys/devices/system/cpu/cpufreq/policy*/scaling_cur_freq | uniq 2151000 1799527 Fixes: ec437d71db77 ("cpufreq: amd-pstate: Introduce a new AMD P-State driver to support future processors") Signed-off-by: Dhananjay Ugwekar Acked-by: Mario Limonciello Acked-by: Gautham R. Shenoy Tested-by: Peter Jung Cc: 5.17+ # 5.17+ Signed-off-by: Rafael J. Wysocki Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 057d429d01fc3..6e5a6e1a0cfe0 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -699,7 +699,7 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) if (state) policy->cpuinfo.max_freq = cpudata->max_freq; else - policy->cpuinfo.max_freq = cpudata->nominal_freq; + policy->cpuinfo.max_freq = cpudata->nominal_freq * 1000; policy->max = policy->cpuinfo.max_freq; From ee86f88bc0a49742a09ff019f8d40a5a1198516b Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 25 Jun 2024 23:27:30 -0500 Subject: [PATCH 023/188] cpufreq: acpi: move MSR_K7_HWCR_CPB_DIS_BIT into msr-index.h commit 4f460bff7b6ac53d7d8509501a9fba7292a4a11a upstream. There are some other drivers also need to use the MSR_K7_HWCR_CPB_DIS_BIT for CPB control bit, so it makes sense to move the definition to a common header file to allow other driver to use it. No intentional functional impact. Suggested-by: Gautham Ranjal Shenoy Signed-off-by: Perry Yuan Acked-by: Rafael J. Wysocki Acked-by: Huang Rui Link: https://lore.kernel.org/r/78b6c75e6cffddce3e950dd543af6ae9f8eeccc3.1718988436.git.perry.yuan@amd.com Link: https://lore.kernel.org/r/20240626042733.3747-2-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- arch/x86/include/asm/msr-index.h | 2 ++ drivers/cpufreq/acpi-cpufreq.c | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 78bf5d4bd9f5b..5f17c9354e92f 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -821,6 +821,8 @@ #define MSR_K7_HWCR_IRPERF_EN BIT_ULL(MSR_K7_HWCR_IRPERF_EN_BIT) #define MSR_K7_FID_VID_CTL 0xc0010041 #define MSR_K7_FID_VID_STATUS 0xc0010042 +#define MSR_K7_HWCR_CPB_DIS_BIT 25 +#define MSR_K7_HWCR_CPB_DIS BIT_ULL(MSR_K7_HWCR_CPB_DIS_BIT) /* K6 MSRs */ #define MSR_K6_WHCR 0xc0000082 diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index ba26fb1778a1b..ba53a1415e6cb 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -50,8 +50,6 @@ enum { #define AMD_MSR_RANGE (0x7) #define HYGON_MSR_RANGE (0x7) -#define MSR_K7_HWCR_CPB_DIS (1ULL << 25) - struct acpi_cpufreq_data { unsigned int resume; unsigned int cpu_feature; From f4f6d9ca92d22d05bdcf783b3673c8586d04c620 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 25 Jun 2024 23:27:31 -0500 Subject: [PATCH 024/188] cpufreq: amd-pstate: initialize core precision boost state commit c8c68c38b56f4036771ebe2f8d664a2e98728318 upstream. The "Core Performance Boost (CPB) feature, when enabled in the BIOS, allows the OS to control the highest performance for each individual core. The active, passive and the guided modes of the amd-pstate driver do support controlling the core frequency boost when this BIOS feature is enabled. Additionally, the amd-pstate driver provides a sysfs interface allowing the user to activate/deactivate this core performance boost feature at runtime. Add support for the set_boost callback in the active mode driver to enable boost control via the cpufreq core. This ensures a consistent boost control interface across all pstate modes, including passive mode, guided mode, and active mode. With this addition, all three pstate modes can support the same boost control interface with unique interface and global CPB control. Each CPU also supports individual boost control, allowing global CPB to change all cores' boost states simultaneously. Specific CPUs can update their boost states separately, ensuring all cores' boost states are synchronized. Cc: Oleksandr Natalenko Closes: https://bugzilla.kernel.org/show_bug.cgi?id=217931 Signed-off-by: Perry Yuan Co-developed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20240626042733.3747-3-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 117 ++++++++++++++++++++++++++++------- drivers/cpufreq/amd-pstate.h | 1 + 2 files changed, 96 insertions(+), 22 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 6e5a6e1a0cfe0..b1c67f16c9ecb 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -686,43 +686,105 @@ static void amd_pstate_adjust_perf(unsigned int cpu, cpufreq_cpu_put(policy); } -static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) +static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) { struct amd_cpudata *cpudata = policy->driver_data; + struct cppc_perf_ctrls perf_ctrls; + u32 highest_perf, nominal_perf, nominal_freq, max_freq; int ret; - if (!cpudata->boost_supported) { - pr_err("Boost mode is not supported by this processor or SBIOS\n"); - return -EINVAL; + highest_perf = READ_ONCE(cpudata->highest_perf); + nominal_perf = READ_ONCE(cpudata->nominal_perf); + nominal_freq = READ_ONCE(cpudata->nominal_freq); + max_freq = READ_ONCE(cpudata->max_freq); + + if (boot_cpu_has(X86_FEATURE_CPPC)) { + u64 value = READ_ONCE(cpudata->cppc_req_cached); + + value &= ~GENMASK_ULL(7, 0); + value |= on ? highest_perf : nominal_perf; + WRITE_ONCE(cpudata->cppc_req_cached, value); + + wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); + } else { + perf_ctrls.max_perf = on ? highest_perf : nominal_perf; + ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); + if (ret) { + cpufreq_cpu_release(policy); + pr_debug("Failed to set max perf on CPU:%d. ret:%d\n", + cpudata->cpu, ret); + return ret; + } } - if (state) - policy->cpuinfo.max_freq = cpudata->max_freq; - else - policy->cpuinfo.max_freq = cpudata->nominal_freq * 1000; + if (on) + policy->cpuinfo.max_freq = max_freq; + else if (policy->cpuinfo.max_freq > nominal_freq * 1000) + policy->cpuinfo.max_freq = nominal_freq * 1000; policy->max = policy->cpuinfo.max_freq; - ret = freq_qos_update_request(&cpudata->req[1], - policy->cpuinfo.max_freq); - if (ret < 0) - return ret; + if (cppc_state == AMD_PSTATE_PASSIVE) { + ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq); + if (ret < 0) + pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu); + } - return 0; + return ret < 0 ? ret : 0; } -static void amd_pstate_boost_init(struct amd_cpudata *cpudata) +static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) { - u32 highest_perf, nominal_perf; + struct amd_cpudata *cpudata = policy->driver_data; + int ret; - highest_perf = READ_ONCE(cpudata->highest_perf); - nominal_perf = READ_ONCE(cpudata->nominal_perf); + if (!cpudata->boost_supported) { + pr_err("Boost mode is not supported by this processor or SBIOS\n"); + return -EOPNOTSUPP; + } + mutex_lock(&amd_pstate_driver_lock); + ret = amd_pstate_cpu_boost_update(policy, state); + WRITE_ONCE(cpudata->boost_state, !ret ? state : false); + policy->boost_enabled = !ret ? state : false; + refresh_frequency_limits(policy); + mutex_unlock(&amd_pstate_driver_lock); - if (highest_perf <= nominal_perf) - return; + return ret; +} - cpudata->boost_supported = true; +static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) +{ + u64 boost_val; + int ret = -1; + + /* + * If platform has no CPB support or disable it, initialize current driver + * boost_enabled state to be false, it is not an error for cpufreq core to handle. + */ + if (!cpu_feature_enabled(X86_FEATURE_CPB)) { + pr_debug_once("Boost CPB capabilities not present in the processor\n"); + ret = 0; + goto exit_err; + } + + /* at least one CPU supports CPB, even if others fail later on to set up */ current_pstate_driver->boost_enabled = true; + + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); + if (ret) { + pr_err_once("failed to read initial CPU boost state!\n"); + ret = -EIO; + goto exit_err; + } + + if (!(boost_val & MSR_K7_HWCR_CPB_DIS)) + cpudata->boost_supported = true; + + return 0; + +exit_err: + cpudata->boost_supported = false; + return ret; } static void amd_perf_ctl_reset(unsigned int cpu) @@ -971,6 +1033,10 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) if (ret) goto free_cpudata1; + ret = amd_pstate_init_boost_support(cpudata); + if (ret) + goto free_cpudata1; + min_freq = READ_ONCE(cpudata->min_freq); max_freq = READ_ONCE(cpudata->max_freq); @@ -983,6 +1049,8 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.min_freq = min_freq; policy->cpuinfo.max_freq = max_freq; + policy->boost_enabled = READ_ONCE(cpudata->boost_supported); + /* It will be updated by governor */ policy->cur = policy->cpuinfo.min_freq; @@ -1008,7 +1076,6 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) policy->driver_data = cpudata; - amd_pstate_boost_init(cpudata); if (!current_pstate_driver->adjust_perf) current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; @@ -1433,6 +1500,10 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (ret) goto free_cpudata1; + ret = amd_pstate_init_boost_support(cpudata); + if (ret) + goto free_cpudata1; + min_freq = READ_ONCE(cpudata->min_freq); max_freq = READ_ONCE(cpudata->max_freq); @@ -1448,6 +1519,8 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->min = policy->cpuinfo.min_freq; policy->max = policy->cpuinfo.max_freq; + policy->boost_enabled = READ_ONCE(cpudata->boost_supported); + /* * Set the policy to provide a valid fallback value in case * the default cpufreq governor is neither powersave nor performance. @@ -1469,7 +1542,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) return ret; WRITE_ONCE(cpudata->cppc_cap1_cached, value); } - amd_pstate_boost_init(cpudata); return 0; @@ -1731,6 +1803,7 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .suspend = amd_pstate_epp_suspend, .resume = amd_pstate_epp_resume, .update_limits = amd_pstate_update_limits, + .set_boost = amd_pstate_set_boost, .name = "amd-pstate-epp", .attr = amd_pstate_epp_attr, }; diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index f80b33fa5d43a..cc8bb2bc325aa 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -100,6 +100,7 @@ struct amd_cpudata { u64 cppc_cap1_cached; bool suspended; s16 epp_default; + bool boost_state; }; #endif /* _LINUX_AMD_PSTATE_H */ From 73c76f8f70171da53f7cbd6cac471bdae152bb28 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Tue, 25 Jun 2024 23:27:32 -0500 Subject: [PATCH 025/188] cpufreq: amd-pstate: Cap the CPPC.max_perf to nominal_perf if CPB is off commit 89ac482d5105b0c6710775bab1aa7d3d0730c642 upstream. When Core Performance Boost is disabled by the user, the CPPC_REQ.max_perf should not exceed the nominal_perf since by definition the frequencies between nominal_perf and the highest_perf are in the boost range. Fix this in amd_pstate_update() Acked-by: Huang Rui Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Signed-off-by: Perry Yuan Link: https://lore.kernel.org/r/66f55232be01092c423f0523f68b82b80c293943.1718988436.git.perry.yuan@amd.com Link: https://lore.kernel.org/r/20240626042733.3747-4-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b1c67f16c9ecb..e9bbae23e8133 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -516,6 +516,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, unsigned long max_freq; struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu); u64 prev = READ_ONCE(cpudata->cppc_req_cached); + u32 nominal_perf = READ_ONCE(cpudata->nominal_perf); u64 value = prev; min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, @@ -538,6 +539,10 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, value &= ~AMD_CPPC_DES_PERF(~0L); value |= AMD_CPPC_DES_PERF(des_perf); + /* limit the max perf when core performance boost feature is disabled */ + if (!cpudata->boost_supported) + max_perf = min_t(unsigned long, nominal_perf, max_perf); + value &= ~AMD_CPPC_MAX_PERF(~0L); value |= AMD_CPPC_MAX_PERF(max_perf); From d40731033c83ae5a71567f49336d573c1a22a7d4 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 4 Jul 2024 12:23:54 +0530 Subject: [PATCH 026/188] cpufreq: pcc: Remove empty exit() callback commit dfd3e8b90b3660b706c3031b0f746377c571b324 upstream. The exit() callback is optional, remove the empty one. Signed-off-by: Viresh Kumar Acked-by: Rafael J. Wysocki Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/pcc-cpufreq.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/cpufreq/pcc-cpufreq.c b/drivers/cpufreq/pcc-cpufreq.c index 6f8b5ea7aeae4..771efbf51a48e 100644 --- a/drivers/cpufreq/pcc-cpufreq.c +++ b/drivers/cpufreq/pcc-cpufreq.c @@ -562,18 +562,12 @@ static int pcc_cpufreq_cpu_init(struct cpufreq_policy *policy) return result; } -static int pcc_cpufreq_cpu_exit(struct cpufreq_policy *policy) -{ - return 0; -} - static struct cpufreq_driver pcc_cpufreq_driver = { .flags = CPUFREQ_CONST_LOOPS, .get = pcc_get_freq, .verify = pcc_cpufreq_verify, .target = pcc_cpufreq_target, .init = pcc_cpufreq_cpu_init, - .exit = pcc_cpufreq_cpu_exit, .name = "pcc-cpufreq", }; From 3dd9e451e1cf1500f9df66698b8e4e040bc483e2 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Tue, 2 Jul 2024 08:14:13 +0000 Subject: [PATCH 027/188] cpufreq/amd-pstate-ut: Convert nominal_freq to khz during comparisons commit f21ab5ed4e8758b06230900f44b9dcbcfdc0c3ae upstream. cpudata->nominal_freq being in MHz whereas other frequencies being in KHz breaks the amd-pstate-ut frequency sanity check. This fixes it. Fixes: e4731baaf294 ("cpufreq: amd-pstate: Fix the inconsistency in max frequency units") Reported-by: David Arcari Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20240702081413.5688-2-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index fc275d41d51e9..66b73c308ce67 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -202,6 +202,7 @@ static void amd_pstate_ut_check_freq(u32 index) int cpu = 0; struct cpufreq_policy *policy = NULL; struct amd_cpudata *cpudata = NULL; + u32 nominal_freq_khz; for_each_possible_cpu(cpu) { policy = cpufreq_cpu_get(cpu); @@ -209,13 +210,14 @@ static void amd_pstate_ut_check_freq(u32 index) break; cpudata = policy->driver_data; - if (!((cpudata->max_freq >= cpudata->nominal_freq) && - (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) && + nominal_freq_khz = cpudata->nominal_freq*1000; + if (!((cpudata->max_freq >= nominal_freq_khz) && + (nominal_freq_khz > cpudata->lowest_nonlinear_freq) && (cpudata->lowest_nonlinear_freq > cpudata->min_freq) && (cpudata->min_freq > 0))) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n", - __func__, cpu, cpudata->max_freq, cpudata->nominal_freq, + __func__, cpu, cpudata->max_freq, nominal_freq_khz, cpudata->lowest_nonlinear_freq, cpudata->min_freq); goto skip_test; } @@ -229,13 +231,13 @@ static void amd_pstate_ut_check_freq(u32 index) if (cpudata->boost_supported) { if ((policy->max == cpudata->max_freq) || - (policy->max == cpudata->nominal_freq)) + (policy->max == nominal_freq_khz)) amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; else { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d policy_max=%d should be equal cpu_max=%d or cpu_nominal=%d !\n", __func__, cpu, policy->max, cpudata->max_freq, - cpudata->nominal_freq); + nominal_freq_khz); goto skip_test; } } else { From ae64370017c10316eb4605a3115c08d241a94770 Mon Sep 17 00:00:00 2001 From: Lizhe Date: Thu, 4 Jul 2024 12:23:55 +0530 Subject: [PATCH 028/188] cpufreq: Make cpufreq_driver->exit() return void commit b4b1ddc9dfe997a5f492fa3a36487f8e7a5de30d upstream. The cpufreq core doesn't check the return type of the exit() callback and there is not much the core can do on failures at that point. Just drop the returned value and make it return void. Signed-off-by: Lizhe [ Viresh: Reworked the patches to fix all missing changes together. ] Signed-off-by: Viresh Kumar Reviewed-by: AngeloGioacchino Del Regno # Mediatek Acked-by: Sudeep Holla # scpi, scmi, vexpress Acked-by: Mario Limonciello # amd Reviewed-by: Florian Fainelli # bmips Acked-by: Rafael J. Wysocki Acked-by: Kevin Hilman # omap Signed-off-by: Rahul Kumar Signed-off-by: mohanasv2 --- drivers/cpufreq/acpi-cpufreq.c | 4 +--- drivers/cpufreq/amd-pstate.c | 7 ++----- drivers/cpufreq/apple-soc-cpufreq.c | 4 +--- drivers/cpufreq/bmips-cpufreq.c | 4 +--- drivers/cpufreq/cppc_cpufreq.c | 3 +-- drivers/cpufreq/cpufreq-dt.c | 3 +-- drivers/cpufreq/e_powersaver.c | 3 +-- drivers/cpufreq/intel_pstate.c | 8 +++----- drivers/cpufreq/mediatek-cpufreq-hw.c | 4 +--- drivers/cpufreq/mediatek-cpufreq.c | 4 +--- drivers/cpufreq/omap-cpufreq.c | 3 +-- drivers/cpufreq/pasemi-cpufreq.c | 6 ++---- drivers/cpufreq/powernow-k6.c | 5 ++--- drivers/cpufreq/powernow-k7.c | 3 +-- drivers/cpufreq/powernow-k8.c | 6 ++---- drivers/cpufreq/powernv-cpufreq.c | 4 +--- drivers/cpufreq/ppc_cbe_cpufreq.c | 3 +-- drivers/cpufreq/qcom-cpufreq-hw.c | 4 +--- drivers/cpufreq/qoriq-cpufreq.c | 4 +--- drivers/cpufreq/scmi-cpufreq.c | 4 +--- drivers/cpufreq/scpi-cpufreq.c | 4 +--- drivers/cpufreq/sh-cpufreq.c | 4 +--- drivers/cpufreq/sparc-us2e-cpufreq.c | 3 +-- drivers/cpufreq/sparc-us3-cpufreq.c | 3 +-- drivers/cpufreq/speedstep-centrino.c | 10 +++------- drivers/cpufreq/tegra194-cpufreq.c | 4 +--- drivers/cpufreq/vexpress-spc-cpufreq.c | 5 ++--- include/linux/cpufreq.h | 2 +- 28 files changed, 37 insertions(+), 84 deletions(-) diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index ba53a1415e6cb..2be14a4165d52 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -933,7 +933,7 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) return result; } -static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) { struct acpi_cpufreq_data *data = policy->driver_data; @@ -946,8 +946,6 @@ static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) free_cpumask_var(data->freqdomain_cpus); kfree(policy->freq_table); kfree(data); - - return 0; } static int acpi_cpufreq_resume(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index e9bbae23e8133..b6312804e9a29 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1093,7 +1093,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) return ret; } -static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) +static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; @@ -1101,8 +1101,6 @@ static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) freq_qos_remove_request(&cpudata->req[0]); policy->fast_switch_possible = false; kfree(cpudata); - - return 0; } static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) @@ -1555,7 +1553,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) return ret; } -static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) +static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; @@ -1565,7 +1563,6 @@ static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) } pr_debug("CPU %d exiting\n", policy->cpu); - return 0; } static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 9ba6b09775f61..c984f879d67d9 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -311,7 +311,7 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) return ret; } -static int apple_soc_cpufreq_exit(struct cpufreq_policy *policy) +static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) { struct apple_cpu_priv *priv = policy->driver_data; @@ -319,8 +319,6 @@ static int apple_soc_cpufreq_exit(struct cpufreq_policy *policy) dev_pm_opp_remove_all_dynamic(priv->cpu_dev); iounmap(priv->reg_base); kfree(priv); - - return 0; } static struct cpufreq_driver apple_soc_cpufreq_driver = { diff --git a/drivers/cpufreq/bmips-cpufreq.c b/drivers/cpufreq/bmips-cpufreq.c index 39221a9a187a7..17a4c174553d3 100644 --- a/drivers/cpufreq/bmips-cpufreq.c +++ b/drivers/cpufreq/bmips-cpufreq.c @@ -121,11 +121,9 @@ static int bmips_cpufreq_target_index(struct cpufreq_policy *policy, return 0; } -static int bmips_cpufreq_exit(struct cpufreq_policy *policy) +static void bmips_cpufreq_exit(struct cpufreq_policy *policy) { kfree(policy->freq_table); - - return 0; } static int bmips_cpufreq_init(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 6ce9c88cbd0bf..073190411c79e 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -693,7 +693,7 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) return ret; } -static int cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy) { struct cppc_cpudata *cpu_data = policy->driver_data; struct cppc_perf_caps *caps = &cpu_data->perf_caps; @@ -710,7 +710,6 @@ static int cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy) caps->lowest_perf, cpu, ret); cppc_cpufreq_put_cpu_data(policy); - return 0; } static inline u64 get_delta(u64 t1, u64 t0) diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index 2a968e62ea56b..fe7dd4e55b003 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -166,10 +166,9 @@ static int cpufreq_offline(struct cpufreq_policy *policy) return 0; } -static int cpufreq_exit(struct cpufreq_policy *policy) +static void cpufreq_exit(struct cpufreq_policy *policy) { clk_put(policy->clk); - return 0; } static struct cpufreq_driver dt_cpufreq_driver = { diff --git a/drivers/cpufreq/e_powersaver.c b/drivers/cpufreq/e_powersaver.c index ab93bce8ae77d..6e958b09e1b52 100644 --- a/drivers/cpufreq/e_powersaver.c +++ b/drivers/cpufreq/e_powersaver.c @@ -360,14 +360,13 @@ static int eps_cpu_init(struct cpufreq_policy *policy) return 0; } -static int eps_cpu_exit(struct cpufreq_policy *policy) +static void eps_cpu_exit(struct cpufreq_policy *policy) { unsigned int cpu = policy->cpu; /* Bye */ kfree(eps_cpu[cpu]); eps_cpu[cpu] = NULL; - return 0; } static struct cpufreq_driver eps_driver = { diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index d2e059a270118..b87f3609fbd51 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2716,13 +2716,11 @@ static int intel_pstate_cpu_offline(struct cpufreq_policy *policy) return intel_cpufreq_cpu_offline(policy); } -static int intel_pstate_cpu_exit(struct cpufreq_policy *policy) +static void intel_pstate_cpu_exit(struct cpufreq_policy *policy) { pr_debug("CPU %d exiting\n", policy->cpu); policy->fast_switch_possible = false; - - return 0; } static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) @@ -3053,7 +3051,7 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) return ret; } -static int intel_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void intel_cpufreq_cpu_exit(struct cpufreq_policy *policy) { struct freq_qos_request *req; @@ -3063,7 +3061,7 @@ static int intel_cpufreq_cpu_exit(struct cpufreq_policy *policy) freq_qos_remove_request(req); kfree(req); - return intel_pstate_cpu_exit(policy); + intel_pstate_cpu_exit(policy); } static int intel_cpufreq_suspend(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/mediatek-cpufreq-hw.c b/drivers/cpufreq/mediatek-cpufreq-hw.c index 55353fe7b9e74..aeb5e63045421 100644 --- a/drivers/cpufreq/mediatek-cpufreq-hw.c +++ b/drivers/cpufreq/mediatek-cpufreq-hw.c @@ -260,7 +260,7 @@ static int mtk_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) return 0; } -static int mtk_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) +static void mtk_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) { struct mtk_cpufreq_data *data = policy->driver_data; struct resource *res = data->res; @@ -270,8 +270,6 @@ static int mtk_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) writel_relaxed(0x0, data->reg_bases[REG_FREQ_ENABLE]); iounmap(base); release_mem_region(res->start, resource_size(res)); - - return 0; } static void mtk_cpufreq_register_em(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/mediatek-cpufreq.c b/drivers/cpufreq/mediatek-cpufreq.c index a0a61919bc4c1..aba34fb7948ea 100644 --- a/drivers/cpufreq/mediatek-cpufreq.c +++ b/drivers/cpufreq/mediatek-cpufreq.c @@ -599,13 +599,11 @@ static int mtk_cpufreq_init(struct cpufreq_policy *policy) return 0; } -static int mtk_cpufreq_exit(struct cpufreq_policy *policy) +static void mtk_cpufreq_exit(struct cpufreq_policy *policy) { struct mtk_cpu_dvfs_info *info = policy->driver_data; dev_pm_opp_free_cpufreq_table(info->cpu_dev, &policy->freq_table); - - return 0; } static struct cpufreq_driver mtk_cpufreq_driver = { diff --git a/drivers/cpufreq/omap-cpufreq.c b/drivers/cpufreq/omap-cpufreq.c index 895690856665d..3458d5cc9b7f2 100644 --- a/drivers/cpufreq/omap-cpufreq.c +++ b/drivers/cpufreq/omap-cpufreq.c @@ -135,11 +135,10 @@ static int omap_cpu_init(struct cpufreq_policy *policy) return 0; } -static int omap_cpu_exit(struct cpufreq_policy *policy) +static void omap_cpu_exit(struct cpufreq_policy *policy) { freq_table_free(); clk_put(policy->clk); - return 0; } static struct cpufreq_driver omap_driver = { diff --git a/drivers/cpufreq/pasemi-cpufreq.c b/drivers/cpufreq/pasemi-cpufreq.c index 039a66bbe1bef..ee925b53b6b92 100644 --- a/drivers/cpufreq/pasemi-cpufreq.c +++ b/drivers/cpufreq/pasemi-cpufreq.c @@ -204,21 +204,19 @@ static int pas_cpufreq_cpu_init(struct cpufreq_policy *policy) return err; } -static int pas_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void pas_cpufreq_cpu_exit(struct cpufreq_policy *policy) { /* * We don't support CPU hotplug. Don't unmap after the system * has already made it to a running state. */ if (system_state >= SYSTEM_RUNNING) - return 0; + return; if (sdcasr_mapbase) iounmap(sdcasr_mapbase); if (sdcpwr_mapbase) iounmap(sdcpwr_mapbase); - - return 0; } static int pas_cpufreq_target(struct cpufreq_policy *policy, diff --git a/drivers/cpufreq/powernow-k6.c b/drivers/cpufreq/powernow-k6.c index 41eefef95d873..f0a4a6c312049 100644 --- a/drivers/cpufreq/powernow-k6.c +++ b/drivers/cpufreq/powernow-k6.c @@ -219,7 +219,7 @@ static int powernow_k6_cpu_init(struct cpufreq_policy *policy) } -static int powernow_k6_cpu_exit(struct cpufreq_policy *policy) +static void powernow_k6_cpu_exit(struct cpufreq_policy *policy) { unsigned int i; @@ -234,10 +234,9 @@ static int powernow_k6_cpu_exit(struct cpufreq_policy *policy) cpufreq_freq_transition_begin(policy, &freqs); powernow_k6_target(policy, i); cpufreq_freq_transition_end(policy, &freqs, 0); - break; + return; } } - return 0; } static unsigned int powernow_k6_get(unsigned int cpu) diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c index 5d515fc348364..4271446c87258 100644 --- a/drivers/cpufreq/powernow-k7.c +++ b/drivers/cpufreq/powernow-k7.c @@ -644,7 +644,7 @@ static int powernow_cpu_init(struct cpufreq_policy *policy) return 0; } -static int powernow_cpu_exit(struct cpufreq_policy *policy) +static void powernow_cpu_exit(struct cpufreq_policy *policy) { #ifdef CONFIG_X86_POWERNOW_K7_ACPI if (acpi_processor_perf) { @@ -655,7 +655,6 @@ static int powernow_cpu_exit(struct cpufreq_policy *policy) #endif kfree(powernow_table); - return 0; } static struct cpufreq_driver powernow_driver = { diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c index b10f7a1b77f11..a01170f7d01c1 100644 --- a/drivers/cpufreq/powernow-k8.c +++ b/drivers/cpufreq/powernow-k8.c @@ -1089,13 +1089,13 @@ static int powernowk8_cpu_init(struct cpufreq_policy *pol) return -ENODEV; } -static int powernowk8_cpu_exit(struct cpufreq_policy *pol) +static void powernowk8_cpu_exit(struct cpufreq_policy *pol) { struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); int cpu; if (!data) - return -EINVAL; + return; powernow_k8_cpu_exit_acpi(data); @@ -1104,8 +1104,6 @@ static int powernowk8_cpu_exit(struct cpufreq_policy *pol) /* pol->cpus will be empty here, use related_cpus instead. */ for_each_cpu(cpu, pol->related_cpus) per_cpu(powernow_data, cpu) = NULL; - - return 0; } static void query_values_on_cpu(void *_err) diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c index fddbd1ea16355..50c62929f7ca6 100644 --- a/drivers/cpufreq/powernv-cpufreq.c +++ b/drivers/cpufreq/powernv-cpufreq.c @@ -874,7 +874,7 @@ static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy) return 0; } -static int powernv_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void powernv_cpufreq_cpu_exit(struct cpufreq_policy *policy) { struct powernv_smp_call_data freq_data; struct global_pstate_info *gpstates = policy->driver_data; @@ -886,8 +886,6 @@ static int powernv_cpufreq_cpu_exit(struct cpufreq_policy *policy) del_timer_sync(&gpstates->timer); kfree(policy->driver_data); - - return 0; } static int powernv_cpufreq_reboot_notifier(struct notifier_block *nb, diff --git a/drivers/cpufreq/ppc_cbe_cpufreq.c b/drivers/cpufreq/ppc_cbe_cpufreq.c index 88afc49941b71..5ee4c7bfdcc53 100644 --- a/drivers/cpufreq/ppc_cbe_cpufreq.c +++ b/drivers/cpufreq/ppc_cbe_cpufreq.c @@ -113,10 +113,9 @@ static int cbe_cpufreq_cpu_init(struct cpufreq_policy *policy) return 0; } -static int cbe_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void cbe_cpufreq_cpu_exit(struct cpufreq_policy *policy) { cbe_cpufreq_pmi_policy_exit(policy); - return 0; } static int cbe_cpufreq_target(struct cpufreq_policy *policy, diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 92c0fc4a3cf60..0daf442aedd4b 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -574,7 +574,7 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) return qcom_cpufreq_hw_lmh_init(policy, index); } -static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) +static void qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) { struct device *cpu_dev = get_cpu_device(policy->cpu); struct qcom_cpufreq_data *data = policy->driver_data; @@ -584,8 +584,6 @@ static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) qcom_cpufreq_hw_lmh_exit(data); kfree(policy->freq_table); kfree(data); - - return 0; } static void qcom_cpufreq_ready(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/qoriq-cpufreq.c b/drivers/cpufreq/qoriq-cpufreq.c index 0aecaecbb0e67..3519bf34d3970 100644 --- a/drivers/cpufreq/qoriq-cpufreq.c +++ b/drivers/cpufreq/qoriq-cpufreq.c @@ -225,7 +225,7 @@ static int qoriq_cpufreq_cpu_init(struct cpufreq_policy *policy) return -ENODEV; } -static int qoriq_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void qoriq_cpufreq_cpu_exit(struct cpufreq_policy *policy) { struct cpu_data *data = policy->driver_data; @@ -233,8 +233,6 @@ static int qoriq_cpufreq_cpu_exit(struct cpufreq_policy *policy) kfree(data->table); kfree(data); policy->driver_data = NULL; - - return 0; } static int qoriq_cpufreq_target(struct cpufreq_policy *policy, diff --git a/drivers/cpufreq/scmi-cpufreq.c b/drivers/cpufreq/scmi-cpufreq.c index 5f9d3bc863c68..a17066521f083 100644 --- a/drivers/cpufreq/scmi-cpufreq.c +++ b/drivers/cpufreq/scmi-cpufreq.c @@ -247,7 +247,7 @@ static int scmi_cpufreq_init(struct cpufreq_policy *policy) return ret; } -static int scmi_cpufreq_exit(struct cpufreq_policy *policy) +static void scmi_cpufreq_exit(struct cpufreq_policy *policy) { struct scmi_data *priv = policy->driver_data; @@ -255,8 +255,6 @@ static int scmi_cpufreq_exit(struct cpufreq_policy *policy) dev_pm_opp_remove_all_dynamic(priv->cpu_dev); free_cpumask_var(priv->opp_shared_cpus); kfree(priv); - - return 0; } static void scmi_cpufreq_register_em(struct cpufreq_policy *policy) diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c index f51dfdc22a709..a33e65c3a6b1d 100644 --- a/drivers/cpufreq/scpi-cpufreq.c +++ b/drivers/cpufreq/scpi-cpufreq.c @@ -175,7 +175,7 @@ static int scpi_cpufreq_init(struct cpufreq_policy *policy) return ret; } -static int scpi_cpufreq_exit(struct cpufreq_policy *policy) +static void scpi_cpufreq_exit(struct cpufreq_policy *policy) { struct scpi_data *priv = policy->driver_data; @@ -183,8 +183,6 @@ static int scpi_cpufreq_exit(struct cpufreq_policy *policy) dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); dev_pm_opp_remove_all_dynamic(priv->cpu_dev); kfree(priv); - - return 0; } static struct cpufreq_driver scpi_cpufreq_driver = { diff --git a/drivers/cpufreq/sh-cpufreq.c b/drivers/cpufreq/sh-cpufreq.c index b8704232c27b2..aa74036d0420e 100644 --- a/drivers/cpufreq/sh-cpufreq.c +++ b/drivers/cpufreq/sh-cpufreq.c @@ -135,14 +135,12 @@ static int sh_cpufreq_cpu_init(struct cpufreq_policy *policy) return 0; } -static int sh_cpufreq_cpu_exit(struct cpufreq_policy *policy) +static void sh_cpufreq_cpu_exit(struct cpufreq_policy *policy) { unsigned int cpu = policy->cpu; struct clk *cpuclk = &per_cpu(sh_cpuclk, cpu); clk_put(cpuclk); - - return 0; } static struct cpufreq_driver sh_cpufreq_driver = { diff --git a/drivers/cpufreq/sparc-us2e-cpufreq.c b/drivers/cpufreq/sparc-us2e-cpufreq.c index 2783d3d55fcee..8a0cd5312a593 100644 --- a/drivers/cpufreq/sparc-us2e-cpufreq.c +++ b/drivers/cpufreq/sparc-us2e-cpufreq.c @@ -296,10 +296,9 @@ static int us2e_freq_cpu_init(struct cpufreq_policy *policy) return 0; } -static int us2e_freq_cpu_exit(struct cpufreq_policy *policy) +static void us2e_freq_cpu_exit(struct cpufreq_policy *policy) { us2e_freq_target(policy, 0); - return 0; } static struct cpufreq_driver cpufreq_us2e_driver = { diff --git a/drivers/cpufreq/sparc-us3-cpufreq.c b/drivers/cpufreq/sparc-us3-cpufreq.c index 6c3657679a880..b50f9d13e6d26 100644 --- a/drivers/cpufreq/sparc-us3-cpufreq.c +++ b/drivers/cpufreq/sparc-us3-cpufreq.c @@ -140,10 +140,9 @@ static int us3_freq_cpu_init(struct cpufreq_policy *policy) return 0; } -static int us3_freq_cpu_exit(struct cpufreq_policy *policy) +static void us3_freq_cpu_exit(struct cpufreq_policy *policy) { us3_freq_target(policy, 0); - return 0; } static struct cpufreq_driver cpufreq_us3_driver = { diff --git a/drivers/cpufreq/speedstep-centrino.c b/drivers/cpufreq/speedstep-centrino.c index 75b10ecdb60f2..0eb51fb73a803 100644 --- a/drivers/cpufreq/speedstep-centrino.c +++ b/drivers/cpufreq/speedstep-centrino.c @@ -400,16 +400,12 @@ static int centrino_cpu_init(struct cpufreq_policy *policy) return 0; } -static int centrino_cpu_exit(struct cpufreq_policy *policy) +static void centrino_cpu_exit(struct cpufreq_policy *policy) { unsigned int cpu = policy->cpu; - if (!per_cpu(centrino_model, cpu)) - return -ENODEV; - - per_cpu(centrino_model, cpu) = NULL; - - return 0; + if (per_cpu(centrino_model, cpu)) + per_cpu(centrino_model, cpu) = NULL; } /** diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c index 386aed3637b4e..454767c2ad6ba 100644 --- a/drivers/cpufreq/tegra194-cpufreq.c +++ b/drivers/cpufreq/tegra194-cpufreq.c @@ -526,14 +526,12 @@ static int tegra194_cpufreq_offline(struct cpufreq_policy *policy) return 0; } -static int tegra194_cpufreq_exit(struct cpufreq_policy *policy) +static void tegra194_cpufreq_exit(struct cpufreq_policy *policy) { struct device *cpu_dev = get_cpu_device(policy->cpu); dev_pm_opp_remove_all_dynamic(cpu_dev); dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); - - return 0; } static int tegra194_cpufreq_set_target(struct cpufreq_policy *policy, diff --git a/drivers/cpufreq/vexpress-spc-cpufreq.c b/drivers/cpufreq/vexpress-spc-cpufreq.c index 9ac4ea50b8746..3fadf536c4298 100644 --- a/drivers/cpufreq/vexpress-spc-cpufreq.c +++ b/drivers/cpufreq/vexpress-spc-cpufreq.c @@ -447,7 +447,7 @@ static int ve_spc_cpufreq_init(struct cpufreq_policy *policy) return 0; } -static int ve_spc_cpufreq_exit(struct cpufreq_policy *policy) +static void ve_spc_cpufreq_exit(struct cpufreq_policy *policy) { struct device *cpu_dev; @@ -455,11 +455,10 @@ static int ve_spc_cpufreq_exit(struct cpufreq_policy *policy) if (!cpu_dev) { pr_err("%s: failed to get cpu%d device\n", __func__, policy->cpu); - return -ENODEV; + return; } put_cluster_clk_and_freq_table(cpu_dev, policy->related_cpus); - return 0; } static struct cpufreq_driver ve_spc_cpufreq_driver = { diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 94e285e77e04b..2a1324dd3148b 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -391,7 +391,7 @@ struct cpufreq_driver { int (*online)(struct cpufreq_policy *policy); int (*offline)(struct cpufreq_policy *policy); - int (*exit)(struct cpufreq_policy *policy); + void (*exit)(struct cpufreq_policy *policy); int (*suspend)(struct cpufreq_policy *policy); int (*resume)(struct cpufreq_policy *policy); From 23bde5c7cf3e490f287d22e411ea2710d463502b Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 13 Aug 2024 15:21:13 +0530 Subject: [PATCH 029/188] cpufreq: amd-pstate: Fix uninitialized variable in amd_pstate_cpu_boost_update() commit 67d95303c84732c2e1de5730756281f648dbefaf upstream. Smatch complains that "ret" could be uninitialized: drivers/cpufreq/amd-pstate.c:734 amd_pstate_cpu_boost_update() error: uninitialized symbol 'ret'. This seems like it probably is a real issue. Initialize "ret" to zero to be safe. Fixes: c8c68c38b56f ("cpufreq: amd-pstate: initialize core precision boost state") Signed-off-by: Dan Carpenter Reviewed-by: Perry Yuan Acked-by: Gautham R. Shenoy Link: https://lore.kernel.org/lkml/7ff53543-6c04-48a0-8d99-7dc010b93b3a@stanley.mountain/T/ Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b6312804e9a29..9f8cd36b587cf 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -696,7 +696,7 @@ static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) struct amd_cpudata *cpudata = policy->driver_data; struct cppc_perf_ctrls perf_ctrls; u32 highest_perf, nominal_perf, nominal_freq, max_freq; - int ret; + int ret = 0; highest_perf = READ_ONCE(cpudata->highest_perf); nominal_perf = READ_ONCE(cpudata->nominal_perf); From ca4eaced48cc1d5e91c10d45f9ade359671c4064 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Tue, 13 Aug 2024 15:21:14 +0530 Subject: [PATCH 030/188] cpufreq/amd-pstate: Use topology_logical_package_id() instead of logical_die_id() commit 0d8584d288a9b4132e945d76bcc04395d158b2e7 upstream. After the commit 63edbaa48a57 ("x86/cpu/topology: Add support for the AMD 0x80000026 leaf"), the topolgy_logical_die_id() function returns the logical Core Chiplet Die (CCD) ID instead of the logical socket ID. Since this is currently used to set MSR_AMD_CPPC_ENABLE, which needs to be set on any one of the threads of the socket, it is prudent to use topology_logical_package_id() in place of topology_logical_die_id(). Fixes: 63edbaa48a57 ("x86/cpu/topology: Add support for the AMD 0x80000026 leaf") cc: stable@vger.kernel.org # 6.10 Signed-off-by: Gautham R. Shenoy Tested-by: Dhananjay Ugwekar Link: https://lore.kernel.org/lkml/20240801124509.3650-1-Dhananjay.Ugwekar@amd.com/ Signed-off-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 9f8cd36b587cf..d98ca71c290aa 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -320,7 +320,7 @@ static inline int pstate_enable(bool enable) return 0; for_each_present_cpu(cpu) { - unsigned long logical_id = topology_logical_die_id(cpu); + unsigned long logical_id = topology_logical_package_id(cpu); if (test_bit(logical_id, &logical_proc_id_mask)) continue; From 69f4bf915cf705debdcc9d3d4b5111c42883a566 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 2 Jul 2024 12:15:14 -0500 Subject: [PATCH 031/188] cpufreq/amd-pstate-ut: Don't check for highest perf matching on prefcore commit 9983a9cd4d429dc9ca01770083c4c1f366214b65 upstream. If a system is using preferred cores the highest perf will be inconsistent as it can change from system events. Skip the checks for it. Fixes: e571a5e2068e ("cpufreq: amd-pstate: Update amd-pstate preferred core ranking dynamically") Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 66b73c308ce67..b7318669485e4 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -160,14 +160,17 @@ static void amd_pstate_ut_check_perf(u32 index) lowest_perf = AMD_CPPC_LOWEST_PERF(cap1); } - if ((highest_perf != READ_ONCE(cpudata->highest_perf)) || - (nominal_perf != READ_ONCE(cpudata->nominal_perf)) || + if (highest_perf != READ_ONCE(cpudata->highest_perf) && !cpudata->hw_prefcore) { + pr_err("%s cpu%d highest=%d %d highest perf doesn't match\n", + __func__, cpu, highest_perf, cpudata->highest_perf); + goto skip_test; + } + if ((nominal_perf != READ_ONCE(cpudata->nominal_perf)) || (lowest_nonlinear_perf != READ_ONCE(cpudata->lowest_nonlinear_perf)) || (lowest_perf != READ_ONCE(cpudata->lowest_perf))) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; - pr_err("%s cpu%d highest=%d %d nominal=%d %d lowest_nonlinear=%d %d lowest=%d %d, they should be equal!\n", - __func__, cpu, highest_perf, cpudata->highest_perf, - nominal_perf, cpudata->nominal_perf, + pr_err("%s cpu%d nominal=%d %d lowest_nonlinear=%d %d lowest=%d %d, they should be equal!\n", + __func__, cpu, nominal_perf, cpudata->nominal_perf, lowest_nonlinear_perf, cpudata->lowest_nonlinear_perf, lowest_perf, cpudata->lowest_perf); goto skip_test; From d472fae2613631fc5b4e45fd12b211d15b8366a6 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Wed, 28 Aug 2024 15:40:35 +0530 Subject: [PATCH 032/188] cpufreq/amd-pstate: Remove warning for X86_FEATURE_CPPC on certain Zen models commit 9c68a3b03e8109f3917fd35f39043499897d4a79 upstream. commit bff7d13c190a ("cpufreq: amd-pstate: add debug message while CPPC is supported and disabled by SBIOS") issues a warning on plaforms where the X86_FEATURE_CPPC is expected to be enabled, but is not due to it being disabled in the BIOS. This feature bit corresponds to CPUID 0x80000008.ebx[27] which is a reserved bit on the Zen1 processors and a reserved bit on Zen2 based models 0x70-0x7F, and is expected to be cleared on these platforms. Thus printing the warning message for these models when X86_FEATURE_CPPC is unavailable is incorrect. Fix this. Modify some of the comments, and use switch-case for model range checking for improved readability while at it. Fixes: bff7d13c190a ("cpufreq: amd-pstate: add debug message while CPPC is supported and disabled by SBIOS") Cc: Xiaojian Du Reported-by: David Wang <00107082@163.com> Closes: https://lore.kernel.org/lkml/20240730140111.4491-1-00107082@163.com/ Signed-off-by: Gautham R. Shenoy Acked-by: Mario Limonciello Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index d98ca71c290aa..ac218450ec275 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1844,20 +1844,34 @@ static bool amd_cppc_supported(void) } /* - * If the CPPC feature is disabled in the BIOS for processors that support MSR-based CPPC, - * the AMD Pstate driver may not function correctly. - * Check the CPPC flag and display a warning message if the platform supports CPPC. - * Note: below checking code will not abort the driver registeration process because of - * the code is added for debugging purposes. + * If the CPPC feature is disabled in the BIOS for processors + * that support MSR-based CPPC, the AMD Pstate driver may not + * function correctly. + * + * For such processors, check the CPPC flag and display a + * warning message if the platform supports CPPC. + * + * Note: The code check below will not abort the driver + * registration process because of the code is added for + * debugging purposes. Besides, it may still be possible for + * the driver to work using the shared-memory mechanism. */ if (!cpu_feature_enabled(X86_FEATURE_CPPC)) { - if (cpu_feature_enabled(X86_FEATURE_ZEN1) || cpu_feature_enabled(X86_FEATURE_ZEN2)) { - if (c->x86_model > 0x60 && c->x86_model < 0xaf) + if (cpu_feature_enabled(X86_FEATURE_ZEN2)) { + switch (c->x86_model) { + case 0x60 ... 0x6F: + case 0x80 ... 0xAF: warn = true; - } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || cpu_feature_enabled(X86_FEATURE_ZEN4)) { - if ((c->x86_model > 0x10 && c->x86_model < 0x1F) || - (c->x86_model > 0x40 && c->x86_model < 0xaf)) + break; + } + } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) || + cpu_feature_enabled(X86_FEATURE_ZEN4)) { + switch (c->x86_model) { + case 0x10 ... 0x1F: + case 0x40 ... 0xAF: warn = true; + break; + } } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) { warn = true; } From 3a576299a70697963fcb44c9790cce22569d8591 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Fri, 9 Aug 2024 06:08:16 +0000 Subject: [PATCH 033/188] cpufreq/amd-pstate: Add the missing cpufreq_cpu_put() commit 49243adc715e6ae34d6cc003827e63bcf5b3a21d upstream. Fix the reference counting of cpufreq_policy object in amd_pstate_update() function by adding the missing cpufreq_cpu_put(). Fixes: e8f555daacd3 ("cpufreq/amd-pstate: fix setting policy current frequency value") Signed-off-by: Dhananjay Ugwekar Reviewed-by: Perry Yuan Signed-off-by: Viresh Kumar Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index ac218450ec275..53e17369d472a 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -553,12 +553,15 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, } if (value == prev) - return; + goto cpufreq_policy_put; WRITE_ONCE(cpudata->cppc_req_cached, value); amd_pstate_update_perf(cpudata, min_perf, des_perf, max_perf, fast_switch); + +cpufreq_policy_put: + cpufreq_cpu_put(policy); } static int amd_pstate_verify(struct cpufreq_policy_data *policy) From a3a30bee15db67b681d753f33484b0e170293979 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sun, 1 Sep 2024 00:00:35 -0500 Subject: [PATCH 034/188] cpufreq/amd-pstate: Catch failures for amd_pstate_epp_update_limit() commit c3e093efbc6cac7bf9dc531dcb751b86daaa65b0 upstream. amd_pstate_set_epp() calls cppc_set_epp_perf() which can fail for a variety of reasons but this is ignored. Change the return flow to allow failures. Reviewed-by: Perry Yuan Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 53e17369d472a..8b3b68563bfaf 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1568,7 +1568,7 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) pr_debug("CPU %d exiting\n", policy->cpu); } -static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy) +static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; u32 max_perf, min_perf, min_limit_perf, max_limit_perf; @@ -1618,7 +1618,7 @@ static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy) * This return value can only be negative for shared_memory * systems where EPP register read/write not supported. */ - return; + return epp; } if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) @@ -1631,12 +1631,13 @@ static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy) } WRITE_ONCE(cpudata->cppc_req_cached, value); - amd_pstate_set_epp(cpudata, epp); + return amd_pstate_set_epp(cpudata, epp); } static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; + int ret; if (!policy->cpuinfo.max_freq) return -ENODEV; @@ -1646,7 +1647,9 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) cpudata->policy = policy->policy; - amd_pstate_epp_update_limit(policy); + ret = amd_pstate_epp_update_limit(policy); + if (ret) + return ret; /* * policy->cur is never updated with the amd_pstate_epp driver, but it From 940291f7f67ab35c06515c6a8082398b83b82608 Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Thu, 13 Jun 2024 10:26:37 +0200 Subject: [PATCH 035/188] x86/CPU/AMD: Always inline amd_clear_divider() commit 501bd734f933f4eb5c080b87936e9d43f471d723 upstream. The routine is used on syscall exit and on non-AMD CPUs is guaranteed to be empty. It probably does not need to be a function call even on CPUs which do need the mitigation. [ bp: Make sure it is always inlined so that noinstr marking works. ] Signed-off-by: Mateusz Guzik Signed-off-by: Borislav Petkov (AMD) Link: https://lore.kernel.org/r/20240613082637.659133-1-mjguzik@gmail.com Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- arch/x86/include/asm/processor.h | 12 +++++++++++- arch/x86/kernel/cpu/amd.c | 11 ----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index f4f914dc47b59..c0e162fb0670a 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -718,7 +718,17 @@ static inline u32 per_cpu_l2c_id(unsigned int cpu) #ifdef CONFIG_CPU_SUP_AMD extern u32 amd_get_highest_perf(void); -extern void amd_clear_divider(void); + +/* + * Issue a DIV 0/1 insn to clear any division data from previous DIV + * operations. + */ +static __always_inline void amd_clear_divider(void) +{ + asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0) + :: "a" (0), "d" (0), "r" (1)); +} + extern void amd_check_microcode(void); #else static inline u32 amd_get_highest_perf(void) { return 0; } diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 1ace04a7a2595..587712a30b058 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -1281,14 +1281,3 @@ void amd_check_microcode(void) if (cpu_feature_enabled(X86_FEATURE_ZEN2)) on_each_cpu(zenbleed_check_cpu, NULL, 1); } - -/* - * Issue a DIV 0/1 insn to clear any division data from previous DIV - * operations. - */ -void noinstr amd_clear_divider(void) -{ - asm volatile(ALTERNATIVE("", "div %2\n\t", X86_BUG_DIV0) - :: "a" (0), "d" (0), "r" (1)); -} -EXPORT_SYMBOL_GPL(amd_clear_divider); From 20240fa8ab10889d93dd1a8adc9653a33a0a57a7 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 26 Aug 2024 16:13:51 -0500 Subject: [PATCH 036/188] x86/amd: Move amd_get_highest_perf() from amd.c to cppc.c commit 2bcec09cc4ae62229e149213499e45b74190a24a upstream. To prepare to let amd_get_highest_perf() detect preferred cores it will require CPPC functions. Move amd_get_highest_perf() to cppc.c to prepare for 'preferred core detection' rework. No functional changes intended. Reviewed-by: Perry Yuan Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- arch/x86/kernel/acpi/cppc.c | 16 ++++++++++++++++ arch/x86/kernel/cpu/amd.c | 16 ---------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index ff8f25faca3dd..7ec8f2ce859c8 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -116,3 +116,19 @@ void init_freq_invariance_cppc(void) init_done = true; mutex_unlock(&freq_invariance_lock); } + +u32 amd_get_highest_perf(void) +{ + struct cpuinfo_x86 *c = &boot_cpu_data; + + if (c->x86 == 0x17 && ((c->x86_model >= 0x30 && c->x86_model < 0x40) || + (c->x86_model >= 0x70 && c->x86_model < 0x80))) + return 166; + + if (c->x86 == 0x19 && ((c->x86_model >= 0x20 && c->x86_model < 0x30) || + (c->x86_model >= 0x40 && c->x86_model < 0x70))) + return 166; + + return 255; +} +EXPORT_SYMBOL_GPL(amd_get_highest_perf); diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 587712a30b058..48cc0ed3d0300 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -1250,22 +1250,6 @@ unsigned long amd_get_dr_addr_mask(unsigned int dr) } EXPORT_SYMBOL_GPL(amd_get_dr_addr_mask); -u32 amd_get_highest_perf(void) -{ - struct cpuinfo_x86 *c = &boot_cpu_data; - - if (c->x86 == 0x17 && ((c->x86_model >= 0x30 && c->x86_model < 0x40) || - (c->x86_model >= 0x70 && c->x86_model < 0x80))) - return 166; - - if (c->x86 == 0x19 && ((c->x86_model >= 0x20 && c->x86_model < 0x30) || - (c->x86_model >= 0x40 && c->x86_model < 0x70))) - return 166; - - return 255; -} -EXPORT_SYMBOL_GPL(amd_get_highest_perf); - static void zenbleed_check_cpu(void *unused) { struct cpuinfo_x86 *c = &cpu_data(smp_processor_id()); From c6a2561f8326c2d3d1b9fdf25b97b38762a90b89 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 3 Sep 2024 15:32:16 -0500 Subject: [PATCH 037/188] ACPI: CPPC: Adjust return code for inline functions in !CONFIG_ACPI_CPPC_LIB commit 01ced022e125f7b328335bb2944a107afcafe351 upstream. Checkpath emits the following warning: ``` WARNING: ENOTSUPP is not a SUSV4 error code, prefer EOPNOTSUPP ``` Adjust the code accordingly. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- include/acpi/cppc_acpi.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 301e432fb0605..7deae49aa159d 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -163,31 +163,31 @@ extern int cppc_set_auto_sel(int cpu, bool enable); #else /* !CONFIG_ACPI_CPPC_LIB */ static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cppc_get_nominal_perf(int cpunum, u64 *nominal_perf) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cppc_get_highest_perf(int cpunum, u64 *highest_perf) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ctrs) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cppc_set_enable(int cpu, bool enable) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cppc_get_perf_caps(int cpu, struct cppc_perf_caps *caps) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline bool cppc_perf_ctrs_in_pcc(void) { @@ -211,27 +211,27 @@ static inline bool cpc_ffh_supported(void) } static inline int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cppc_get_epp_perf(int cpunum, u64 *epp_perf) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cppc_set_auto_sel(int cpu, bool enable) { - return -ENOTSUPP; + return -EOPNOTSUPP; } static inline int cppc_get_auto_sel_caps(int cpunum, struct cppc_perf_caps *perf_caps) { - return -ENOTSUPP; + return -EOPNOTSUPP; } #endif /* !CONFIG_ACPI_CPPC_LIB */ From a2e21cbd89f68b46ffaada35be378c77ca529d1a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 26 Aug 2024 16:13:52 -0500 Subject: [PATCH 038/188] x86/amd: Rename amd_get_highest_perf() to amd_get_boost_ratio_numerator() commit 6c09e3b445a1a647a5b57ea6afd23e846225dd8f upstream. The function name is ambiguous because it returns an intermediate value for calculating maximum frequency rather than the CPPC 'Highest Perf' register. Rename the function to clarify its use and allow the function to return errors. Adjust the consumer in acpi-cpufreq to catch errors. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- arch/x86/include/asm/processor.h | 3 --- arch/x86/kernel/acpi/cppc.c | 44 +++++++++++++++++++++++--------- drivers/cpufreq/acpi-cpufreq.c | 12 ++++++--- include/acpi/cppc_acpi.h | 5 ++++ 4 files changed, 46 insertions(+), 18 deletions(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index c0e162fb0670a..bcbb57a31b2a8 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -717,8 +717,6 @@ static inline u32 per_cpu_l2c_id(unsigned int cpu) } #ifdef CONFIG_CPU_SUP_AMD -extern u32 amd_get_highest_perf(void); - /* * Issue a DIV 0/1 insn to clear any division data from previous DIV * operations. @@ -731,7 +729,6 @@ static __always_inline void amd_clear_divider(void) extern void amd_check_microcode(void); #else -static inline u32 amd_get_highest_perf(void) { return 0; } static inline void amd_clear_divider(void) { } static inline void amd_check_microcode(void) { } #endif diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index 7ec8f2ce859c8..660cfeb6384ba 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -69,7 +69,7 @@ int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) static void amd_set_max_freq_ratio(void) { struct cppc_perf_caps perf_caps; - u64 highest_perf, nominal_perf; + u64 numerator, nominal_perf; u64 perf_ratio; int rc; @@ -79,15 +79,19 @@ static void amd_set_max_freq_ratio(void) return; } - highest_perf = amd_get_highest_perf(); + rc = amd_get_boost_ratio_numerator(0, &numerator); + if (rc) { + pr_debug("Could not retrieve highest performance (%d)\n", rc); + return; + } nominal_perf = perf_caps.nominal_perf; - if (!highest_perf || !nominal_perf) { - pr_debug("Could not retrieve highest or nominal performance\n"); + if (!nominal_perf) { + pr_debug("Could not retrieve nominal performance\n"); return; } - perf_ratio = div_u64(highest_perf * SCHED_CAPACITY_SCALE, nominal_perf); + perf_ratio = div_u64(numerator * SCHED_CAPACITY_SCALE, nominal_perf); /* midpoint between max_boost and max_P */ perf_ratio = (perf_ratio + SCHED_CAPACITY_SCALE) >> 1; if (!perf_ratio) { @@ -117,18 +121,34 @@ void init_freq_invariance_cppc(void) mutex_unlock(&freq_invariance_lock); } -u32 amd_get_highest_perf(void) +/** + * amd_get_boost_ratio_numerator: Get the numerator to use for boost ratio calculation + * @cpu: CPU to get numerator for. + * @numerator: Output variable for numerator. + * + * Determine the numerator to use for calculating the boost ratio on + * a CPU. On systems that support preferred cores, this will be a hardcoded + * value. On other systems this will the highest performance register value. + * + * Return: 0 for success, negative error code otherwise. + */ +int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) { struct cpuinfo_x86 *c = &boot_cpu_data; if (c->x86 == 0x17 && ((c->x86_model >= 0x30 && c->x86_model < 0x40) || - (c->x86_model >= 0x70 && c->x86_model < 0x80))) - return 166; + (c->x86_model >= 0x70 && c->x86_model < 0x80))) { + *numerator = 166; + return 0; + } if (c->x86 == 0x19 && ((c->x86_model >= 0x20 && c->x86_model < 0x30) || - (c->x86_model >= 0x40 && c->x86_model < 0x70))) - return 166; + (c->x86_model >= 0x40 && c->x86_model < 0x70))) { + *numerator = 166; + return 0; + } + *numerator = 255; - return 255; + return 0; } -EXPORT_SYMBOL_GPL(amd_get_highest_perf); +EXPORT_SYMBOL_GPL(amd_get_boost_ratio_numerator); diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 2be14a4165d52..fae5c2a9de35c 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -649,10 +649,16 @@ static u64 get_max_boost_ratio(unsigned int cpu, u64 *nominal_freq) return 0; } - if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) - highest_perf = amd_get_highest_perf(); - else + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + ret = amd_get_boost_ratio_numerator(cpu, &highest_perf); + if (ret) { + pr_debug("CPU%d: Unable to get boost ratio numerator (%d)\n", + cpu, ret); + return 0; + } + } else { highest_perf = perf_caps.highest_perf; + } nominal_perf = perf_caps.nominal_perf; diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 7deae49aa159d..078403569a647 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -160,6 +160,7 @@ extern int cppc_get_epp_perf(int cpunum, u64 *epp_perf); extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable); extern int cppc_get_auto_sel_caps(int cpunum, struct cppc_perf_caps *perf_caps); extern int cppc_set_auto_sel(int cpu, bool enable); +extern int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator); #else /* !CONFIG_ACPI_CPPC_LIB */ static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) { @@ -233,6 +234,10 @@ static inline int cppc_get_auto_sel_caps(int cpunum, struct cppc_perf_caps *perf { return -EOPNOTSUPP; } +static inline int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) +{ + return -EOPNOTSUPP; +} #endif /* !CONFIG_ACPI_CPPC_LIB */ #endif /* _CPPC_ACPI_H*/ From 5647e28f1832f8f1c4acd6e6136e47b8c511695a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 26 Aug 2024 16:13:54 -0500 Subject: [PATCH 039/188] x86/amd: Move amd_get_highest_perf() out of amd-pstate commit 2819bfef6483c66c55064ca678f2630a1a09f3f9 upstream. amd_pstate_get_highest_perf() is a helper used to get the highest perf value on AMD systems. It's used in amd-pstate as part of preferred core handling, but applicable for acpi-cpufreq as well. Move it out to cppc handling code as amd_get_highest_perf(). Reviewed-by: Perry Yuan Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- arch/x86/kernel/acpi/cppc.c | 30 ++++++++++++++++++++++++++++++ drivers/cpufreq/amd-pstate.c | 34 ++-------------------------------- include/acpi/cppc_acpi.h | 5 +++++ 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index 660cfeb6384ba..951830aa1e807 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -121,6 +121,36 @@ void init_freq_invariance_cppc(void) mutex_unlock(&freq_invariance_lock); } +/* + * Get the highest performance register value. + * @cpu: CPU from which to get highest performance. + * @highest_perf: Return address for highest performance value. + * + * Return: 0 for success, negative error code otherwise. + */ +int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf) +{ + u64 val; + int ret; + + if (cpu_feature_enabled(X86_FEATURE_CPPC)) { + ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &val); + if (ret) + goto out; + + val = AMD_CPPC_HIGHEST_PERF(val); + } else { + ret = cppc_get_highest_perf(cpu, &val); + if (ret) + goto out; + } + + WRITE_ONCE(*highest_perf, (u32)val); +out: + return ret; +} +EXPORT_SYMBOL_GPL(amd_get_highest_perf); + /** * amd_get_boost_ratio_numerator: Get the numerator to use for boost ratio calculation * @cpu: CPU to get numerator for. diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 8b3b68563bfaf..2705d415ffd11 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -810,36 +810,6 @@ static void amd_pstste_sched_prefcore_workfn(struct work_struct *work) } static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn); -/* - * Get the highest performance register value. - * @cpu: CPU from which to get highest performance. - * @highest_perf: Return address. - * - * Return: 0 for success, -EIO otherwise. - */ -static int amd_pstate_get_highest_perf(int cpu, u32 *highest_perf) -{ - int ret; - - if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - u64 cap1; - - ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); - if (ret) - return ret; - WRITE_ONCE(*highest_perf, AMD_CPPC_HIGHEST_PERF(cap1)); - } else { - u64 cppc_highest_perf; - - ret = cppc_get_highest_perf(cpu, &cppc_highest_perf); - if (ret) - return ret; - WRITE_ONCE(*highest_perf, cppc_highest_perf); - } - - return (ret); -} - #define CPPC_MAX_PERF U8_MAX static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) @@ -847,7 +817,7 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) int ret, prio; u32 highest_perf; - ret = amd_pstate_get_highest_perf(cpudata->cpu, &highest_perf); + ret = amd_get_highest_perf(cpudata->cpu, &highest_perf); if (ret) return; @@ -886,7 +856,7 @@ static void amd_pstate_update_limits(unsigned int cpu) if ((!amd_pstate_prefcore) || (!cpudata->hw_prefcore)) goto free_cpufreq_put; - ret = amd_pstate_get_highest_perf(cpu, &cur_high); + ret = amd_get_highest_perf(cpu, &cur_high); if (ret) goto free_cpufreq_put; diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 078403569a647..1715e85459451 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -160,6 +160,7 @@ extern int cppc_get_epp_perf(int cpunum, u64 *epp_perf); extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable); extern int cppc_get_auto_sel_caps(int cpunum, struct cppc_perf_caps *perf_caps); extern int cppc_set_auto_sel(int cpu, bool enable); +extern int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf); extern int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator); #else /* !CONFIG_ACPI_CPPC_LIB */ static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) @@ -234,6 +235,10 @@ static inline int cppc_get_auto_sel_caps(int cpunum, struct cppc_perf_caps *perf { return -EOPNOTSUPP; } +static inline int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf) +{ + return -ENODEV; +} static inline int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) { return -EOPNOTSUPP; From 2a5c8e2cba2cb17c0fe732425bc4376646da65a4 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 26 Aug 2024 16:13:55 -0500 Subject: [PATCH 040/188] x86/amd: Detect preferred cores in amd_get_boost_ratio_numerator() commit 279f838a61f96cbfeb1f9ba060e4a452e6e041d0 upstream. AMD systems that support preferred cores will use "166" as their numerator for max frequency calculations instead of "255". Add a function for detecting preferred cores by looking at the highest perf value on all cores. If preferred cores are enabled return 166 and if disabled the value in the highest perf register. As the function will be called multiple times, cache the values for the boost numerator and if preferred cores will be enabled in global variables. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- arch/x86/kernel/acpi/cppc.c | 93 ++++++++++++++++++++++++++++++++---- drivers/cpufreq/amd-pstate.c | 34 +++++-------- include/acpi/cppc_acpi.h | 5 ++ 3 files changed, 101 insertions(+), 31 deletions(-) diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index 951830aa1e807..c29fc185fa1f5 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -9,6 +9,16 @@ #include #include +#define CPPC_HIGHEST_PERF_PREFCORE 166 + +enum amd_pref_core { + AMD_PREF_CORE_UNKNOWN = 0, + AMD_PREF_CORE_SUPPORTED, + AMD_PREF_CORE_UNSUPPORTED, +}; +static enum amd_pref_core amd_pref_core_detected; +static u64 boost_numerator; + /* Refer to drivers/acpi/cppc_acpi.c for the description of functions */ bool cpc_supported_by_cpu(void) @@ -151,6 +161,66 @@ int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf) } EXPORT_SYMBOL_GPL(amd_get_highest_perf); +/** + * amd_detect_prefcore: Detect if CPUs in the system support preferred cores + * @detected: Output variable for the result of the detection. + * + * Determine whether CPUs in the system support preferred cores. On systems + * that support preferred cores, different highest perf values will be found + * on different cores. On other systems, the highest perf value will be the + * same on all cores. + * + * The result of the detection will be stored in the 'detected' parameter. + * + * Return: 0 for success, negative error code otherwise + */ +int amd_detect_prefcore(bool *detected) +{ + int cpu, count = 0; + u64 highest_perf[2] = {0}; + + if (WARN_ON(!detected)) + return -EINVAL; + + switch (amd_pref_core_detected) { + case AMD_PREF_CORE_SUPPORTED: + *detected = true; + return 0; + case AMD_PREF_CORE_UNSUPPORTED: + *detected = false; + return 0; + default: + break; + } + + for_each_present_cpu(cpu) { + u32 tmp; + int ret; + + ret = amd_get_highest_perf(cpu, &tmp); + if (ret) + return ret; + + if (!count || (count == 1 && tmp != highest_perf[0])) + highest_perf[count++] = tmp; + + if (count == 2) + break; + } + + *detected = (count == 2); + boost_numerator = highest_perf[0]; + + amd_pref_core_detected = *detected ? AMD_PREF_CORE_SUPPORTED : + AMD_PREF_CORE_UNSUPPORTED; + + pr_debug("AMD CPPC preferred core is %ssupported (highest perf: 0x%llx)\n", + *detected ? "" : "un", highest_perf[0]); + + return 0; +} +EXPORT_SYMBOL_GPL(amd_detect_prefcore); + /** * amd_get_boost_ratio_numerator: Get the numerator to use for boost ratio calculation * @cpu: CPU to get numerator for. @@ -160,24 +230,27 @@ EXPORT_SYMBOL_GPL(amd_get_highest_perf); * a CPU. On systems that support preferred cores, this will be a hardcoded * value. On other systems this will the highest performance register value. * + * If booting the system with amd-pstate enabled but preferred cores disabled then + * the correct boost numerator will be returned to match hardware capabilities + * even if the preferred cores scheduling hints are not enabled. + * * Return: 0 for success, negative error code otherwise. */ int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) { - struct cpuinfo_x86 *c = &boot_cpu_data; + bool prefcore; + int ret; - if (c->x86 == 0x17 && ((c->x86_model >= 0x30 && c->x86_model < 0x40) || - (c->x86_model >= 0x70 && c->x86_model < 0x80))) { - *numerator = 166; - return 0; - } + ret = amd_detect_prefcore(&prefcore); + if (ret) + return ret; - if (c->x86 == 0x19 && ((c->x86_model >= 0x20 && c->x86_model < 0x30) || - (c->x86_model >= 0x40 && c->x86_model < 0x70))) { - *numerator = 166; + /* without preferred cores, return the highest perf register value */ + if (!prefcore) { + *numerator = boost_numerator; return 0; } - *numerator = 255; + *numerator = CPPC_HIGHEST_PERF_PREFCORE; return 0; } diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 2705d415ffd11..6d745e22a92d3 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -814,32 +814,18 @@ static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn); static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) { - int ret, prio; - u32 highest_perf; - - ret = amd_get_highest_perf(cpudata->cpu, &highest_perf); - if (ret) + /* user disabled or not detected */ + if (!amd_pstate_prefcore) return; cpudata->hw_prefcore = true; - /* check if CPPC preferred core feature is enabled*/ - if (highest_perf < CPPC_MAX_PERF) - prio = (int)highest_perf; - else { - pr_debug("AMD CPPC preferred core is unsupported!\n"); - cpudata->hw_prefcore = false; - return; - } - - if (!amd_pstate_prefcore) - return; /* * The priorities can be set regardless of whether or not * sched_set_itmt_support(true) has been called and it is valid to * update them at any time after it has been called. */ - sched_set_itmt_core_prio(prio, cpudata->cpu); + sched_set_itmt_core_prio((int)READ_ONCE(cpudata->highest_perf), cpudata->cpu); schedule_work(&sched_prefcore_work); } @@ -1001,12 +987,12 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) cpudata->cpu = policy->cpu; - amd_pstate_init_prefcore(cpudata); - ret = amd_pstate_init_perf(cpudata); if (ret) goto free_cpudata1; + amd_pstate_init_prefcore(cpudata); + ret = amd_pstate_init_freq(cpudata); if (ret) goto free_cpudata1; @@ -1466,12 +1452,12 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) cpudata->cpu = policy->cpu; cpudata->epp_policy = 0; - amd_pstate_init_prefcore(cpudata); - ret = amd_pstate_init_perf(cpudata); if (ret) goto free_cpudata1; + amd_pstate_init_prefcore(cpudata); + ret = amd_pstate_init_freq(cpudata); if (ret) goto free_cpudata1; @@ -1933,6 +1919,12 @@ static int __init amd_pstate_init(void) static_call_update(amd_pstate_update_perf, cppc_update_perf); } + if (amd_pstate_prefcore) { + ret = amd_detect_prefcore(&amd_pstate_prefcore); + if (ret) + return ret; + } + /* enable amd pstate feature */ ret = amd_pstate_enable(true); if (ret) { diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 1715e85459451..867765770e7ce 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -162,6 +162,7 @@ extern int cppc_get_auto_sel_caps(int cpunum, struct cppc_perf_caps *perf_caps); extern int cppc_set_auto_sel(int cpu, bool enable); extern int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf); extern int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator); +extern int amd_detect_prefcore(bool *detected); #else /* !CONFIG_ACPI_CPPC_LIB */ static inline int cppc_get_desired_perf(int cpunum, u64 *desired_perf) { @@ -243,6 +244,10 @@ static inline int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator { return -EOPNOTSUPP; } +static inline int amd_detect_prefcore(bool *detected) +{ + return -ENODEV; +} #endif /* !CONFIG_ACPI_CPPC_LIB */ #endif /* _CPPC_ACPI_H*/ From 6aaa927c5a160d37beef15f7c014029cca776518 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 26 Aug 2024 16:13:56 -0500 Subject: [PATCH 041/188] cpufreq: amd-pstate: Merge amd_pstate_highest_perf_set() into amd_get_boost_ratio_numerator() commit ad4caad58d91d3293880f8074f7ad125490ce636 upstream. The special case in amd_pstate_highest_perf_set() is the value used for calculating the boost numerator. Merge this into amd_get_boost_ratio_numerator() and then use that to calculate boost ratio. This allows dropping more special casing of the highest perf value. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 3 +- arch/x86/kernel/acpi/cppc.c | 16 +++++++ drivers/cpufreq/amd-pstate.c | 50 +++++---------------- 3 files changed, 28 insertions(+), 41 deletions(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 1cf40f69278cd..573c79184383c 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -251,7 +251,8 @@ performance supported in `AMD CPPC Performance Capability `_). In some ASICs, the highest CPPC performance is not the one in the ``_CPC`` table, so we need to expose it to sysfs. If boost is not active, but still supported, this maximum frequency will be larger than the one in -``cpuinfo``. +``cpuinfo``. On systems that support preferred core, the driver will have +different values for some cores than others. This attribute is read-only. ``amd_pstate_lowest_nonlinear_freq`` diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index c29fc185fa1f5..ed0290a8f4de9 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -9,6 +9,7 @@ #include #include +#define CPPC_HIGHEST_PERF_PERFORMANCE 196 #define CPPC_HIGHEST_PERF_PREFCORE 166 enum amd_pref_core { @@ -250,6 +251,21 @@ int amd_get_boost_ratio_numerator(unsigned int cpu, u64 *numerator) *numerator = boost_numerator; return 0; } + + /* + * For AMD CPUs with Family ID 19H and Model ID range 0x70 to 0x7f, + * the highest performance level is set to 196. + * https://bugzilla.kernel.org/show_bug.cgi?id=218759 + */ + if (cpu_feature_enabled(X86_FEATURE_ZEN4)) { + switch (boot_cpu_data.x86_model) { + case 0x70 ... 0x7f: + *numerator = CPPC_HIGHEST_PERF_PERFORMANCE; + return 0; + default: + break; + } + } *numerator = CPPC_HIGHEST_PERF_PREFCORE; return 0; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 6d745e22a92d3..922e019ddf173 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -371,43 +371,17 @@ static inline int amd_pstate_enable(bool enable) return static_call(amd_pstate_enable)(enable); } -static u32 amd_pstate_highest_perf_set(struct amd_cpudata *cpudata) -{ - struct cpuinfo_x86 *c = &cpu_data(0); - - /* - * For AMD CPUs with Family ID 19H and Model ID range 0x70 to 0x7f, - * the highest performance level is set to 196. - * https://bugzilla.kernel.org/show_bug.cgi?id=218759 - */ - if (c->x86 == 0x19 && (c->x86_model >= 0x70 && c->x86_model <= 0x7f)) - return CPPC_HIGHEST_PERF_PERFORMANCE; - - return CPPC_HIGHEST_PERF_DEFAULT; -} - static int pstate_init_perf(struct amd_cpudata *cpudata) { u64 cap1; - u32 highest_perf; int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &cap1); if (ret) return ret; - /* For platforms that do not support the preferred core feature, the - * highest_pef may be configured with 166 or 255, to avoid max frequency - * calculated wrongly. we take the AMD_CPPC_HIGHEST_PERF(cap1) value as - * the default max perf. - */ - if (cpudata->hw_prefcore) - highest_perf = amd_pstate_highest_perf_set(cpudata); - else - highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); - - WRITE_ONCE(cpudata->highest_perf, highest_perf); - WRITE_ONCE(cpudata->max_limit_perf, highest_perf); + WRITE_ONCE(cpudata->highest_perf, AMD_CPPC_HIGHEST_PERF(cap1)); + WRITE_ONCE(cpudata->max_limit_perf, AMD_CPPC_HIGHEST_PERF(cap1)); WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); @@ -419,19 +393,13 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) static int cppc_init_perf(struct amd_cpudata *cpudata) { struct cppc_perf_caps cppc_perf; - u32 highest_perf; int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); if (ret) return ret; - if (cpudata->hw_prefcore) - highest_perf = amd_pstate_highest_perf_set(cpudata); - else - highest_perf = cppc_perf.highest_perf; - - WRITE_ONCE(cpudata->highest_perf, highest_perf); - WRITE_ONCE(cpudata->max_limit_perf, highest_perf); + WRITE_ONCE(cpudata->highest_perf, cppc_perf.highest_perf); + WRITE_ONCE(cpudata->max_limit_perf, cppc_perf.highest_perf); WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); WRITE_ONCE(cpudata->lowest_nonlinear_perf, cppc_perf.lowest_nonlinear_perf); @@ -908,8 +876,8 @@ static u32 amd_pstate_get_transition_latency(unsigned int cpu) static int amd_pstate_init_freq(struct amd_cpudata *cpudata) { int ret; - u32 min_freq; - u32 highest_perf, max_freq; + u32 min_freq, max_freq; + u64 numerator; u32 nominal_perf, nominal_freq; u32 lowest_nonlinear_perf, lowest_nonlinear_freq; u32 boost_ratio, lowest_nonlinear_ratio; @@ -931,8 +899,10 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) nominal_perf = READ_ONCE(cpudata->nominal_perf); - highest_perf = READ_ONCE(cpudata->highest_perf); - boost_ratio = div_u64(highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); + ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); + if (ret) + return ret; + boost_ratio = div_u64(numerator << SCHED_CAPACITY_SHIFT, nominal_perf); max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT) * 1000; lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); From 873a7ab3dd562045eb37184abc1581966f63db6d Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 26 Aug 2024 16:13:57 -0500 Subject: [PATCH 042/188] cpufreq: amd-pstate: Optimize amd_pstate_update_limits() commit 45722e777fd99ea863fe653c1838d39f678506e2 upstream. Don't take and release the mutex when prefcore isn't present and avoid initialization of variables that will be initially set in the function. Reviewed-by: Gautham R. Shenoy Reviewed-by: Perry Yuan Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 922e019ddf173..ddabf8c8bd5ac 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -806,17 +806,17 @@ static void amd_pstate_update_limits(unsigned int cpu) int ret; bool highest_perf_changed = false; - mutex_lock(&amd_pstate_driver_lock); - if ((!amd_pstate_prefcore) || (!cpudata->hw_prefcore)) - goto free_cpufreq_put; + if (!amd_pstate_prefcore) + return; + mutex_lock(&amd_pstate_driver_lock); ret = amd_get_highest_perf(cpu, &cur_high); if (ret) goto free_cpufreq_put; prev_high = READ_ONCE(cpudata->prefcore_ranking); - if (prev_high != cur_high) { - highest_perf_changed = true; + highest_perf_changed = (prev_high != cur_high); + if (highest_perf_changed) { WRITE_ONCE(cpudata->prefcore_ranking, cur_high); if (cur_high < CPPC_MAX_PERF) From e027431dfe9542721324c15c2686fca3952b8667 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 31 Aug 2024 21:49:11 -0500 Subject: [PATCH 043/188] cpufreq/amd-pstate: Export symbols for changing modes commit 8d916815b0afad2a12a9f1b945c79fffc144dba8 upstream. In order to effectively test all mode switch combinations export everything necessarily for amd-pstate-ut to trigger a mode switch. Reviewed-by: Perry Yuan Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 23 ++++++++++------------- drivers/cpufreq/amd-pstate.h | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index ddabf8c8bd5ac..2be964d8261ae 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -59,18 +59,6 @@ #define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF #define AMD_CPPC_EPP_POWERSAVE 0xFF -/* - * enum amd_pstate_mode - driver working mode of amd pstate - */ -enum amd_pstate_mode { - AMD_PSTATE_UNDEFINED = 0, - AMD_PSTATE_DISABLE, - AMD_PSTATE_PASSIVE, - AMD_PSTATE_ACTIVE, - AMD_PSTATE_GUIDED, - AMD_PSTATE_MAX, -}; - static const char * const amd_pstate_mode_string[] = { [AMD_PSTATE_UNDEFINED] = "undefined", [AMD_PSTATE_DISABLE] = "disable", @@ -80,6 +68,14 @@ static const char * const amd_pstate_mode_string[] = { NULL, }; +const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode) +{ + if (mode < 0 || mode >= AMD_PSTATE_MAX) + return NULL; + return amd_pstate_mode_string[mode]; +} +EXPORT_SYMBOL_GPL(amd_pstate_get_mode_string); + struct quirk_entry { u32 nominal_freq; u32 lowest_freq; @@ -1288,7 +1284,7 @@ static ssize_t amd_pstate_show_status(char *buf) return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); } -static int amd_pstate_update_status(const char *buf, size_t size) +int amd_pstate_update_status(const char *buf, size_t size) { int mode_idx; @@ -1305,6 +1301,7 @@ static int amd_pstate_update_status(const char *buf, size_t size) return 0; } +EXPORT_SYMBOL_GPL(amd_pstate_update_status); static ssize_t status_show(struct device *dev, struct device_attribute *attr, char *buf) diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index cc8bb2bc325aa..cd573bc6b6db8 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -103,4 +103,18 @@ struct amd_cpudata { bool boost_state; }; +/* + * enum amd_pstate_mode - driver working mode of amd pstate + */ +enum amd_pstate_mode { + AMD_PSTATE_UNDEFINED = 0, + AMD_PSTATE_DISABLE, + AMD_PSTATE_PASSIVE, + AMD_PSTATE_ACTIVE, + AMD_PSTATE_GUIDED, + AMD_PSTATE_MAX, +}; +const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode); +int amd_pstate_update_status(const char *buf, size_t size); + #endif /* _LINUX_AMD_PSTATE_H */ From 70656fdd6b8e25969d5252d085b6ab3d687656d1 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 31 Aug 2024 21:49:12 -0500 Subject: [PATCH 044/188] cpufreq/amd-pstate-ut: Add test case for mode switches commit 3e39e68dfbb6d4fe8c1943003d2f2a6f3255c902 upstream. There is a state machine in the amd-pstate driver utilized for switches for all modes. To make sure that cleanup and setup works properly for each mode add a unit test case that tries all combinations. Reviewed-by: Perry Yuan Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 41 ++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index b7318669485e4..c291b3dbec381 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -54,12 +54,14 @@ static void amd_pstate_ut_acpi_cpc_valid(u32 index); static void amd_pstate_ut_check_enabled(u32 index); static void amd_pstate_ut_check_perf(u32 index); static void amd_pstate_ut_check_freq(u32 index); +static void amd_pstate_ut_check_driver(u32 index); static struct amd_pstate_ut_struct amd_pstate_ut_cases[] = { {"amd_pstate_ut_acpi_cpc_valid", amd_pstate_ut_acpi_cpc_valid }, {"amd_pstate_ut_check_enabled", amd_pstate_ut_check_enabled }, {"amd_pstate_ut_check_perf", amd_pstate_ut_check_perf }, - {"amd_pstate_ut_check_freq", amd_pstate_ut_check_freq } + {"amd_pstate_ut_check_freq", amd_pstate_ut_check_freq }, + {"amd_pstate_ut_check_driver", amd_pstate_ut_check_driver } }; static bool get_shared_mem(void) @@ -257,6 +259,43 @@ static void amd_pstate_ut_check_freq(u32 index) cpufreq_cpu_put(policy); } +static int amd_pstate_set_mode(enum amd_pstate_mode mode) +{ + const char *mode_str = amd_pstate_get_mode_string(mode); + + pr_debug("->setting mode to %s\n", mode_str); + + return amd_pstate_update_status(mode_str, strlen(mode_str)); +} + +static void amd_pstate_ut_check_driver(u32 index) +{ + enum amd_pstate_mode mode1, mode2; + int ret; + + for (mode1 = AMD_PSTATE_DISABLE; mode1 < AMD_PSTATE_MAX; mode1++) { + ret = amd_pstate_set_mode(mode1); + if (ret) + goto out; + for (mode2 = AMD_PSTATE_DISABLE; mode2 < AMD_PSTATE_MAX; mode2++) { + if (mode1 == mode2) + continue; + ret = amd_pstate_set_mode(mode2); + if (ret) + goto out; + } + } +out: + if (ret) + pr_warn("%s: failed to update status for %s->%s: %d\n", __func__, + amd_pstate_get_mode_string(mode1), + amd_pstate_get_mode_string(mode2), ret); + + amd_pstate_ut_cases[index].result = ret ? + AMD_PSTATE_UT_RESULT_FAIL : + AMD_PSTATE_UT_RESULT_PASS; +} + static int __init amd_pstate_ut_init(void) { u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases); From abe08e0ab37fdc93dbd0c391aee0b4a5b25ad5b3 Mon Sep 17 00:00:00 2001 From: Qianqiang Liu Date: Wed, 11 Sep 2024 07:39:24 +0800 Subject: [PATCH 045/188] cpufreq/amd-pstate-ut: Fix an "Uninitialized variables" issue commit 93497752dfed196b41d2804503e80b9a04318adb upstream. Using uninitialized value "mode2" when calling "amd_pstate_get_mode_string". Set "mode2" to "AMD_PSTATE_DISABLE" by default. Signed-off-by: Qianqiang Liu Link: https://lore.kernel.org/r/20240910233923.46470-1-qianqiang.liu@163.com Acked-by: Mario Limonciello Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index c291b3dbec381..f66701514d906 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -270,7 +270,7 @@ static int amd_pstate_set_mode(enum amd_pstate_mode mode) static void amd_pstate_ut_check_driver(u32 index) { - enum amd_pstate_mode mode1, mode2; + enum amd_pstate_mode mode1, mode2 = AMD_PSTATE_DISABLE; int ret; for (mode1 = AMD_PSTATE_DISABLE; mode1 < AMD_PSTATE_MAX; mode1++) { From 5a68ce04828d33a45be58bc1fed0fd183454194c Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 12 Oct 2024 12:45:16 -0500 Subject: [PATCH 046/188] cpufreq/amd-pstate: Use nominal perf for limits when boost is disabled commit 18d9b52271213890da295a7c63ef8880ed570cd8 upstream. When boost has been disabled the limit for perf should be nominal perf not the highest perf. Using the latter to do calculations will lead to incorrect values that are still above nominal. Fixes: ad4caad58d91 ("cpufreq: amd-pstate: Merge amd_pstate_highest_perf_set() into amd_get_boost_ratio_numerator()") Reported-by: Peter Jung Closes: https://bugzilla.kernel.org/show_bug.cgi?id=219348 Reviewed-by: Perry Yuan Reviewed-by: Gautham R. Shenoy Tested-by: Dhananjay Ugwekar Link: https://lore.kernel.org/r/20241012174519.897-1-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 2be964d8261ae..f7e566244d703 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -537,11 +537,16 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy) static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) { - u32 max_limit_perf, min_limit_perf, lowest_perf; + u32 max_limit_perf, min_limit_perf, lowest_perf, max_perf; struct amd_cpudata *cpudata = policy->driver_data; - max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); - min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); + if (cpudata->boost_supported && !policy->boost_enabled) + max_perf = READ_ONCE(cpudata->nominal_perf); + else + max_perf = READ_ONCE(cpudata->highest_perf); + + max_limit_perf = div_u64(policy->max * max_perf, policy->cpuinfo.max_freq); + min_limit_perf = div_u64(policy->min * max_perf, policy->cpuinfo.max_freq); lowest_perf = READ_ONCE(cpudata->lowest_perf); if (min_limit_perf < lowest_perf) @@ -1498,10 +1503,13 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) u64 value; s16 epp; - max_perf = READ_ONCE(cpudata->highest_perf); + if (cpudata->boost_supported && !policy->boost_enabled) + max_perf = READ_ONCE(cpudata->nominal_perf); + else + max_perf = READ_ONCE(cpudata->highest_perf); min_perf = READ_ONCE(cpudata->lowest_perf); - max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); - min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); + max_limit_perf = div_u64(policy->max * max_perf, policy->cpuinfo.max_freq); + min_limit_perf = div_u64(policy->min * max_perf, policy->cpuinfo.max_freq); if (min_limit_perf < min_perf) min_limit_perf = min_perf; From cade13ac0655c7a834d62e1f7711b385f3bd740a Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Tue, 17 Sep 2024 09:14:35 +0000 Subject: [PATCH 047/188] cpufreq/amd-pstate: Rename MSR and shared memory specific functions commit 1bfe6a54d2fd979d86cc830be082834da81fa421 upstream. Existing function names "cppc_*" and "pstate_*" for shared memory and MSR based systems are not intuitive enough, replace them with "shmem_*" and "msr_*" respectively. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20240917091434.10685-1-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: TauqirAzam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index f7e566244d703..71226396b2fe2 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -234,7 +234,7 @@ static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) return index; } -static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, +static void msr_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, u32 max_perf, bool fast_switch) { if (fast_switch) @@ -244,7 +244,7 @@ static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, READ_ONCE(cpudata->cppc_req_cached)); } -DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); +DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, @@ -307,7 +307,7 @@ static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, return ret; } -static inline int pstate_enable(bool enable) +static inline int msr_enable(bool enable) { int ret, cpu; unsigned long logical_proc_id_mask = 0; @@ -333,7 +333,7 @@ static inline int pstate_enable(bool enable) return 0; } -static int cppc_enable(bool enable) +static int shmem_enable(bool enable) { int cpu, ret = 0; struct cppc_perf_ctrls perf_ctrls; @@ -360,14 +360,14 @@ static int cppc_enable(bool enable) return ret; } -DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); +DEFINE_STATIC_CALL(amd_pstate_enable, msr_enable); static inline int amd_pstate_enable(bool enable) { return static_call(amd_pstate_enable)(enable); } -static int pstate_init_perf(struct amd_cpudata *cpudata) +static int msr_init_perf(struct amd_cpudata *cpudata) { u64 cap1; @@ -386,7 +386,7 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) return 0; } -static int cppc_init_perf(struct amd_cpudata *cpudata) +static int shmem_init_perf(struct amd_cpudata *cpudata) { struct cppc_perf_caps cppc_perf; @@ -421,14 +421,14 @@ static int cppc_init_perf(struct amd_cpudata *cpudata) return ret; } -DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); +DEFINE_STATIC_CALL(amd_pstate_init_perf, msr_init_perf); static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) { return static_call(amd_pstate_init_perf)(cpudata); } -static void cppc_update_perf(struct amd_cpudata *cpudata, +static void shmem_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, u32 max_perf, bool fast_switch) { @@ -1889,9 +1889,9 @@ static int __init amd_pstate_init(void) current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; } else { pr_debug("AMD CPPC shared memory based functionality is supported\n"); - static_call_update(amd_pstate_enable, cppc_enable); - static_call_update(amd_pstate_init_perf, cppc_init_perf); - static_call_update(amd_pstate_update_perf, cppc_update_perf); + static_call_update(amd_pstate_enable, shmem_enable); + static_call_update(amd_pstate_init_perf, shmem_init_perf); + static_call_update(amd_pstate_update_perf, shmem_update_perf); } if (amd_pstate_prefcore) { From a813f28b3fcfcfc2d878dd2626ec9ffbafdeceb4 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 5 Sep 2024 11:23:51 -0500 Subject: [PATCH 048/188] cpufreq/amd-pstate: Fix non kerneldoc comment commit 7820e8050d622b9bc9bf9acbe8a60f62888a135e upstream. The comment for amd_cppc_supported() isn't meant to be kernel doc. Fixes: cb817ec6673b7 ("cpufreq: amd-pstate: show CPPC debug message if CPPC is not supported") Link: https://lore.kernel.org/r/20240905162351.1345560-1-superm1@kernel.org Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 71226396b2fe2..c950b0b01f554 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1766,7 +1766,7 @@ static int __init amd_pstate_set_driver(int mode_idx) return -EINVAL; } -/** +/* * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. * show the debug message that helps to check if the CPU has CPPC support for loading issue. */ From 3b0431c74409fb35c076b9da59aa8ce8a89ae71d Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 12 Oct 2024 12:45:17 -0500 Subject: [PATCH 049/188] cpufreq/amd-pstate: Don't update CPPC request in amd_pstate_cpu_boost_update() commit 67c08d303e0a1a5665b3f198037c9fae2d808090 upstream. When boost is changed the CPPC value is changed in amd_pstate_cpu_boost_update() but then changed again when refresh_frequency_limits() and all it's callbacks occur. The first is a pointless write, so instead just update the limits for the policy and let the policy refresh anchor everything properly. Fixes: c8c68c38b56f ("cpufreq: amd-pstate: initialize core precision boost state") Reviewed-by: Gautham R. Shenoy Reviewed-by: Perry Yuan Tested-by: Dhananjay Ugwekar Link: https://lore.kernel.org/r/20241012174519.897-2-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index c950b0b01f554..90fa0c1f87b94 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -666,34 +666,12 @@ static void amd_pstate_adjust_perf(unsigned int cpu, static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) { struct amd_cpudata *cpudata = policy->driver_data; - struct cppc_perf_ctrls perf_ctrls; - u32 highest_perf, nominal_perf, nominal_freq, max_freq; + u32 nominal_freq, max_freq; int ret = 0; - highest_perf = READ_ONCE(cpudata->highest_perf); - nominal_perf = READ_ONCE(cpudata->nominal_perf); nominal_freq = READ_ONCE(cpudata->nominal_freq); max_freq = READ_ONCE(cpudata->max_freq); - if (boot_cpu_has(X86_FEATURE_CPPC)) { - u64 value = READ_ONCE(cpudata->cppc_req_cached); - - value &= ~GENMASK_ULL(7, 0); - value |= on ? highest_perf : nominal_perf; - WRITE_ONCE(cpudata->cppc_req_cached, value); - - wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); - } else { - perf_ctrls.max_perf = on ? highest_perf : nominal_perf; - ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); - if (ret) { - cpufreq_cpu_release(policy); - pr_debug("Failed to set max perf on CPU:%d. ret:%d\n", - cpudata->cpu, ret); - return ret; - } - } - if (on) policy->cpuinfo.max_freq = max_freq; else if (policy->cpuinfo.max_freq > nominal_freq * 1000) From 88a3be8b4dbfdf506ed24acf871904389dc9546d Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 12 Oct 2024 12:45:18 -0500 Subject: [PATCH 050/188] cpufreq/amd-pstate: Use amd_pstate_update_min_max_limit() for EPP limits commit 047a2d0c83a425e83aa58b43c2557c42fdd0af29 upstream. When the EPP updates are set the maximum capable frequency for the CPU is used to set the upper limit instead of that of the policy. Adjust amd_pstate_epp_update_limit() to reuse policy calculation code from amd_pstate_update_min_max_limit(). Reviewed-by: Gautham R. Shenoy Reviewed-by: Perry Yuan Tested-by: Dhananjay Ugwekar Link: https://lore.kernel.org/r/20241012174519.897-3-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 90fa0c1f87b94..e479510e992ba 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1477,26 +1477,13 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - u32 max_perf, min_perf, min_limit_perf, max_limit_perf; + u32 max_perf, min_perf; u64 value; s16 epp; - if (cpudata->boost_supported && !policy->boost_enabled) - max_perf = READ_ONCE(cpudata->nominal_perf); - else - max_perf = READ_ONCE(cpudata->highest_perf); + max_perf = READ_ONCE(cpudata->highest_perf); min_perf = READ_ONCE(cpudata->lowest_perf); - max_limit_perf = div_u64(policy->max * max_perf, policy->cpuinfo.max_freq); - min_limit_perf = div_u64(policy->min * max_perf, policy->cpuinfo.max_freq); - - if (min_limit_perf < min_perf) - min_limit_perf = min_perf; - - if (max_limit_perf < min_limit_perf) - max_limit_perf = min_limit_perf; - - WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); - WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); + amd_pstate_update_min_max_limit(policy); max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, cpudata->max_limit_perf); From ab840fc2a592d2e159b424441961426055006158 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 12 Oct 2024 12:45:19 -0500 Subject: [PATCH 051/188] cpufreq/amd-pstate: Drop needless EPP initialization commit 508239724bdddf34482c4d134262f2292c1c78f0 upstream. The EPP value doesn't need to be cached to the CPPC request in amd_pstate_epp_update_limit() because it's passed as an argument at the end to amd_pstate_set_epp() and stored at that time. Reviewed-by: Gautham R. Shenoy Reviewed-by: Perry Yuan Tested-by: Dhananjay Ugwekar Link: https://lore.kernel.org/r/20241012174519.897-4-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index e479510e992ba..203f0043d8fae 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1520,12 +1520,6 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) epp = 0; - /* Set initial EPP value */ - if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - value &= ~GENMASK_ULL(31, 24); - value |= (u64)epp << 24; - } - WRITE_ONCE(cpudata->cppc_req_cached, value); return amd_pstate_set_epp(cpudata, epp); } From d934e44b0edd87e8808c24ff1e3bdbb2f1030f2b Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Thu, 17 Oct 2024 05:39:28 +0000 Subject: [PATCH 052/188] cpufreq/amd-pstate: Remove the redundant verify() function commit 205cb215d08748fdd8f3fd67bc7aa626d4e278b0 upstream. Merge the two verify() callback functions and rename the cpufreq_policy_data argument for better readability. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241017053927.25285-2-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 203f0043d8fae..11e4fd32bf709 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -528,10 +528,10 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, cpufreq_cpu_put(policy); } -static int amd_pstate_verify(struct cpufreq_policy_data *policy) +static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) { - cpufreq_verify_within_cpu_limits(policy); - + cpufreq_verify_within_cpu_limits(policy_data); + pr_debug("policy_max =%d, policy_min=%d\n", policy_data->max, policy_data->min); return 0; } @@ -1630,13 +1630,6 @@ static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) return 0; } -static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) -{ - cpufreq_verify_within_cpu_limits(policy); - pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); - return 0; -} - static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; @@ -1692,7 +1685,7 @@ static struct cpufreq_driver amd_pstate_driver = { static struct cpufreq_driver amd_pstate_epp_driver = { .flags = CPUFREQ_CONST_LOOPS, - .verify = amd_pstate_epp_verify_policy, + .verify = amd_pstate_verify, .setpolicy = amd_pstate_epp_set_policy, .init = amd_pstate_epp_cpu_init, .exit = amd_pstate_epp_cpu_exit, From fc80a2d243ef1179fd1fd811b897d789eae011f8 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Thu, 17 Oct 2024 05:39:30 +0000 Subject: [PATCH 053/188] cpufreq/amd-pstate: Set the initial min_freq to lowest_nonlinear_freq commit 5d9a354cf839a652f8f6ca2b920696c6a9041801 upstream. According to the AMD architectural programmer's manual volume 2 [1], in section "17.6.4.1 CPPC_CAPABILITY_1" lowest_nonlinear_perf is described as "Reports the most energy efficient performance level (in terms of performance per watt). Above this threshold, lower performance levels generally result in increased energy efficiency. Reducing performance below this threshold does not result in total energy savings for a given computation, although it reduces instantaneous power consumption". So lowest_nonlinear_perf is the most power efficient performance level, and going below that would lead to a worse performance/watt. Also, setting the minimum frequency to lowest_nonlinear_freq (instead of lowest_freq) allows the CPU to idle at a higher frequency which leads to more time being spent in a deeper idle state (as trivial idle tasks are completed sooner). This has shown a power benefit in some systems, in other systems, power consumption has increased but so has the throughput/watt. Modify the initial policy_data->min set by cpufreq-core to lowest_nonlinear_freq, in the ->verify() callback. Also set the cpudata->req[0] to FREQ_QOS_MIN_DEFAULT_VALUE (i.e. 0), so that it also gets overriden by the check in verify function. Link: https://www.amd.com/content/dam/amd/en/documents/processor-tech-docs/programmer-references/24593.pdf [1] Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20241017053927.25285-3-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 11e4fd32bf709..2040173affcbd 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -530,8 +530,27 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) { + /* + * Initialize lower frequency limit (i.e.policy->min) with + * lowest_nonlinear_frequency which is the most energy efficient + * frequency. Override the initial value set by cpufreq core and + * amd-pstate qos_requests. + */ + if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { + struct cpufreq_policy *policy = cpufreq_cpu_get(policy_data->cpu); + struct amd_cpudata *cpudata; + + if (!policy) + return -EINVAL; + + cpudata = policy->driver_data; + policy_data->min = cpudata->lowest_nonlinear_freq; + cpufreq_cpu_put(policy); + } + cpufreq_verify_within_cpu_limits(policy_data); pr_debug("policy_max =%d, policy_min=%d\n", policy_data->max, policy_data->min); + return 0; } @@ -971,7 +990,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) policy->fast_switch_possible = true; ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], - FREQ_QOS_MIN, policy->cpuinfo.min_freq); + FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); if (ret < 0) { dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); goto free_cpudata1; From 768e9da6b3e640b421d66753127ba3f2b105c886 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Thu, 17 Oct 2024 10:05:27 +0000 Subject: [PATCH 054/188] cpufreq/amd-pstate: Call amd_pstate_register() in amd_pstate_init() commit 6f241fa50a1a45018d01f2d79a122eb519618f0e upstream. Replace a similar chunk of code in amd_pstate_init() with amd_pstate_register() call. Suggested-by: Mario Limonciello Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20241017100528.300143-2-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 2040173affcbd..834ef6cbc4872 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1871,17 +1871,10 @@ static int __init amd_pstate_init(void) return ret; } - /* enable amd pstate feature */ - ret = amd_pstate_enable(true); - if (ret) { - pr_err("failed to enable driver mode(%d)\n", cppc_state); - return ret; - } - - ret = cpufreq_register_driver(current_pstate_driver); + ret = amd_pstate_register_driver(cppc_state); if (ret) { pr_err("failed to register with return %d\n", ret); - goto disable_driver; + return ret; } dev_root = bus_get_dev_root(&cpu_subsys); @@ -1898,7 +1891,6 @@ static int __init amd_pstate_init(void) global_attr_free: cpufreq_unregister_driver(current_pstate_driver); -disable_driver: amd_pstate_enable(false); return ret; } From 08788394576c8a7c9d51d55989b4f4bb0ff03163 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Thu, 17 Oct 2024 10:05:29 +0000 Subject: [PATCH 055/188] cpufreq/amd-pstate: Call amd_pstate_set_driver() in amd_pstate_register_driver() commit e3591eebeca78117ce1d07985d42b75e6506252a upstream. Replace a similar chunk of code in amd_pstate_register_driver() with amd_pstate_set_driver() call. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20241017100528.300143-3-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 47 +++++++++++++++++------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 834ef6cbc4872..c6d78c6d7e21c 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1183,16 +1183,32 @@ static void amd_pstate_driver_cleanup(void) current_pstate_driver = NULL; } +static int amd_pstate_set_driver(int mode_idx) +{ + if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { + cppc_state = mode_idx; + if (cppc_state == AMD_PSTATE_DISABLE) + pr_info("driver is explicitly disabled\n"); + + if (cppc_state == AMD_PSTATE_ACTIVE) + current_pstate_driver = &amd_pstate_epp_driver; + + if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) + current_pstate_driver = &amd_pstate_driver; + + return 0; + } + + return -EINVAL; +} + static int amd_pstate_register_driver(int mode) { int ret; - if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED) - current_pstate_driver = &amd_pstate_driver; - else if (mode == AMD_PSTATE_ACTIVE) - current_pstate_driver = &amd_pstate_epp_driver; - else - return -EINVAL; + ret = amd_pstate_set_driver(mode); + if (ret) + return ret; cppc_state = mode; @@ -1718,25 +1734,6 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .attr = amd_pstate_epp_attr, }; -static int __init amd_pstate_set_driver(int mode_idx) -{ - if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { - cppc_state = mode_idx; - if (cppc_state == AMD_PSTATE_DISABLE) - pr_info("driver is explicitly disabled\n"); - - if (cppc_state == AMD_PSTATE_ACTIVE) - current_pstate_driver = &amd_pstate_epp_driver; - - if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) - current_pstate_driver = &amd_pstate_driver; - - return 0; - } - - return -EINVAL; -} - /* * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F. * show the debug message that helps to check if the CPU has CPPC support for loading issue. From 3056def4aa24c04ea1434eaf88fd2838419563dc Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Thu, 17 Oct 2024 10:05:31 +0000 Subject: [PATCH 056/188] cpufreq/amd-pstate: Remove the switch case in amd_pstate_init() commit 162cfa4eba7350cdcbd610c4b2af860a0ecc5f21 upstream. Replace the switch case with a more readable if condition. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20241017100528.300143-4-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index c6d78c6d7e21c..f88c814e1405b 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1835,21 +1835,15 @@ static int __init amd_pstate_init(void) cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE; } - switch (cppc_state) { - case AMD_PSTATE_DISABLE: + if (cppc_state == AMD_PSTATE_DISABLE) { pr_info("driver load is disabled, boot with specific mode to enable this\n"); return -ENODEV; - case AMD_PSTATE_PASSIVE: - case AMD_PSTATE_ACTIVE: - case AMD_PSTATE_GUIDED: - ret = amd_pstate_set_driver(cppc_state); - if (ret) - return ret; - break; - default: - return -EINVAL; } + ret = amd_pstate_set_driver(cppc_state); + if (ret) + return ret; + /* capability check */ if (cpu_feature_enabled(X86_FEATURE_CPPC)) { pr_debug("AMD CPPC MSR based functionality is supported\n"); From 33c2e1a840479a5d7f00f4bbdabed87bd6549730 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Thu, 17 Oct 2024 10:05:33 +0000 Subject: [PATCH 057/188] cpufreq/amd-pstate: Remove the redundant amd_pstate_set_driver() call commit b427ac4084753d7c62ef797bed7f30203f0ef5fb upstream. amd_pstate_set_driver() is called twice, once in amd_pstate_init() and once as part of amd_pstate_register_driver(). Move around code and eliminate the redundancy. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20241017100528.300143-5-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index f88c814e1405b..a040bd1d31585 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1840,9 +1840,11 @@ static int __init amd_pstate_init(void) return -ENODEV; } - ret = amd_pstate_set_driver(cppc_state); - if (ret) + ret = amd_pstate_register_driver(cppc_state); + if (ret) { + pr_err("failed to register with return %d\n", ret); return ret; + } /* capability check */ if (cpu_feature_enabled(X86_FEATURE_CPPC)) { @@ -1862,12 +1864,6 @@ static int __init amd_pstate_init(void) return ret; } - ret = amd_pstate_register_driver(cppc_state); - if (ret) { - pr_err("failed to register with return %d\n", ret); - return ret; - } - dev_root = bus_get_dev_root(&cpu_subsys); if (dev_root) { ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); From 2fae9d9cd6e4447fc45814ff095c0f9b72cf5781 Mon Sep 17 00:00:00 2001 From: Swapnil Sapkal Date: Mon, 21 Oct 2024 15:48:36 +0530 Subject: [PATCH 058/188] amd-pstate: Switch to amd-pstate by default on some Server platforms commit 54ab7d7c59c9444a28b67eaa10983f86ebf73c69 upstream. Currently the default cpufreq driver for all the AMD EPYC servers is acpi-cpufreq. Going forward, switch to amd-pstate as the default driver on the AMD EPYC server platforms with CPU family 0x1A or higher. The default mode will be active mode. Testing shows that amd-pstate with active mode and performance governor provides comparable or better performance per-watt against acpi-cpufreq + performance governor. Likewise, amd-pstate with active mode and powersave governor with the energy_performance_preference=power (EPP=255) provides comparable or better performance per-watt against acpi-cpufreq + schedutil governor for a wide range of workloads. Users can still revert to using acpi-cpufreq driver on these platforms with the "amd_pstate=disable" kernel commandline parameter. Signed-off-by: Swapnil Sapkal Signed-off-by: Gautham R. Shenoy Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20241021101836.9047-3-gautham.shenoy@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index a040bd1d31585..30acd8f6ce954 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1824,10 +1824,10 @@ static int __init amd_pstate_init(void) if (cppc_state == AMD_PSTATE_UNDEFINED) { /* Disable on the following configs by default: * 1. Undefined platforms - * 2. Server platforms + * 2. Server platforms with CPUs older than Family 0x1A. */ if (amd_pstate_acpi_pm_profile_undefined() || - amd_pstate_acpi_pm_profile_server()) { + (amd_pstate_acpi_pm_profile_server() && boot_cpu_data.x86 < 0x1A)) { pr_info("driver load is disabled, boot with specific mode to enable this\n"); return -ENODEV; } From 8df426e851a28787e56081d37663c9de55c7001d Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 17 Oct 2024 12:34:39 -0500 Subject: [PATCH 059/188] cpufreq/amd-pstate-ut: Add fix for min freq unit test commit 31f9056ed243002a2ec1d8cfa871d2c3eb84cf83 upstream. commit 642aff3964b0f ("cpufreq/amd-pstate: Set the initial min_freq to lowest_nonlinear_freq") changed the initial minimum frequency to lowest nonlinear frequency, but the unit tests weren't updated and now fail. Update them to match this same change. Fixes: 642aff3964b0f ("cpufreq/amd-pstate: Set the initial min_freq to lowest_nonlinear_freq") Link: https://lore.kernel.org/r/20241017173439.4924-1-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index f66701514d906..a261d7300951e 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -227,10 +227,10 @@ static void amd_pstate_ut_check_freq(u32 index) goto skip_test; } - if (cpudata->min_freq != policy->min) { + if (cpudata->lowest_nonlinear_freq != policy->min) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; - pr_err("%s cpu%d cpudata_min_freq=%d policy_min=%d, they should be equal!\n", - __func__, cpu, cpudata->min_freq, policy->min); + pr_err("%s cpu%d cpudata_lowest_nonlinear_freq=%d policy_min=%d, they should be equal!\n", + __func__, cpu, cpudata->lowest_nonlinear_freq, policy->min); goto skip_test; } From 16e85a2d7a35a77761da6fd4a235e96103ea7eb0 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 23 Oct 2024 10:21:06 +0000 Subject: [PATCH 060/188] cpufreq/amd-pstate: Rename functions that enable CPPC commit 7fb463aac84577b93f666658ef61e59d78c53627 upstream. Explicitly rename functions that enable CPPC as *_cppc_*. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Perry Yuan Link: https://lore.kernel.org/r/20241023102108.5980-2-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 30acd8f6ce954..e293455a7b6d6 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -307,7 +307,7 @@ static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, return ret; } -static inline int msr_enable(bool enable) +static inline int msr_cppc_enable(bool enable) { int ret, cpu; unsigned long logical_proc_id_mask = 0; @@ -333,7 +333,7 @@ static inline int msr_enable(bool enable) return 0; } -static int shmem_enable(bool enable) +static int shmem_cppc_enable(bool enable) { int cpu, ret = 0; struct cppc_perf_ctrls perf_ctrls; @@ -360,11 +360,11 @@ static int shmem_enable(bool enable) return ret; } -DEFINE_STATIC_CALL(amd_pstate_enable, msr_enable); +DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); -static inline int amd_pstate_enable(bool enable) +static inline int amd_pstate_cppc_enable(bool enable) { - return static_call(amd_pstate_enable)(enable); + return static_call(amd_pstate_cppc_enable)(enable); } static int msr_init_perf(struct amd_cpudata *cpudata) @@ -1034,7 +1034,7 @@ static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) { int ret; - ret = amd_pstate_enable(true); + ret = amd_pstate_cppc_enable(true); if (ret) pr_err("failed to enable amd-pstate during resume, return %d\n", ret); @@ -1045,7 +1045,7 @@ static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) { int ret; - ret = amd_pstate_enable(false); + ret = amd_pstate_cppc_enable(false); if (ret) pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); @@ -1178,7 +1178,7 @@ static ssize_t show_energy_performance_preference( static void amd_pstate_driver_cleanup(void) { - amd_pstate_enable(false); + amd_pstate_cppc_enable(false); cppc_state = AMD_PSTATE_DISABLE; current_pstate_driver = NULL; } @@ -1212,7 +1212,7 @@ static int amd_pstate_register_driver(int mode) cppc_state = mode; - ret = amd_pstate_enable(true); + ret = amd_pstate_cppc_enable(true); if (ret) { pr_err("failed to enable cppc during amd-pstate driver registration, return %d\n", ret); @@ -1591,7 +1591,7 @@ static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) u64 value, max_perf; int ret; - ret = amd_pstate_enable(true); + ret = amd_pstate_cppc_enable(true); if (ret) pr_err("failed to enable amd pstate during resume, return %d\n", ret); @@ -1678,7 +1678,7 @@ static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) cpudata->suspended = true; /* disable CPPC in lowlevel firmware */ - ret = amd_pstate_enable(false); + ret = amd_pstate_cppc_enable(false); if (ret) pr_err("failed to suspend, return %d\n", ret); @@ -1853,7 +1853,7 @@ static int __init amd_pstate_init(void) current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; } else { pr_debug("AMD CPPC shared memory based functionality is supported\n"); - static_call_update(amd_pstate_enable, shmem_enable); + static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); static_call_update(amd_pstate_init_perf, shmem_init_perf); static_call_update(amd_pstate_update_perf, shmem_update_perf); } @@ -1878,7 +1878,7 @@ static int __init amd_pstate_init(void) global_attr_free: cpufreq_unregister_driver(current_pstate_driver); - amd_pstate_enable(false); + amd_pstate_cppc_enable(false); return ret; } device_initcall(amd_pstate_init); From af433dd634fa7685cee958f431620eb12774b16d Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 23 Oct 2024 10:21:08 +0000 Subject: [PATCH 061/188] cpufreq/amd-pstate: Do not attempt to clear MSR_AMD_CPPC_ENABLE commit 73070a9169b16e08b191066e513bce6c237c9221 upstream. MSR_AMD_CPPC_ENABLE is a write once register, i.e. attempting to clear it is futile, it will not take effect. Hence, return if disable (0) argument is passed to the msr_cppc_enable() Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241023102108.5980-3-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index e293455a7b6d6..396df44f1de33 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -312,6 +312,12 @@ static inline int msr_cppc_enable(bool enable) int ret, cpu; unsigned long logical_proc_id_mask = 0; + /* + * MSR_AMD_CPPC_ENABLE is write-once, once set it cannot be cleared. + */ + if (!enable) + return 0; + if (enable == cppc_enabled) return 0; From 24851d234d320548edc6931149ea845bea11c44f Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 23 Oct 2024 10:21:10 +0000 Subject: [PATCH 062/188] cpufreq/amd-pstate: Call cppc_set_epp_perf in the reenable function commit 796ff50e127af8362035f87ba29b6b84e2dd9742 upstream. The EPP value being set in perf_ctrls.energy_perf is not being propagated to the shared memory, fix that. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Perry Yuan Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241023102108.5980-4-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 396df44f1de33..9436eddc50bd6 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1608,8 +1608,9 @@ static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); } else { perf_ctrls.max_perf = max_perf; - perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); cppc_set_perf(cpudata->cpu, &perf_ctrls); + perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); + cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); } } @@ -1650,8 +1651,9 @@ static void amd_pstate_epp_offline(struct cpufreq_policy *policy) } else { perf_ctrls.desired_perf = 0; perf_ctrls.max_perf = min_perf; - perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); cppc_set_perf(cpudata->cpu, &perf_ctrls); + perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); + cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); } mutex_unlock(&amd_pstate_limits_lock); } From d8ac17ac92d0434d0023806837c7eabdc7881f68 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 23 Oct 2024 10:21:12 +0000 Subject: [PATCH 063/188] cpufreq/amd-pstate: Align offline flow of shared memory and MSR based systems commit a6960e6b1b0e2cb268f427a99040c408a8d10665 upstream. Set min_perf to lowest_perf for shared memory systems, similar to the MSR based systems. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241023102108.5980-5-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 9436eddc50bd6..01d1a65b47d11 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1650,6 +1650,7 @@ static void amd_pstate_epp_offline(struct cpufreq_policy *policy) wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); } else { perf_ctrls.desired_perf = 0; + perf_ctrls.min_perf = min_perf; perf_ctrls.max_perf = min_perf; cppc_set_perf(cpudata->cpu, &perf_ctrls); perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); From d6cc498900be99cb26b89839a06a9148885f73b4 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 28 Oct 2024 09:55:41 -0500 Subject: [PATCH 064/188] cpufreq/amd-pstate: Push adjust_perf vfunc init into cpu_init commit 3ac757e8db3deacc8058912d55efd1390f985d80 upstream. As the driver can be changed in and out of different modes it's possible that adjust_perf is assigned when it shouldn't be. This could happen if an MSR design is started up in passive mode and then switches to active mode. To solve this explicitly clear `adjust_perf` in amd_pstate_epp_cpu_init(). Tested-by: Klara Modin Tested-by: Dhananjay Ugwekar Link: https://lore.kernel.org/r/20241028145542.1739160-1-superm1@kernel.org Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 01d1a65b47d11..89aae328ad54e 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1496,6 +1496,8 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) WRITE_ONCE(cpudata->cppc_cap1_cached, value); } + current_pstate_driver->adjust_perf = NULL; + return 0; free_cpudata1: @@ -1858,8 +1860,6 @@ static int __init amd_pstate_init(void) /* capability check */ if (cpu_feature_enabled(X86_FEATURE_CPPC)) { pr_debug("AMD CPPC MSR based functionality is supported\n"); - if (cppc_state != AMD_PSTATE_ACTIVE) - current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; } else { pr_debug("AMD CPPC shared memory based functionality is supported\n"); static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); From fa1792b4b23a7bfb036d46f7a60c78869df64e0a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 28 Oct 2024 09:55:42 -0500 Subject: [PATCH 065/188] cpufreq/amd-pstate: Move registration after static function call update commit ff2653ded4d99b709f62f35656d976c47a7aa476 upstream. On shared memory designs the static functions need to work before registration is done or the system can hang at bootup. Move the registration later in amd_pstate_init() to solve this. Fixes: b427ac408475 ("cpufreq/amd-pstate: Remove the redundant amd_pstate_set_driver() call") Reported-by: Klara Modin Closes: https://lore.kernel.org/linux-pm/cf9c146d-bacf-444e-92e2-15ebf513af96@gmail.com/#t Tested-by: Klara Modin Tested-by: Dhananjay Ugwekar Link: https://lore.kernel.org/r/20241028145542.1739160-2-superm1@kernel.org Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 89aae328ad54e..993c169141e35 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1851,12 +1851,6 @@ static int __init amd_pstate_init(void) return -ENODEV; } - ret = amd_pstate_register_driver(cppc_state); - if (ret) { - pr_err("failed to register with return %d\n", ret); - return ret; - } - /* capability check */ if (cpu_feature_enabled(X86_FEATURE_CPPC)) { pr_debug("AMD CPPC MSR based functionality is supported\n"); @@ -1867,6 +1861,12 @@ static int __init amd_pstate_init(void) static_call_update(amd_pstate_update_perf, shmem_update_perf); } + ret = amd_pstate_register_driver(cppc_state); + if (ret) { + pr_err("failed to register with return %d\n", ret); + return ret; + } + if (amd_pstate_prefcore) { ret = amd_detect_prefcore(&amd_pstate_prefcore); if (ret) From f8a1030a825cf64be15aabcab210bf01a40dfa2f Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Tue, 10 Dec 2024 03:25:57 +0000 Subject: [PATCH 066/188] cpufreq/amd-pstate: Detect preferred core support before driver registration commit 919bfa9b2dbf3bc0c478afd4e44445836381dacb upstream. Booting with amd-pstate on 3rd Generation EPYC system incorrectly enabled ITMT support despite the system not supporting Preferred Core ranking. amd_pstate_init_prefcore() called during amd_pstate*_cpu_init() requires "amd_pstate_prefcore" to be set correctly however the preferred core support is detected only after driver registration which is too late. Swap the function calls around to detect preferred core support before registring the driver via amd_pstate_register_driver(). This ensures amd_pstate*_cpu_init() sees the correct value of "amd_pstate_prefcore" considering the platform support. Fixes: 279f838a61f9 ("x86/amd: Detect preferred cores in amd_get_boost_ratio_numerator()") Fixes: ff2653ded4d9 ("cpufreq/amd-pstate: Move registration after static function call update") Signed-off-by: K Prateek Nayak Acked-by: Mario Limonciello Link: https://lore.kernel.org/r/20241210032557.754-1-kprateek.nayak@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 993c169141e35..7fe2c7ffb2c66 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1861,18 +1861,18 @@ static int __init amd_pstate_init(void) static_call_update(amd_pstate_update_perf, shmem_update_perf); } - ret = amd_pstate_register_driver(cppc_state); - if (ret) { - pr_err("failed to register with return %d\n", ret); - return ret; - } - if (amd_pstate_prefcore) { ret = amd_detect_prefcore(&amd_pstate_prefcore); if (ret) return ret; } + ret = amd_pstate_register_driver(cppc_state); + if (ret) { + pr_err("failed to register with return %d\n", ret); + return ret; + } + dev_root = bus_get_dev_root(&cpu_subsys); if (dev_root) { ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); From 142939077a4df84a2543f4c1b72a7b40b31bb15a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 26 Aug 2024 16:13:58 -0500 Subject: [PATCH 067/188] cpufreq: amd-pstate: Add documentation for `amd_pstate_hw_prefcore` commit b96b82d1af7fbf35e3c7d50368275005bd6b6a03 upstream. Explain that the sysfs file represents both preferred core being enabled by the user and supported by the hardware. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 573c79184383c..2b005bb65e8f9 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -263,6 +263,11 @@ lowest non-linear performance in `AMD CPPC Performance Capability `_.) This attribute is read-only. +``amd_pstate_hw_prefcore`` + +Whether the platform supports the preferred core feature and it has been +enabled. This attribute is read-only. + ``energy_performance_available_preferences`` A list of all the supported EPP preferences that could be used for From 10949b4b0105af5484104e2684401d03cea10299 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 3 Sep 2024 14:43:10 -0500 Subject: [PATCH 068/188] amd-pstate: Add missing documentation for `amd_pstate_prefcore_ranking` commit 15a2b764ea7c16dd2b1ecfd86ba27809f5bd8580 upstream. `amd_pstate_prefcore_ranking` reflects the dynamic rankings of a CPU core based on platform conditions. Explicitly include it in the documentation. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 2b005bb65e8f9..34e49ee49dc1c 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -252,7 +252,8 @@ In some ASICs, the highest CPPC performance is not the one in the ``_CPC`` table, so we need to expose it to sysfs. If boost is not active, but still supported, this maximum frequency will be larger than the one in ``cpuinfo``. On systems that support preferred core, the driver will have -different values for some cores than others. +different values for some cores than others and this will reflect the values +advertised by the platform at bootup. This attribute is read-only. ``amd_pstate_lowest_nonlinear_freq`` @@ -268,6 +269,12 @@ This attribute is read-only. Whether the platform supports the preferred core feature and it has been enabled. This attribute is read-only. +``amd_pstate_prefcore_ranking`` + +The performance ranking of the core. This number doesn't have any unit, but +larger numbers are preferred at the time of reading. This can change at +runtime based on platform conditions. This attribute is read-only. + ``energy_performance_available_preferences`` A list of all the supported EPP preferences that could be used for From 34ae2b4c7601189cd7d570034f581458b80c8b79 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:34 -0600 Subject: [PATCH 069/188] cpufreq/amd-pstate: Store the boost numerator as highest perf again commit 50a062a7620051c09adacd6d140ebd56881a333b upstream. commit ad4caad58d91d ("cpufreq: amd-pstate: Merge amd_pstate_highest_perf_set() into amd_get_boost_ratio_numerator()") changed the semantics for highest perf and commit 18d9b52271213 ("cpufreq/amd-pstate: Use nominal perf for limits when boost is disabled") worked around those semantic changes. This however is a confusing result and furthermore makes it awkward to change frequency limits and boost due to the scaling differences. Restore the boost numerator to highest perf again. Suggested-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Fixes: ad4caad58d91 ("cpufreq: amd-pstate: Merge amd_pstate_highest_perf_set() into amd_get_boost_ratio_numerator()") Link: https://lore.kernel.org/r/20241209185248.16301-2-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 4 +--- drivers/cpufreq/amd-pstate.c | 25 ++++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 34e49ee49dc1c..fa6cf11a1ae3f 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -251,9 +251,7 @@ performance supported in `AMD CPPC Performance Capability `_). In some ASICs, the highest CPPC performance is not the one in the ``_CPC`` table, so we need to expose it to sysfs. If boost is not active, but still supported, this maximum frequency will be larger than the one in -``cpuinfo``. On systems that support preferred core, the driver will have -different values for some cores than others and this will reflect the values -advertised by the platform at bootup. +``cpuinfo``. This attribute is read-only. ``amd_pstate_lowest_nonlinear_freq`` diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 7fe2c7ffb2c66..78593c86b8b41 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -375,15 +375,19 @@ static inline int amd_pstate_cppc_enable(bool enable) static int msr_init_perf(struct amd_cpudata *cpudata) { - u64 cap1; + u64 cap1, numerator; int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &cap1); if (ret) return ret; - WRITE_ONCE(cpudata->highest_perf, AMD_CPPC_HIGHEST_PERF(cap1)); - WRITE_ONCE(cpudata->max_limit_perf, AMD_CPPC_HIGHEST_PERF(cap1)); + ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); + if (ret) + return ret; + + WRITE_ONCE(cpudata->highest_perf, numerator); + WRITE_ONCE(cpudata->max_limit_perf, numerator); WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); @@ -395,13 +399,18 @@ static int msr_init_perf(struct amd_cpudata *cpudata) static int shmem_init_perf(struct amd_cpudata *cpudata) { struct cppc_perf_caps cppc_perf; + u64 numerator; int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); if (ret) return ret; - WRITE_ONCE(cpudata->highest_perf, cppc_perf.highest_perf); - WRITE_ONCE(cpudata->max_limit_perf, cppc_perf.highest_perf); + ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); + if (ret) + return ret; + + WRITE_ONCE(cpudata->highest_perf, numerator); + WRITE_ONCE(cpudata->max_limit_perf, numerator); WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); WRITE_ONCE(cpudata->lowest_nonlinear_perf, cppc_perf.lowest_nonlinear_perf); @@ -881,7 +890,6 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) { int ret; u32 min_freq, max_freq; - u64 numerator; u32 nominal_perf, nominal_freq; u32 lowest_nonlinear_perf, lowest_nonlinear_freq; u32 boost_ratio, lowest_nonlinear_ratio; @@ -903,10 +911,7 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) nominal_perf = READ_ONCE(cpudata->nominal_perf); - ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator); - if (ret) - return ret; - boost_ratio = div_u64(numerator << SCHED_CAPACITY_SHIFT, nominal_perf); + boost_ratio = div_u64(cpudata->highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT) * 1000; lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); From 4aac802ce872d3a1253e763b854faf92b31230f5 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:35 -0600 Subject: [PATCH 070/188] cpufreq/amd-pstate: Use boost numerator for upper bound of frequencies commit 2993b29b2a98f2bc9d55dfd37ef39f56a2908748 upstream. commit 18d9b5227121 ("cpufreq/amd-pstate: Use nominal perf for limits when boost is disabled") introduced different semantics for min/max limits based upon whether the user turned off boost from sysfs. This however is not necessary when the highest perf value is the boost numerator. Suggested-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Fixes: 18d9b5227121 ("cpufreq/amd-pstate: Use nominal perf for limits when boost is disabled") Link: https://lore.kernel.org/r/20241209185248.16301-3-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 78593c86b8b41..18437b01e9b8f 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -571,16 +571,13 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) { - u32 max_limit_perf, min_limit_perf, lowest_perf, max_perf; + u32 max_limit_perf, min_limit_perf, lowest_perf, max_perf, max_freq; struct amd_cpudata *cpudata = policy->driver_data; - if (cpudata->boost_supported && !policy->boost_enabled) - max_perf = READ_ONCE(cpudata->nominal_perf); - else - max_perf = READ_ONCE(cpudata->highest_perf); - - max_limit_perf = div_u64(policy->max * max_perf, policy->cpuinfo.max_freq); - min_limit_perf = div_u64(policy->min * max_perf, policy->cpuinfo.max_freq); + max_perf = READ_ONCE(cpudata->highest_perf); + max_freq = READ_ONCE(cpudata->max_freq); + max_limit_perf = div_u64(policy->max * max_perf, max_freq); + min_limit_perf = div_u64(policy->min * max_perf, max_freq); lowest_perf = READ_ONCE(cpudata->lowest_perf); if (min_limit_perf < lowest_perf) From b4885c765530e0ab2217bf8a7127c9bfa35503a8 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 4 Dec 2024 14:48:38 +0000 Subject: [PATCH 071/188] cpufreq/amd-pstate: Convert the amd_pstate_get/set_epp() to static calls commit 16c977f8177f9c2ecb88319c944722107c952731 upstream. MSR and shared memory based systems have different mechanisms to get and set the epp value. Split those mechanisms into different functions and assign them appropriately to the static calls at boot time. This eliminates the need for the "if(cpu_feature_enabled(X86_FEATURE_CPPC))" checks at runtime. Also, propagate the error code from rdmsrl_on_cpu() and cppc_get_epp_perf() to *_get_epp()'s caller, instead of returning -EIO unconditionally. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241204144842.164178-2-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 92 +++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 32 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 18437b01e9b8f..8b25c2e876fae 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -181,26 +181,40 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) static DEFINE_MUTEX(amd_pstate_limits_lock); static DEFINE_MUTEX(amd_pstate_driver_lock); -static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) +static s16 msr_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) { u64 epp; int ret; - if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - if (!cppc_req_cached) { - epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, - &cppc_req_cached); - if (epp) - return epp; - } - epp = (cppc_req_cached >> 24) & 0xFF; - } else { - ret = cppc_get_epp_perf(cpudata->cpu, &epp); + if (!cppc_req_cached) { + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req_cached); if (ret < 0) { pr_debug("Could not retrieve energy perf value (%d)\n", ret); - return -EIO; + return ret; } } + epp = (cppc_req_cached >> 24) & 0xFF; + + return (s16)epp; +} + +DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); + +static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) +{ + return static_call(amd_pstate_get_epp)(cpudata, cppc_req_cached); +} + +static s16 shmem_get_epp(struct amd_cpudata *cpudata, u64 dummy) +{ + u64 epp; + int ret; + + ret = cppc_get_epp_perf(cpudata->cpu, &epp); + if (ret < 0) { + pr_debug("Could not retrieve energy perf value (%d)\n", ret); + return ret; + } return (s16)(epp & 0xff); } @@ -254,33 +268,45 @@ static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, max_perf, fast_switch); } -static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) +static int msr_set_epp(struct amd_cpudata *cpudata, u32 epp) { int ret; - struct cppc_perf_ctrls perf_ctrls; - - if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - u64 value = READ_ONCE(cpudata->cppc_req_cached); - value &= ~GENMASK_ULL(31, 24); - value |= (u64)epp << 24; - WRITE_ONCE(cpudata->cppc_req_cached, value); + u64 value = READ_ONCE(cpudata->cppc_req_cached); - ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); - if (!ret) - cpudata->epp_cached = epp; - } else { - amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, - cpudata->max_limit_perf, false); + value &= ~GENMASK_ULL(31, 24); + value |= (u64)epp << 24; + WRITE_ONCE(cpudata->cppc_req_cached, value); - perf_ctrls.energy_perf = epp; - ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); - if (ret) { - pr_debug("failed to set energy perf value (%d)\n", ret); - return ret; - } + ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); + if (!ret) cpudata->epp_cached = epp; + + return ret; +} + +DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); + +static inline int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) +{ + return static_call(amd_pstate_set_epp)(cpudata, epp); +} + +static int shmem_set_epp(struct amd_cpudata *cpudata, u32 epp) +{ + int ret; + struct cppc_perf_ctrls perf_ctrls; + + amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, + cpudata->max_limit_perf, false); + + perf_ctrls.energy_perf = epp; + ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); + if (ret) { + pr_debug("failed to set energy perf value (%d)\n", ret); + return ret; } + cpudata->epp_cached = epp; return ret; } @@ -1861,6 +1887,8 @@ static int __init amd_pstate_init(void) static_call_update(amd_pstate_cppc_enable, shmem_cppc_enable); static_call_update(amd_pstate_init_perf, shmem_init_perf); static_call_update(amd_pstate_update_perf, shmem_update_perf); + static_call_update(amd_pstate_get_epp, shmem_get_epp); + static_call_update(amd_pstate_set_epp, shmem_set_epp); } if (amd_pstate_prefcore) { From d2aba1acef935e1f253cf1e98bd049c9b75a57a1 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 4 Dec 2024 14:48:39 +0000 Subject: [PATCH 072/188] cpufreq/amd-pstate: Move the invocation of amd_pstate_update_perf() commit 57a2b25e45cd40eaa2e505452384fa1b7248895a upstream. amd_pstate_update_perf() should not be a part of shmem_set_epp() function, so move it to the amd_pstate_epp_update_limit() function, where it is needed. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241204144842.164178-3-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 8b25c2e876fae..ebcd484ea5f0b 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -297,9 +297,6 @@ static int shmem_set_epp(struct amd_cpudata *cpudata, u32 epp) int ret; struct cppc_perf_ctrls perf_ctrls; - amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, - cpudata->max_limit_perf, false); - perf_ctrls.energy_perf = epp; ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); if (ret) { @@ -1592,6 +1589,10 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) epp = 0; WRITE_ONCE(cpudata->cppc_req_cached, value); + + amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, + cpudata->max_limit_perf, false); + return amd_pstate_set_epp(cpudata, epp); } From b3b7619ed1fac81aa013d4db05219e4bcc4c51db Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 4 Dec 2024 14:48:40 +0000 Subject: [PATCH 073/188] cpufreq/amd-pstate: Refactor amd_pstate_epp_reenable() and amd_pstate_epp_offline() commit b1089e0c8817fda93d474eaa82ad86386887aefe upstream. Replace similar code chunks with amd_pstate_update_perf() and amd_pstate_set_epp() function calls. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241204144842.164178-4-Dhananjay.Ugwekar@amd.com [ML: Fix LKP reported error about unused variable] Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 38 +++++++----------------------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index ebcd484ea5f0b..f1fffca683ee7 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1624,25 +1624,17 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) { - struct cppc_perf_ctrls perf_ctrls; - u64 value, max_perf; + u64 max_perf; int ret; ret = amd_pstate_cppc_enable(true); if (ret) pr_err("failed to enable amd pstate during resume, return %d\n", ret); - value = READ_ONCE(cpudata->cppc_req_cached); max_perf = READ_ONCE(cpudata->highest_perf); - if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); - } else { - perf_ctrls.max_perf = max_perf; - cppc_set_perf(cpudata->cpu, &perf_ctrls); - perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); - cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); - } + amd_pstate_update_perf(cpudata, 0, 0, max_perf, false); + amd_pstate_set_epp(cpudata, cpudata->epp_cached); } static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) @@ -1662,31 +1654,15 @@ static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) static void amd_pstate_epp_offline(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - struct cppc_perf_ctrls perf_ctrls; int min_perf; - u64 value; min_perf = READ_ONCE(cpudata->lowest_perf); - value = READ_ONCE(cpudata->cppc_req_cached); mutex_lock(&amd_pstate_limits_lock); - if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; - - /* Set max perf same as min perf */ - value &= ~AMD_CPPC_MAX_PERF(~0L); - value |= AMD_CPPC_MAX_PERF(min_perf); - value &= ~AMD_CPPC_MIN_PERF(~0L); - value |= AMD_CPPC_MIN_PERF(min_perf); - wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); - } else { - perf_ctrls.desired_perf = 0; - perf_ctrls.min_perf = min_perf; - perf_ctrls.max_perf = min_perf; - cppc_set_perf(cpudata->cpu, &perf_ctrls); - perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); - cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); - } + + amd_pstate_update_perf(cpudata, min_perf, 0, min_perf, false); + amd_pstate_set_epp(cpudata, AMD_CPPC_EPP_BALANCE_POWERSAVE); + mutex_unlock(&amd_pstate_limits_lock); } From 098858a074052a7577c2a9df86db7eb294c33f4d Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 4 Dec 2024 14:48:41 +0000 Subject: [PATCH 074/188] cpufreq/amd-pstate: Remove the cppc_state check in offline/online functions commit b78f8c87ec3e7499bb049986838636d3afbc7ece upstream. Only amd_pstate_epp driver (i.e. cppc_state = ACTIVE) enters the amd_pstate_epp_offline() and amd_pstate_epp_cpu_online() functions, so remove the unnecessary if condition checking if cppc_state is equal to AMD_PSTATE_ACTIVE. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241204144842.164178-5-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index f1fffca683ee7..a0adbbb456211 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1643,10 +1643,8 @@ static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); - if (cppc_state == AMD_PSTATE_ACTIVE) { - amd_pstate_epp_reenable(cpudata); - cpudata->suspended = false; - } + amd_pstate_epp_reenable(cpudata); + cpudata->suspended = false; return 0; } @@ -1675,8 +1673,7 @@ static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) if (cpudata->suspended) return 0; - if (cppc_state == AMD_PSTATE_ACTIVE) - amd_pstate_epp_offline(policy); + amd_pstate_epp_offline(policy); return 0; } From be83aaaf35a04ce19682cac8deb95d5e611d3351 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 4 Dec 2024 14:48:42 +0000 Subject: [PATCH 075/188] cpufreq/amd-pstate: Merge amd_pstate_epp_cpu_offline() and amd_pstate_epp_offline() commit 53ec2101dfede8fecdd240662281a12e537c3411 upstream. amd_pstate_epp_offline() is only called from within amd_pstate_epp_cpu_offline() and doesn't make much sense to have it at all. Hence, remove it. Also remove the unncessary debug print in the offline path while at it. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20241204144842.164178-6-Dhananjay.Ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index a0adbbb456211..3413d81a08edc 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1649,11 +1649,14 @@ static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) return 0; } -static void amd_pstate_epp_offline(struct cpufreq_policy *policy) +static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; int min_perf; + if (cpudata->suspended) + return 0; + min_perf = READ_ONCE(cpudata->lowest_perf); mutex_lock(&amd_pstate_limits_lock); @@ -1662,18 +1665,6 @@ static void amd_pstate_epp_offline(struct cpufreq_policy *policy) amd_pstate_set_epp(cpudata, AMD_CPPC_EPP_BALANCE_POWERSAVE); mutex_unlock(&amd_pstate_limits_lock); -} - -static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) -{ - struct amd_cpudata *cpudata = policy->driver_data; - - pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); - - if (cpudata->suspended) - return 0; - - amd_pstate_epp_offline(policy); return 0; } From 243bf374deb0ab8c546b859a27a8ad6ac98b367a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:36 -0600 Subject: [PATCH 076/188] cpufreq/amd-pstate: Add trace event for EPP perf updates commit 4dcd130151a654108a414b298df9e21a0d3575c9 upstream. In "active" mode the most important thing for debugging whether an issue is hardware or software based is to look at what was the last thing written to the CPPC request MSR or shared memory region. The 'amd_pstate_epp_perf' trace event shows the values being written for all CPUs. Reviewed-by: Perry Yuan Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-4-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-trace.h | 45 ++++++++++++++++++++++++++++++ drivers/cpufreq/amd-pstate.c | 28 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/drivers/cpufreq/amd-pstate-trace.h b/drivers/cpufreq/amd-pstate-trace.h index 35f38ae67fb13..e2221a4b6901c 100644 --- a/drivers/cpufreq/amd-pstate-trace.h +++ b/drivers/cpufreq/amd-pstate-trace.h @@ -88,6 +88,51 @@ TRACE_EVENT(amd_pstate_perf, ) ); +TRACE_EVENT(amd_pstate_epp_perf, + + TP_PROTO(unsigned int cpu_id, + unsigned int highest_perf, + unsigned int epp, + unsigned int min_perf, + unsigned int max_perf, + bool boost + ), + + TP_ARGS(cpu_id, + highest_perf, + epp, + min_perf, + max_perf, + boost), + + TP_STRUCT__entry( + __field(unsigned int, cpu_id) + __field(unsigned int, highest_perf) + __field(unsigned int, epp) + __field(unsigned int, min_perf) + __field(unsigned int, max_perf) + __field(bool, boost) + ), + + TP_fast_assign( + __entry->cpu_id = cpu_id; + __entry->highest_perf = highest_perf; + __entry->epp = epp; + __entry->min_perf = min_perf; + __entry->max_perf = max_perf; + __entry->boost = boost; + ), + + TP_printk("cpu%u: [%u<->%u]/%u, epp=%u, boost=%u", + (unsigned int)__entry->cpu_id, + (unsigned int)__entry->min_perf, + (unsigned int)__entry->max_perf, + (unsigned int)__entry->highest_perf, + (unsigned int)__entry->epp, + (bool)__entry->boost + ) +); + #endif /* _AMD_PSTATE_TRACE_H */ /* This part must be outside protection */ diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 3413d81a08edc..8c5baa8dc39d9 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -325,6 +325,14 @@ static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, return -EBUSY; } + if (trace_amd_pstate_epp_perf_enabled()) { + trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, + epp, + AMD_CPPC_MIN_PERF(cpudata->cppc_req_cached), + AMD_CPPC_MAX_PERF(cpudata->cppc_req_cached), + cpudata->boost_state); + } + ret = amd_pstate_set_epp(cpudata, epp); return ret; @@ -1590,6 +1598,13 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) WRITE_ONCE(cpudata->cppc_req_cached, value); + if (trace_amd_pstate_epp_perf_enabled()) { + trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, epp, + cpudata->min_limit_perf, + cpudata->max_limit_perf, + policy->boost_enabled); + } + amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, cpudata->max_limit_perf, false); @@ -1633,6 +1648,13 @@ static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) max_perf = READ_ONCE(cpudata->highest_perf); + if (trace_amd_pstate_epp_perf_enabled()) { + trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, + cpudata->epp_cached, + AMD_CPPC_MIN_PERF(cpudata->cppc_req_cached), + max_perf, cpudata->boost_state); + } + amd_pstate_update_perf(cpudata, 0, 0, max_perf, false); amd_pstate_set_epp(cpudata, cpudata->epp_cached); } @@ -1661,6 +1683,12 @@ static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) mutex_lock(&amd_pstate_limits_lock); + if (trace_amd_pstate_epp_perf_enabled()) { + trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, + AMD_CPPC_EPP_BALANCE_POWERSAVE, + min_perf, min_perf, policy->boost_enabled); + } + amd_pstate_update_perf(cpudata, min_perf, 0, min_perf, false); amd_pstate_set_epp(cpudata, AMD_CPPC_EPP_BALANCE_POWERSAVE); From 20842a801b3d86b907016e14f97185c50d541fc9 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:37 -0600 Subject: [PATCH 077/188] cpufreq/amd-pstate: convert mutex use to guard() commit 6c093d5a5b73ec1caf1e706510ae6031af2f9d43 upstream. Using scoped guard declaration will unlock mutexes automatically. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-5-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 8c5baa8dc39d9..812743f3959e5 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -759,12 +759,12 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) pr_err("Boost mode is not supported by this processor or SBIOS\n"); return -EOPNOTSUPP; } - mutex_lock(&amd_pstate_driver_lock); + guard(mutex)(&amd_pstate_driver_lock); + ret = amd_pstate_cpu_boost_update(policy, state); WRITE_ONCE(cpudata->boost_state, !ret ? state : false); policy->boost_enabled = !ret ? state : false; refresh_frequency_limits(policy); - mutex_unlock(&amd_pstate_driver_lock); return ret; } @@ -850,7 +850,8 @@ static void amd_pstate_update_limits(unsigned int cpu) if (!amd_pstate_prefcore) return; - mutex_lock(&amd_pstate_driver_lock); + guard(mutex)(&amd_pstate_driver_lock); + ret = amd_get_highest_perf(cpu, &cur_high); if (ret) goto free_cpufreq_put; @@ -870,7 +871,6 @@ static void amd_pstate_update_limits(unsigned int cpu) if (!highest_perf_changed) cpufreq_update_policy(cpu); - mutex_unlock(&amd_pstate_driver_lock); } /* @@ -1195,11 +1195,11 @@ static ssize_t store_energy_performance_preference( if (ret < 0) return -EINVAL; - mutex_lock(&amd_pstate_limits_lock); + guard(mutex)(&amd_pstate_limits_lock); + ret = amd_pstate_set_energy_pref_index(cpudata, ret); - mutex_unlock(&amd_pstate_limits_lock); - return ret ?: count; + return ret ? ret : count; } static ssize_t show_energy_performance_preference( @@ -1363,13 +1363,10 @@ EXPORT_SYMBOL_GPL(amd_pstate_update_status); static ssize_t status_show(struct device *dev, struct device_attribute *attr, char *buf) { - ssize_t ret; - mutex_lock(&amd_pstate_driver_lock); - ret = amd_pstate_show_status(buf); - mutex_unlock(&amd_pstate_driver_lock); + guard(mutex)(&amd_pstate_driver_lock); - return ret; + return amd_pstate_show_status(buf); } static ssize_t status_store(struct device *a, struct device_attribute *b, @@ -1378,9 +1375,8 @@ static ssize_t status_store(struct device *a, struct device_attribute *b, char *p = memchr(buf, '\n', count); int ret; - mutex_lock(&amd_pstate_driver_lock); + guard(mutex)(&amd_pstate_driver_lock); ret = amd_pstate_update_status(buf, p ? p - buf : count); - mutex_unlock(&amd_pstate_driver_lock); return ret < 0 ? ret : count; } @@ -1681,7 +1677,7 @@ static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) min_perf = READ_ONCE(cpudata->lowest_perf); - mutex_lock(&amd_pstate_limits_lock); + guard(mutex)(&amd_pstate_limits_lock); if (trace_amd_pstate_epp_perf_enabled()) { trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, @@ -1692,8 +1688,6 @@ static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) amd_pstate_update_perf(cpudata, min_perf, 0, min_perf, false); amd_pstate_set_epp(cpudata, AMD_CPPC_EPP_BALANCE_POWERSAVE); - mutex_unlock(&amd_pstate_limits_lock); - return 0; } @@ -1722,13 +1716,11 @@ static int amd_pstate_epp_resume(struct cpufreq_policy *policy) struct amd_cpudata *cpudata = policy->driver_data; if (cpudata->suspended) { - mutex_lock(&amd_pstate_limits_lock); + guard(mutex)(&amd_pstate_limits_lock); /* enable amd pstate from suspend state*/ amd_pstate_epp_reenable(cpudata); - mutex_unlock(&amd_pstate_limits_lock); - cpudata->suspended = false; } From c0168b975163e56a4514f5a1484dad34b2209244 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:38 -0600 Subject: [PATCH 078/188] cpufreq/amd-pstate: Drop cached epp_policy variable commit 3b43739824a6b617d8213dd2bce6fb1b2747c377 upstream. epp_policy is not used by any of the current code and there is no need to cache it. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-6-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 3 --- drivers/cpufreq/amd-pstate.h | 2 -- 2 files changed, 5 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 812743f3959e5..1ad09612bacb2 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1470,7 +1470,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) return -ENOMEM; cpudata->cpu = policy->cpu; - cpudata->epp_policy = 0; ret = amd_pstate_init_perf(cpudata); if (ret) @@ -1577,8 +1576,6 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) value &= ~AMD_CPPC_DES_PERF(~0L); value |= AMD_CPPC_DES_PERF(0); - cpudata->epp_policy = cpudata->policy; - /* Get BIOS pre-defined epp value */ epp = amd_pstate_get_epp(cpudata, value); if (epp < 0) { diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index cd573bc6b6db8..7765c82f975c6 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -57,7 +57,6 @@ struct amd_aperf_mperf { * @hw_prefcore: check whether HW supports preferred core featue. * Only when hw_prefcore and early prefcore param are true, * AMD P-State driver supports preferred core featue. - * @epp_policy: Last saved policy used to set energy-performance preference * @epp_cached: Cached CPPC energy-performance preference value * @policy: Cpufreq policy value * @cppc_cap1_cached Cached MSR_AMD_CPPC_CAP1 register value @@ -94,7 +93,6 @@ struct amd_cpudata { bool hw_prefcore; /* EPP feature related attributes*/ - s16 epp_policy; s16 epp_cached; u32 policy; u64 cppc_cap1_cached; From d5a48c1d4b81d2a47fb966e8df58f5c8f72aa736 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:39 -0600 Subject: [PATCH 079/188] cpufreq/amd-pstate: Use FIELD_PREP and FIELD_GET macros commit 88a95ba066a962d4d39c6a36b18bf665f51d3767 upstream. The FIELD_PREP and FIELD_GET macros improve readability and help to avoid shifting bugs. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-7-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 51 ++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 1ad09612bacb2..326814929544f 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -22,6 +22,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -89,6 +90,11 @@ static bool cppc_enabled; static bool amd_pstate_prefcore = true; static struct quirk_entry *quirks; +#define AMD_CPPC_MAX_PERF_MASK GENMASK(7, 0) +#define AMD_CPPC_MIN_PERF_MASK GENMASK(15, 8) +#define AMD_CPPC_DES_PERF_MASK GENMASK(23, 16) +#define AMD_CPPC_EPP_PERF_MASK GENMASK(31, 24) + /* * AMD Energy Preference Performance (EPP) * The EPP is used in the CCLK DPM controller to drive @@ -183,7 +189,6 @@ static DEFINE_MUTEX(amd_pstate_driver_lock); static s16 msr_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) { - u64 epp; int ret; if (!cppc_req_cached) { @@ -193,9 +198,8 @@ static s16 msr_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) return ret; } } - epp = (cppc_req_cached >> 24) & 0xFF; - return (s16)epp; + return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cppc_req_cached); } DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); @@ -270,12 +274,11 @@ static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, static int msr_set_epp(struct amd_cpudata *cpudata, u32 epp) { - int ret; - u64 value = READ_ONCE(cpudata->cppc_req_cached); + int ret; - value &= ~GENMASK_ULL(31, 24); - value |= (u64)epp << 24; + value &= ~AMD_CPPC_EPP_PERF_MASK; + value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); WRITE_ONCE(cpudata->cppc_req_cached, value); ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); @@ -328,8 +331,8 @@ static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, if (trace_amd_pstate_epp_perf_enabled()) { trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, epp, - AMD_CPPC_MIN_PERF(cpudata->cppc_req_cached), - AMD_CPPC_MAX_PERF(cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), cpudata->boost_state); } @@ -543,18 +546,15 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, des_perf = 0; } - value &= ~AMD_CPPC_MIN_PERF(~0L); - value |= AMD_CPPC_MIN_PERF(min_perf); - - value &= ~AMD_CPPC_DES_PERF(~0L); - value |= AMD_CPPC_DES_PERF(des_perf); - /* limit the max perf when core performance boost feature is disabled */ if (!cpudata->boost_supported) max_perf = min_t(unsigned long, nominal_perf, max_perf); - value &= ~AMD_CPPC_MAX_PERF(~0L); - value |= AMD_CPPC_MAX_PERF(max_perf); + value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | + AMD_CPPC_DES_PERF_MASK); + value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); + value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); + value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, @@ -1565,16 +1565,11 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) min_perf = min(cpudata->nominal_perf, max_perf); - /* Initial min/max values for CPPC Performance Controls Register */ - value &= ~AMD_CPPC_MIN_PERF(~0L); - value |= AMD_CPPC_MIN_PERF(min_perf); - - value &= ~AMD_CPPC_MAX_PERF(~0L); - value |= AMD_CPPC_MAX_PERF(max_perf); - - /* CPPC EPP feature require to set zero to the desire perf bit */ - value &= ~AMD_CPPC_DES_PERF(~0L); - value |= AMD_CPPC_DES_PERF(0); + value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | + AMD_CPPC_DES_PERF_MASK); + value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); + value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, 0); + value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); /* Get BIOS pre-defined epp value */ epp = amd_pstate_get_epp(cpudata, value); @@ -1644,7 +1639,7 @@ static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) if (trace_amd_pstate_epp_perf_enabled()) { trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, cpudata->epp_cached, - AMD_CPPC_MIN_PERF(cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), max_perf, cpudata->boost_state); } From b9f27228f5d2fa5c16a78b31e9bb00b7104c3d19 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:40 -0600 Subject: [PATCH 080/188] cpufreq/amd-pstate: Only update the cached value in msr_set_epp() on success commit 474e7218e81e7932ed18f91969b72169005ff038 upstream. If writing the MSR MSR_AMD_CPPC_REQ fails then the cached value in the amd_cpudata structure should not be updated. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-8-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 326814929544f..f65cd443dde13 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -279,11 +279,15 @@ static int msr_set_epp(struct amd_cpudata *cpudata, u32 epp) value &= ~AMD_CPPC_EPP_PERF_MASK; value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); - WRITE_ONCE(cpudata->cppc_req_cached, value); ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); - if (!ret) - cpudata->epp_cached = epp; + if (ret) { + pr_err("failed to set energy perf value (%d)\n", ret); + return ret; + } + + cpudata->epp_cached = epp; + WRITE_ONCE(cpudata->cppc_req_cached, value); return ret; } From edb2ed47580cee30acc3df9f0b9d8931cec0609c Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:41 -0600 Subject: [PATCH 081/188] cpufreq/amd-pstate: store all values in cpudata struct in khz commit 68cb0e77b6439fea64c6907c563b7bd27f2ee57f upstream. Storing values in the cpudata structure in different units leads to confusion and hardcoded conversions elsewhere. After ratios are calculated store everything in khz for any future use. Adjust all relevant consumers for this change as well. Suggested-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-9-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 12 +++++------- drivers/cpufreq/amd-pstate.c | 28 ++++++++++++++-------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index a261d7300951e..3a0a380c3590c 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -207,7 +207,6 @@ static void amd_pstate_ut_check_freq(u32 index) int cpu = 0; struct cpufreq_policy *policy = NULL; struct amd_cpudata *cpudata = NULL; - u32 nominal_freq_khz; for_each_possible_cpu(cpu) { policy = cpufreq_cpu_get(cpu); @@ -215,14 +214,13 @@ static void amd_pstate_ut_check_freq(u32 index) break; cpudata = policy->driver_data; - nominal_freq_khz = cpudata->nominal_freq*1000; - if (!((cpudata->max_freq >= nominal_freq_khz) && - (nominal_freq_khz > cpudata->lowest_nonlinear_freq) && + if (!((cpudata->max_freq >= cpudata->nominal_freq) && + (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) && (cpudata->lowest_nonlinear_freq > cpudata->min_freq) && (cpudata->min_freq > 0))) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n", - __func__, cpu, cpudata->max_freq, nominal_freq_khz, + __func__, cpu, cpudata->max_freq, cpudata->nominal_freq, cpudata->lowest_nonlinear_freq, cpudata->min_freq); goto skip_test; } @@ -236,13 +234,13 @@ static void amd_pstate_ut_check_freq(u32 index) if (cpudata->boost_supported) { if ((policy->max == cpudata->max_freq) || - (policy->max == nominal_freq_khz)) + (policy->max == cpudata->nominal_freq)) amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; else { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d policy_max=%d should be equal cpu_max=%d or cpu_nominal=%d !\n", __func__, cpu, policy->max, cpudata->max_freq, - nominal_freq_khz); + cpudata->nominal_freq); goto skip_test; } } else { diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index f65cd443dde13..314555197ad2e 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -740,8 +740,8 @@ static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) if (on) policy->cpuinfo.max_freq = max_freq; - else if (policy->cpuinfo.max_freq > nominal_freq * 1000) - policy->cpuinfo.max_freq = nominal_freq * 1000; + else if (policy->cpuinfo.max_freq > nominal_freq) + policy->cpuinfo.max_freq = nominal_freq; policy->max = policy->cpuinfo.max_freq; @@ -932,29 +932,29 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) return ret; if (quirks && quirks->lowest_freq) - min_freq = quirks->lowest_freq * 1000; + min_freq = quirks->lowest_freq; else - min_freq = cppc_perf.lowest_freq * 1000; + min_freq = cppc_perf.lowest_freq; if (quirks && quirks->nominal_freq) - nominal_freq = quirks->nominal_freq ; + nominal_freq = quirks->nominal_freq; else nominal_freq = cppc_perf.nominal_freq; nominal_perf = READ_ONCE(cpudata->nominal_perf); boost_ratio = div_u64(cpudata->highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); - max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT) * 1000; + max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT); lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, nominal_perf); - lowest_nonlinear_freq = (nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT) * 1000; + lowest_nonlinear_freq = (nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT); - WRITE_ONCE(cpudata->min_freq, min_freq); - WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); - WRITE_ONCE(cpudata->nominal_freq, nominal_freq); - WRITE_ONCE(cpudata->max_freq, max_freq); + WRITE_ONCE(cpudata->min_freq, min_freq * 1000); + WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq * 1000); + WRITE_ONCE(cpudata->nominal_freq, nominal_freq * 1000); + WRITE_ONCE(cpudata->max_freq, max_freq * 1000); /** * Below values need to be initialized correctly, otherwise driver will fail to load @@ -964,13 +964,13 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) */ if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) { pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n", - min_freq, max_freq, nominal_freq * 1000); + min_freq, max_freq, nominal_freq); return -EINVAL; } - if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq * 1000) { + if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq) { pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n", - lowest_nonlinear_freq, min_freq, nominal_freq * 1000); + lowest_nonlinear_freq, min_freq, nominal_freq); return -EINVAL; } From 87eafbb92fd3a8b853a0de1e0bdbd95892ad9143 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:42 -0600 Subject: [PATCH 082/188] cpufreq/amd-pstate: Change amd_pstate_update_perf() to return an int commit 942718f2a236cb3b27d2dbb5942538681b6e0e88 upstream. As msr_update_perf() calls an MSR it's possible that it fails. Pass this return code up to the caller. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-10-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 314555197ad2e..6aa84e5530aa6 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -252,24 +252,26 @@ static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) return index; } -static void msr_update_perf(struct amd_cpudata *cpudata, u32 min_perf, +static int msr_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, u32 max_perf, bool fast_switch) { - if (fast_switch) + if (fast_switch) { wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); - else - wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, - READ_ONCE(cpudata->cppc_req_cached)); + return 0; + } + + return wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, + READ_ONCE(cpudata->cppc_req_cached)); } DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); -static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, +static inline int amd_pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, u32 max_perf, bool fast_switch) { - static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, - max_perf, fast_switch); + return static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, + max_perf, fast_switch); } static int msr_set_epp(struct amd_cpudata *cpudata, u32 epp) @@ -481,7 +483,7 @@ static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) return static_call(amd_pstate_init_perf)(cpudata); } -static void shmem_update_perf(struct amd_cpudata *cpudata, +static int shmem_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, u32 max_perf, bool fast_switch) { @@ -491,7 +493,7 @@ static void shmem_update_perf(struct amd_cpudata *cpudata, perf_ctrls.min_perf = min_perf; perf_ctrls.desired_perf = des_perf; - cppc_set_perf(cpudata->cpu, &perf_ctrls); + return cppc_set_perf(cpudata->cpu, &perf_ctrls); } static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) From 466bc6bf887debc9076d05d312c72495e2b7d9b0 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:43 -0600 Subject: [PATCH 083/188] cpufreq/amd-pstate: Move limit updating code commit 3f7b835fa4d0d06f82249a3aca989fdf9bdf4656 upstream. The limit updating code in amd_pstate_epp_update_limit() should not only apply to EPP updates. Move it to amd_pstate_update_min_max_limit() so other callers can benefit as well. With this move it's not necessary to have clamp_t calls anymore because the verify callback is called when setting limits. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-11-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 6aa84e5530aa6..8f6dc26a6ff03 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -538,10 +538,6 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, u32 nominal_perf = READ_ONCE(cpudata->nominal_perf); u64 value = prev; - min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, - cpudata->max_limit_perf); - max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, - cpudata->max_limit_perf); des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); max_freq = READ_ONCE(cpudata->max_limit_freq); @@ -608,7 +604,7 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) { - u32 max_limit_perf, min_limit_perf, lowest_perf, max_perf, max_freq; + u32 max_limit_perf, min_limit_perf, max_perf, max_freq; struct amd_cpudata *cpudata = policy->driver_data; max_perf = READ_ONCE(cpudata->highest_perf); @@ -616,12 +612,8 @@ static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) max_limit_perf = div_u64(policy->max * max_perf, max_freq); min_limit_perf = div_u64(policy->min * max_perf, max_freq); - lowest_perf = READ_ONCE(cpudata->lowest_perf); - if (min_limit_perf < lowest_perf) - min_limit_perf = lowest_perf; - - if (max_limit_perf < min_limit_perf) - max_limit_perf = min_limit_perf; + if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) + min_limit_perf = min(cpudata->nominal_perf, max_limit_perf); WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); @@ -1554,28 +1546,18 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - u32 max_perf, min_perf; u64 value; s16 epp; - max_perf = READ_ONCE(cpudata->highest_perf); - min_perf = READ_ONCE(cpudata->lowest_perf); amd_pstate_update_min_max_limit(policy); - max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, - cpudata->max_limit_perf); - min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, - cpudata->max_limit_perf); value = READ_ONCE(cpudata->cppc_req_cached); - if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) - min_perf = min(cpudata->nominal_perf, max_perf); - value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | AMD_CPPC_DES_PERF_MASK); - value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); + value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, cpudata->max_limit_perf); value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, 0); - value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); + value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, cpudata->min_limit_perf); /* Get BIOS pre-defined epp value */ epp = amd_pstate_get_epp(cpudata, value); From 9dc67948d962c9a7d94052c9a0693c0c82678b36 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:44 -0600 Subject: [PATCH 084/188] cpufreq/amd-pstate: Cache EPP value and use that everywhere commit b3781f30bfcfd7db12de2595adb01779e565e1c6 upstream. Cache the value in cpudata->epp_cached, and use that for all callers. As all callers use cached value merge amd_pstate_get_energy_pref_index() into show_energy_performance_preference(). Check if the EPP value is changed before writing it to MSR or shared memory region. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-12-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 105 ++++++++++++++--------------------- 1 file changed, 43 insertions(+), 62 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 8f6dc26a6ff03..a86b446b7eb3c 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -187,29 +187,28 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) static DEFINE_MUTEX(amd_pstate_limits_lock); static DEFINE_MUTEX(amd_pstate_driver_lock); -static s16 msr_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) +static s16 msr_get_epp(struct amd_cpudata *cpudata) { + u64 value; int ret; - if (!cppc_req_cached) { - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req_cached); - if (ret < 0) { - pr_debug("Could not retrieve energy perf value (%d)\n", ret); - return ret; - } + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); + if (ret < 0) { + pr_debug("Could not retrieve energy perf value (%d)\n", ret); + return ret; } - return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cppc_req_cached); + return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, value); } DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); -static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) +static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) { - return static_call(amd_pstate_get_epp)(cpudata, cppc_req_cached); + return static_call(amd_pstate_get_epp)(cpudata); } -static s16 shmem_get_epp(struct amd_cpudata *cpudata, u64 dummy) +static s16 shmem_get_epp(struct amd_cpudata *cpudata) { u64 epp; int ret; @@ -223,35 +222,6 @@ static s16 shmem_get_epp(struct amd_cpudata *cpudata, u64 dummy) return (s16)(epp & 0xff); } -static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) -{ - s16 epp; - int index = -EINVAL; - - epp = amd_pstate_get_epp(cpudata, 0); - if (epp < 0) - return epp; - - switch (epp) { - case AMD_CPPC_EPP_PERFORMANCE: - index = EPP_INDEX_PERFORMANCE; - break; - case AMD_CPPC_EPP_BALANCE_PERFORMANCE: - index = EPP_INDEX_BALANCE_PERFORMANCE; - break; - case AMD_CPPC_EPP_BALANCE_POWERSAVE: - index = EPP_INDEX_BALANCE_POWERSAVE; - break; - case AMD_CPPC_EPP_POWERSAVE: - index = EPP_INDEX_POWERSAVE; - break; - default: - break; - } - - return index; -} - static int msr_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, u32 max_perf, bool fast_switch) { @@ -276,19 +246,23 @@ static inline int amd_pstate_update_perf(struct amd_cpudata *cpudata, static int msr_set_epp(struct amd_cpudata *cpudata, u32 epp) { - u64 value = READ_ONCE(cpudata->cppc_req_cached); + u64 value, prev; int ret; + value = prev = READ_ONCE(cpudata->cppc_req_cached); value &= ~AMD_CPPC_EPP_PERF_MASK; value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + if (value == prev) + return 0; + ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); if (ret) { pr_err("failed to set energy perf value (%d)\n", ret); return ret; } - cpudata->epp_cached = epp; + WRITE_ONCE(cpudata->epp_cached, epp); WRITE_ONCE(cpudata->cppc_req_cached, value); return ret; @@ -306,13 +280,16 @@ static int shmem_set_epp(struct amd_cpudata *cpudata, u32 epp) int ret; struct cppc_perf_ctrls perf_ctrls; + if (epp == cpudata->epp_cached) + return 0; + perf_ctrls.energy_perf = epp; ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); if (ret) { pr_debug("failed to set energy perf value (%d)\n", ret); return ret; } - cpudata->epp_cached = epp; + WRITE_ONCE(cpudata->epp_cached, epp); return ret; } @@ -1206,9 +1183,22 @@ static ssize_t show_energy_performance_preference( struct amd_cpudata *cpudata = policy->driver_data; int preference; - preference = amd_pstate_get_energy_pref_index(cpudata); - if (preference < 0) - return preference; + switch (cpudata->epp_cached) { + case AMD_CPPC_EPP_PERFORMANCE: + preference = EPP_INDEX_PERFORMANCE; + break; + case AMD_CPPC_EPP_BALANCE_PERFORMANCE: + preference = EPP_INDEX_BALANCE_PERFORMANCE; + break; + case AMD_CPPC_EPP_BALANCE_POWERSAVE: + preference = EPP_INDEX_BALANCE_POWERSAVE; + break; + case AMD_CPPC_EPP_POWERSAVE: + preference = EPP_INDEX_POWERSAVE; + break; + default: + return -EINVAL; + } return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); } @@ -1493,7 +1483,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->driver_data = cpudata; - cpudata->epp_cached = cpudata->epp_default = amd_pstate_get_epp(cpudata, 0); + cpudata->epp_cached = cpudata->epp_default = amd_pstate_get_epp(cpudata); policy->min = policy->cpuinfo.min_freq; policy->max = policy->cpuinfo.max_freq; @@ -1547,35 +1537,26 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; u64 value; - s16 epp; amd_pstate_update_min_max_limit(policy); value = READ_ONCE(cpudata->cppc_req_cached); value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | - AMD_CPPC_DES_PERF_MASK); + AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, cpudata->max_limit_perf); value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, 0); value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, cpudata->min_limit_perf); - /* Get BIOS pre-defined epp value */ - epp = amd_pstate_get_epp(cpudata, value); - if (epp < 0) { - /** - * This return value can only be negative for shared_memory - * systems where EPP register read/write not supported. - */ - return epp; - } - if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) - epp = 0; + WRITE_ONCE(cpudata->epp_cached, 0); + value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, cpudata->epp_cached); WRITE_ONCE(cpudata->cppc_req_cached, value); if (trace_amd_pstate_epp_perf_enabled()) { - trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, epp, + trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, + cpudata->epp_cached, cpudata->min_limit_perf, cpudata->max_limit_perf, policy->boost_enabled); @@ -1584,7 +1565,7 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, cpudata->max_limit_perf, false); - return amd_pstate_set_epp(cpudata, epp); + return amd_pstate_set_epp(cpudata, READ_ONCE(cpudata->epp_cached)); } static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) From 0679f709e9b4ed2a30db14ea6cf27d47b8a34078 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:45 -0600 Subject: [PATCH 085/188] cpufreq/amd-pstate: Always write EPP value when updating perf commit fff395796917ac3fe3b4c4607cb74a8dbdc17593 upstream. For MSR systems the EPP value is in the same register as perf targets and so divding them into two separate MSR writes is wasteful. In msr_update_perf(), update both EPP and perf values in one write to MSR_AMD_CPPC_REQ, and cache them if successful. To accomplish this plumb the EPP value into the update_perf call and modify all its callers to check the return value. As this unifies calls, ensure that the MSR write is necessary before flushing a write out. Also drop the comparison from the passive flow tracing. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-13-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-trace.h | 7 +- drivers/cpufreq/amd-pstate.c | 108 +++++++++++++++-------------- 2 files changed, 56 insertions(+), 59 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-trace.h b/drivers/cpufreq/amd-pstate-trace.h index e2221a4b6901c..8d692415d9050 100644 --- a/drivers/cpufreq/amd-pstate-trace.h +++ b/drivers/cpufreq/amd-pstate-trace.h @@ -32,7 +32,6 @@ TRACE_EVENT(amd_pstate_perf, u64 aperf, u64 tsc, unsigned int cpu_id, - bool changed, bool fast_switch ), @@ -44,7 +43,6 @@ TRACE_EVENT(amd_pstate_perf, aperf, tsc, cpu_id, - changed, fast_switch ), @@ -57,7 +55,6 @@ TRACE_EVENT(amd_pstate_perf, __field(unsigned long long, aperf) __field(unsigned long long, tsc) __field(unsigned int, cpu_id) - __field(bool, changed) __field(bool, fast_switch) ), @@ -70,11 +67,10 @@ TRACE_EVENT(amd_pstate_perf, __entry->aperf = aperf; __entry->tsc = tsc; __entry->cpu_id = cpu_id; - __entry->changed = changed; __entry->fast_switch = fast_switch; ), - TP_printk("amd_min_perf=%lu amd_des_perf=%lu amd_max_perf=%lu freq=%llu mperf=%llu aperf=%llu tsc=%llu cpu_id=%u changed=%s fast_switch=%s", + TP_printk("amd_min_perf=%lu amd_des_perf=%lu amd_max_perf=%lu freq=%llu mperf=%llu aperf=%llu tsc=%llu cpu_id=%u fast_switch=%s", (unsigned long)__entry->min_perf, (unsigned long)__entry->target_perf, (unsigned long)__entry->capacity, @@ -83,7 +79,6 @@ TRACE_EVENT(amd_pstate_perf, (unsigned long long)__entry->aperf, (unsigned long long)__entry->tsc, (unsigned int)__entry->cpu_id, - (__entry->changed) ? "true" : "false", (__entry->fast_switch) ? "true" : "false" ) ); diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index a86b446b7eb3c..bd028022f1dbd 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -223,25 +223,47 @@ static s16 shmem_get_epp(struct amd_cpudata *cpudata) } static int msr_update_perf(struct amd_cpudata *cpudata, u32 min_perf, - u32 des_perf, u32 max_perf, bool fast_switch) + u32 des_perf, u32 max_perf, u32 epp, bool fast_switch) { + u64 value, prev; + + value = prev = READ_ONCE(cpudata->cppc_req_cached); + + value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | + AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); + value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); + value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); + value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); + value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + + if (value == prev) + return 0; + if (fast_switch) { - wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); + wrmsrl(MSR_AMD_CPPC_REQ, value); return 0; + } else { + int ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); + + if (ret) + return ret; } - return wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, - READ_ONCE(cpudata->cppc_req_cached)); + WRITE_ONCE(cpudata->cppc_req_cached, value); + WRITE_ONCE(cpudata->epp_cached, epp); + + return 0; } DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); static inline int amd_pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, u32 des_perf, - u32 max_perf, bool fast_switch) + u32 max_perf, u32 epp, + bool fast_switch) { return static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, - max_perf, fast_switch); + max_perf, epp, fast_switch); } static int msr_set_epp(struct amd_cpudata *cpudata, u32 epp) @@ -262,6 +284,7 @@ static int msr_set_epp(struct amd_cpudata *cpudata, u32 epp) return ret; } + /* update both so that msr_update_perf() can effectively check */ WRITE_ONCE(cpudata->epp_cached, epp); WRITE_ONCE(cpudata->cppc_req_cached, value); @@ -460,12 +483,18 @@ static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) return static_call(amd_pstate_init_perf)(cpudata); } -static int shmem_update_perf(struct amd_cpudata *cpudata, - u32 min_perf, u32 des_perf, - u32 max_perf, bool fast_switch) +static int shmem_update_perf(struct amd_cpudata *cpudata, u32 min_perf, + u32 des_perf, u32 max_perf, u32 epp, bool fast_switch) { struct cppc_perf_ctrls perf_ctrls; + if (cppc_state == AMD_PSTATE_ACTIVE) { + int ret = shmem_set_epp(cpudata, epp); + + if (ret) + return ret; + } + perf_ctrls.max_perf = max_perf; perf_ctrls.min_perf = min_perf; perf_ctrls.desired_perf = des_perf; @@ -511,9 +540,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, { unsigned long max_freq; struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu); - u64 prev = READ_ONCE(cpudata->cppc_req_cached); u32 nominal_perf = READ_ONCE(cpudata->nominal_perf); - u64 value = prev; des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); @@ -529,27 +556,14 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, if (!cpudata->boost_supported) max_perf = min_t(unsigned long, nominal_perf, max_perf); - value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | - AMD_CPPC_DES_PERF_MASK); - value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); - value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); - value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); - if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, - cpudata->cpu, (value != prev), fast_switch); + cpudata->cpu, fast_switch); } - if (value == prev) - goto cpufreq_policy_put; + amd_pstate_update_perf(cpudata, min_perf, des_perf, max_perf, 0, fast_switch); - WRITE_ONCE(cpudata->cppc_req_cached, value); - - amd_pstate_update_perf(cpudata, min_perf, des_perf, - max_perf, fast_switch); - -cpufreq_policy_put: cpufreq_cpu_put(policy); } @@ -1536,36 +1550,24 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - u64 value; + u32 epp; amd_pstate_update_min_max_limit(policy); - value = READ_ONCE(cpudata->cppc_req_cached); - - value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | - AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); - value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, cpudata->max_limit_perf); - value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, 0); - value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, cpudata->min_limit_perf); - if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) - WRITE_ONCE(cpudata->epp_cached, 0); - value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, cpudata->epp_cached); - - WRITE_ONCE(cpudata->cppc_req_cached, value); + epp = 0; + else + epp = READ_ONCE(cpudata->epp_cached); if (trace_amd_pstate_epp_perf_enabled()) { - trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, - cpudata->epp_cached, + trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, epp, cpudata->min_limit_perf, cpudata->max_limit_perf, policy->boost_enabled); } - amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, - cpudata->max_limit_perf, false); - - return amd_pstate_set_epp(cpudata, READ_ONCE(cpudata->epp_cached)); + return amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, + cpudata->max_limit_perf, epp, false); } static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) @@ -1594,7 +1596,7 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) return 0; } -static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) +static int amd_pstate_epp_reenable(struct amd_cpudata *cpudata) { u64 max_perf; int ret; @@ -1612,17 +1614,19 @@ static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) max_perf, cpudata->boost_state); } - amd_pstate_update_perf(cpudata, 0, 0, max_perf, false); - amd_pstate_set_epp(cpudata, cpudata->epp_cached); + return amd_pstate_update_perf(cpudata, 0, 0, max_perf, cpudata->epp_cached, false); } static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; + int ret; pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); - amd_pstate_epp_reenable(cpudata); + ret = amd_pstate_epp_reenable(cpudata); + if (ret) + return ret; cpudata->suspended = false; return 0; @@ -1646,10 +1650,8 @@ static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) min_perf, min_perf, policy->boost_enabled); } - amd_pstate_update_perf(cpudata, min_perf, 0, min_perf, false); - amd_pstate_set_epp(cpudata, AMD_CPPC_EPP_BALANCE_POWERSAVE); - - return 0; + return amd_pstate_update_perf(cpudata, min_perf, 0, min_perf, + AMD_CPPC_EPP_BALANCE_POWERSAVE, false); } static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) From f0e4d87662d259bd1e70ba69a5a129cece4b48cb Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:46 -0600 Subject: [PATCH 086/188] cpufreq/amd-pstate: Drop ret variable from amd_pstate_set_energy_pref_index() commit f8fde687c911a366a6132aed85f4ee6b647b9160 upstream. The ret variable is not necessary. Reviewed-and-tested-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-14-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index bd028022f1dbd..e38c089013b4a 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -320,13 +320,11 @@ static int shmem_set_epp(struct amd_cpudata *cpudata, u32 epp) static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, int pref_index) { - int epp = -EINVAL; - int ret; + int epp; if (!pref_index) epp = cpudata->epp_default; - - if (epp == -EINVAL) + else epp = epp_values[pref_index]; if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { @@ -342,9 +340,7 @@ static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, cpudata->boost_state); } - ret = amd_pstate_set_epp(cpudata, epp); - - return ret; + return amd_pstate_set_epp(cpudata, epp); } static inline int msr_cppc_enable(bool enable) From 642e960b484c989f6939bee07c86ebc6c1c5772c Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:47 -0600 Subject: [PATCH 087/188] cpufreq/amd-pstate: Set different default EPP policy for Epyc and Ryzen commit f9a378ff6443cdcd4387e5dbb76fa5fa549a83ec upstream. For Ryzen systems the EPP policy set by the BIOS is generally configured to performance as this is the default register value for the CPPC request MSR. If a user doesn't use additional software to configure EPP then the system will default biased towards performance and consume extra battery. Instead configure the default to "balanced_performance" for this case. Suggested-by: Artem S. Tashkinov Reviewed-by: Dhananjay Ugwekar Tested-by: Dhananjay Ugwekar Closes: https://bugzilla.kernel.org/show_bug.cgi?id=219526 Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-15-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index e38c089013b4a..d542f623aef5e 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1493,8 +1493,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->driver_data = cpudata; - cpudata->epp_cached = cpudata->epp_default = amd_pstate_get_epp(cpudata); - policy->min = policy->cpuinfo.min_freq; policy->max = policy->cpuinfo.max_freq; @@ -1505,10 +1503,13 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) * the default cpufreq governor is neither powersave nor performance. */ if (amd_pstate_acpi_pm_profile_server() || - amd_pstate_acpi_pm_profile_undefined()) + amd_pstate_acpi_pm_profile_undefined()) { policy->policy = CPUFREQ_POLICY_PERFORMANCE; - else + cpudata->epp_default = amd_pstate_get_epp(cpudata); + } else { policy->policy = CPUFREQ_POLICY_POWERSAVE; + cpudata->epp_default = AMD_CPPC_EPP_BALANCE_PERFORMANCE; + } if (cpu_feature_enabled(X86_FEATURE_CPPC)) { ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); @@ -1521,6 +1522,9 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) return ret; WRITE_ONCE(cpudata->cppc_cap1_cached, value); } + ret = amd_pstate_set_epp(cpudata, cpudata->epp_default); + if (ret) + return ret; current_pstate_driver->adjust_perf = NULL; From 0d5b956b3c7ee36666b69f277b307060814c5b1c Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:52:48 -0600 Subject: [PATCH 088/188] cpufreq/amd-pstate: Drop boost_state variable commit 95fad7fb58cfaa2a295aa54a1f001a16b9324963 upstream. Currently boost_state is cached for every processor in cpudata structure and driver boost state is set for every processor. Both of these aren't necessary as the driver only needs to set once and the policy stores whether boost is enabled. Move the driver boost setting to registration and adjust all references to cached value to pull from the policy instead. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241209185248.16301-16-mario.limonciello@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 26 +++++++++++++------------- drivers/cpufreq/amd-pstate.h | 1 - 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index d542f623aef5e..fb9347e8e70d0 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -317,9 +317,10 @@ static int shmem_set_epp(struct amd_cpudata *cpudata, u32 epp) return ret; } -static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, - int pref_index) +static int amd_pstate_set_energy_pref_index(struct cpufreq_policy *policy, + int pref_index) { + struct amd_cpudata *cpudata = policy->driver_data; int epp; if (!pref_index) @@ -337,7 +338,7 @@ static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, epp, FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), - cpudata->boost_state); + policy->boost_enabled); } return amd_pstate_set_epp(cpudata, epp); @@ -747,7 +748,6 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) guard(mutex)(&amd_pstate_driver_lock); ret = amd_pstate_cpu_boost_update(policy, state); - WRITE_ONCE(cpudata->boost_state, !ret ? state : false); policy->boost_enabled = !ret ? state : false; refresh_frequency_limits(policy); @@ -769,9 +769,6 @@ static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata) goto exit_err; } - /* at least one CPU supports CPB, even if others fail later on to set up */ - current_pstate_driver->boost_enabled = true; - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val); if (ret) { pr_err_once("failed to read initial CPU boost state!\n"); @@ -1168,7 +1165,6 @@ static ssize_t show_energy_performance_available_preferences( static ssize_t store_energy_performance_preference( struct cpufreq_policy *policy, const char *buf, size_t count) { - struct amd_cpudata *cpudata = policy->driver_data; char str_preference[21]; ssize_t ret; @@ -1182,7 +1178,7 @@ static ssize_t store_energy_performance_preference( guard(mutex)(&amd_pstate_limits_lock); - ret = amd_pstate_set_energy_pref_index(cpudata, ret); + ret = amd_pstate_set_energy_pref_index(policy, ret); return ret ? ret : count; } @@ -1257,6 +1253,9 @@ static int amd_pstate_register_driver(int mode) return ret; } + /* at least one CPU supports CPB */ + current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); + ret = cpufreq_register_driver(current_pstate_driver); if (ret) { amd_pstate_driver_cleanup(); @@ -1596,8 +1595,9 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) return 0; } -static int amd_pstate_epp_reenable(struct amd_cpudata *cpudata) +static int amd_pstate_epp_reenable(struct cpufreq_policy *policy) { + struct amd_cpudata *cpudata = policy->driver_data; u64 max_perf; int ret; @@ -1611,7 +1611,7 @@ static int amd_pstate_epp_reenable(struct amd_cpudata *cpudata) trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, cpudata->epp_cached, FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), - max_perf, cpudata->boost_state); + max_perf, policy->boost_enabled); } return amd_pstate_update_perf(cpudata, 0, 0, max_perf, cpudata->epp_cached, false); @@ -1624,7 +1624,7 @@ static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); - ret = amd_pstate_epp_reenable(cpudata); + ret = amd_pstate_epp_reenable(policy); if (ret) return ret; cpudata->suspended = false; @@ -1682,7 +1682,7 @@ static int amd_pstate_epp_resume(struct cpufreq_policy *policy) guard(mutex)(&amd_pstate_limits_lock); /* enable amd pstate from suspend state*/ - amd_pstate_epp_reenable(cpudata); + amd_pstate_epp_reenable(policy); cpudata->suspended = false; } diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 7765c82f975c6..9747e3be6ceee 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -98,7 +98,6 @@ struct amd_cpudata { u64 cppc_cap1_cached; bool suspended; s16 epp_default; - bool boost_state; }; /* From 2e58110318c62fdd8bba6446bfd5a2ecd67c4d81 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 2 Jan 2025 08:12:04 -0600 Subject: [PATCH 089/188] cpufreq/amd-pstate: Fix prefcore rankings commit fd604ae6c261c5a56bb977ae99f875bbd7264a3f upstream. commit 50a062a76200 ("cpufreq/amd-pstate: Store the boost numerator as highest perf again") updated the value stored for highest perf to no longer store the highest perf value but instead the boost numerator. This is a fixed value for systems with preferred cores and not appropriate for use ITMT rankings. Update the value used for ITMT rankings to be the preferred core ranking. Reported-and-tested-by: Sebastian Closes: https://bugzilla.kernel.org/show_bug.cgi?id=219640 Fixes: 50a062a76200 ("cpufreq/amd-pstate: Store the boost numerator as highest perf again") Reviewed-by: Dhananjay Ugwekar Link: https://lore.kernel.org/r/20250102141204.3413202-1-superm1@kernel.org Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index fb9347e8e70d0..fe5473bcc12ee 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -816,7 +816,7 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) * sched_set_itmt_support(true) has been called and it is valid to * update them at any time after it has been called. */ - sched_set_itmt_core_prio((int)READ_ONCE(cpudata->highest_perf), cpudata->cpu); + sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); schedule_work(&sched_prefcore_work); } From b5d793d68ff7a55386bf294498187623165d418b Mon Sep 17 00:00:00 2001 From: Naresh Solanki Date: Fri, 20 Dec 2024 01:48:32 +0530 Subject: [PATCH 090/188] cpufreq/amd-pstate: Refactor max frequency calculation commit 857a61c2ce74e30fc3b10bc89d68ddd8d05b188c upstream. The previous approach introduced roundoff errors during division when calculating the boost ratio. This, in turn, affected the maximum frequency calculation, often resulting in reporting lower frequency values. For example, on the Glinda SoC based board with the following parameters: max_perf = 208 nominal_perf = 100 nominal_freq = 2600 MHz The Linux kernel previously calculated the frequency as: freq = ((max_perf * 1024 / nominal_perf) * nominal_freq) / 1024 freq = 5405 MHz // Integer arithmetic. With the updated formula: freq = (max_perf * nominal_freq) / nominal_perf freq = 5408 MHz This change ensures more accurate frequency calculations by eliminating unnecessary shifts and divisions, thereby improving precision. Signed-off-by: Naresh Solanki [ML: trim the changelog from commit message] Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20241219201833.2750998-1-naresh.solanki@9elements.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index fe5473bcc12ee..291cd4db9af57 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -900,9 +900,8 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) { int ret; u32 min_freq, max_freq; - u32 nominal_perf, nominal_freq; + u32 highest_perf, nominal_perf, nominal_freq; u32 lowest_nonlinear_perf, lowest_nonlinear_freq; - u32 boost_ratio, lowest_nonlinear_ratio; struct cppc_perf_caps cppc_perf; ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); @@ -919,16 +918,12 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) else nominal_freq = cppc_perf.nominal_freq; + highest_perf = READ_ONCE(cpudata->highest_perf); nominal_perf = READ_ONCE(cpudata->nominal_perf); - - boost_ratio = div_u64(cpudata->highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); - max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT); + max_freq = div_u64((u64)highest_perf * nominal_freq, nominal_perf); lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); - lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, - nominal_perf); - lowest_nonlinear_freq = (nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT); - + lowest_nonlinear_freq = div_u64((u64)nominal_freq * lowest_nonlinear_perf, nominal_perf); WRITE_ONCE(cpudata->min_freq, min_freq * 1000); WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq * 1000); WRITE_ONCE(cpudata->nominal_freq, nominal_freq * 1000); From 0cac3f7ddbc319ceff25daab7aef87912f4e1d9d Mon Sep 17 00:00:00 2001 From: Lifeng Zheng Date: Fri, 10 Jan 2025 17:19:49 +0800 Subject: [PATCH 091/188] cpufreq/amd-pstate: Fix per-policy boost flag incorrect when fail commit fa803513ab68ba07369643393f1754b845160030 upstream. Commit c8c68c38b56f ("cpufreq: amd-pstate: initialize core precision boost state") sets per-policy boost flag to false when boost fail. However, this boost flag will be set to reverse value in store_local_boost() and cpufreq_boost_trigger_state() in cpufreq.c. This will cause the per-policy boost flag set to true when fail to set boost. Remove the extra assignment in amd_pstate_set_boost() and keep all operations on per-policy boost flag outside of set_boost() to fix this problem. Fixes: c8c68c38b56f ("cpufreq: amd-pstate: initialize core precision boost state") Signed-off-by: Lifeng Zheng Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20250110091949.3610770-1-zhenglifeng1@huawei.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 291cd4db9af57..272022a5a3bc8 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -748,7 +748,6 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) guard(mutex)(&amd_pstate_driver_lock); ret = amd_pstate_cpu_boost_update(policy, state); - policy->boost_enabled = !ret ? state : false; refresh_frequency_limits(policy); return ret; From 8eaa4e8d2995a2651d56a7c8f80414183088a2f0 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:12 +0000 Subject: [PATCH 092/188] cpufreq/amd-pstate: Remove the goto label in amd_pstate_update_limits commit d364eee14c682b141f4667efc3c65191339d88bd upstream. Scope based guard/cleanup macros should not be used together with goto labels. Hence, remove the goto label. Fixes: 6c093d5a5b73 ("cpufreq/amd-pstate: convert mutex use to guard()") Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20250205112523.201101-2-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 272022a5a3bc8..9f00249b6722d 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -834,8 +834,10 @@ static void amd_pstate_update_limits(unsigned int cpu) guard(mutex)(&amd_pstate_driver_lock); ret = amd_get_highest_perf(cpu, &cur_high); - if (ret) - goto free_cpufreq_put; + if (ret) { + cpufreq_cpu_put(policy); + return; + } prev_high = READ_ONCE(cpudata->prefcore_ranking); highest_perf_changed = (prev_high != cur_high); @@ -845,8 +847,6 @@ static void amd_pstate_update_limits(unsigned int cpu) if (cur_high < CPPC_MAX_PERF) sched_set_itmt_core_prio((int)cur_high, cpu); } - -free_cpufreq_put: cpufreq_cpu_put(policy); if (!highest_perf_changed) From deed13a4da35d3a78bfbb9ee0a941d7118ae8d6b Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:13 +0000 Subject: [PATCH 093/188] cpufreq/amd-pstate: Fix max_perf updation with schedutil commit 55db9b73c3a77544efc671d5e796d9674772c330 upstream. In adjust_perf() callback, we are setting the max_perf to highest_perf, as opposed to the correct limit value i.e. max_limit_perf. Fix that. Fixes: 3f7b835fa4d0 ("cpufreq/amd-pstate: Move limit updating code") Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20250205112523.201101-3-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 9f00249b6722d..47a579e1d223f 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -700,7 +700,7 @@ static void amd_pstate_adjust_perf(unsigned int cpu, if (min_perf < lowest_nonlinear_perf) min_perf = lowest_nonlinear_perf; - max_perf = cap_perf; + max_perf = cpudata->max_limit_perf; if (max_perf < min_perf) max_perf = min_perf; From 70c8fc0bb0d443d4939caddf434e6fe9a59baaf6 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:20 +0000 Subject: [PATCH 094/188] cpufreq/amd-pstate: Fix cpufreq_policy ref counting commit 3ace20038e19f23fe73259513f1f08d4bf1a3c83 upstream. amd_pstate_update_limits() takes a cpufreq_policy reference but doesn't decrement the refcount in one of the exit paths, fix that. [Backport Changes] 1. In drivers/cpufreq/amd-pstate.c, within amd_pstate_update_limits(), the relevant changes from upstream commit 5493f9714e4cdaf were backported manually to avoid a possible NULL-dereference caused by cpufreq_cpu_get(). The backport initializes policy to NULL, adds a NULL check after cpufreq_cpu_get(), and assigns cpudata only after validating policy. The original commit 5493f9714e4cdaf could not be applied directly because part of it is already present in the Velinux tree through commit 8859eeee99c6, while the implementation of amd_pstate_update_limits() differs from upstream. Fixes: 45722e777fd9 ("cpufreq: amd-pstate: Optimize amd_pstate_update_limits()") Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20250205112523.201101-10-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 47a579e1d223f..e3a7ce6569514 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -822,8 +822,8 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) static void amd_pstate_update_limits(unsigned int cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); - struct amd_cpudata *cpudata = policy->driver_data; + struct cpufreq_policy *policy = NULL; + struct amd_cpudata *cpudata; u32 prev_high = 0, cur_high = 0; int ret; bool highest_perf_changed = false; @@ -831,6 +831,12 @@ static void amd_pstate_update_limits(unsigned int cpu) if (!amd_pstate_prefcore) return; + policy = cpufreq_cpu_get(cpu); + if (!policy) + return; + + cpudata = policy->driver_data; + guard(mutex)(&amd_pstate_driver_lock); ret = amd_get_highest_perf(cpu, &cur_high); From 5b70e2285c36949c1046a1733f28020140fb5c99 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Thu, 30 Jan 2025 08:52:52 +0000 Subject: [PATCH 095/188] cpufreq: amd-pstate: Remove unnecessary driver_lock in set_boost commit db1cafc77aaaf871509da06f4a864e9af6d6791f upstream. set_boost is a per-policy function call, hence a driver wide lock is unnecessary. Also this mutex_acquire can collide with the mutex_acquire from the mode-switch path in status_store(), which can lead to a deadlock. So, remove it. Signed-off-by: Dhananjay Ugwekar Acked-by: Mario Limonciello Signed-off-by: Viresh Kumar Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index e3a7ce6569514..18a185e0348d1 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -745,7 +745,6 @@ static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) pr_err("Boost mode is not supported by this processor or SBIOS\n"); return -EOPNOTSUPP; } - guard(mutex)(&amd_pstate_driver_lock); ret = amd_pstate_cpu_boost_update(policy, state); refresh_frequency_limits(policy); From 8ef32ae4c60b427d323a48089e104e6895efd1d8 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 23 Jan 2025 14:38:03 +0530 Subject: [PATCH 096/188] cpufreq: amd: Set policy->boost_supported commit 98f39e93d102af743d173f76ca12fd5fcacbb0ea upstream. With a later commit, the cpufreq core will call the ->set_boost() callback only if the policy supports boost frequency. The boost_supported flag is set by the cpufreq core if policy->freq_table is set and one or more boost frequencies are present. For other drivers, the flag must be set explicitly. The policy->boost_enabled flag is set by the cpufreq core once the policy is initialized, don't set it anymore. Signed-off-by: Viresh Kumar Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 18a185e0348d1..6b5f452ebde7a 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1001,7 +1001,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.min_freq = min_freq; policy->cpuinfo.max_freq = max_freq; - policy->boost_enabled = READ_ONCE(cpudata->boost_supported); + policy->boost_supported = READ_ONCE(cpudata->boost_supported); /* It will be updated by governor */ policy->cur = policy->cpuinfo.min_freq; @@ -1494,7 +1494,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->min = policy->cpuinfo.min_freq; policy->max = policy->cpuinfo.max_freq; - policy->boost_enabled = READ_ONCE(cpudata->boost_supported); + policy->boost_supported = READ_ONCE(cpudata->boost_supported); /* * Set the policy to provide a valid fallback value in case From 84b82593e6ca78e701c897243570e6b30b468c37 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:14 +0000 Subject: [PATCH 097/188] cpufreq/amd-pstate: Modify the min_perf calculation in adjust_perf callback commit 6ceb877d5cecd5417d63239bf833a1cd5f8f271c upstream. Instead of setting a fixed floor at lowest_nonlinear_perf, use the min_limit_perf value, so that it gives the user the freedom to lower the floor further. There are two minimum frequency/perf limits that we need to consider in the adjust_perf callback. One provided by schedutil i.e. the sg_cpu->bw_min value passed in _min_perf arg, another is the effective value of min_freq_qos request that is updated in cpudata->min_limit_perf. Modify the code to use the bigger of these two values. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20250205112523.201101-4-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 6b5f452ebde7a..7fa9abb5aaaba 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -673,7 +673,7 @@ static void amd_pstate_adjust_perf(unsigned int cpu, unsigned long capacity) { unsigned long max_perf, min_perf, des_perf, - cap_perf, lowest_nonlinear_perf; + cap_perf, min_limit_perf; struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); struct amd_cpudata *cpudata; @@ -685,20 +685,20 @@ static void amd_pstate_adjust_perf(unsigned int cpu, if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) amd_pstate_update_min_max_limit(policy); - cap_perf = READ_ONCE(cpudata->highest_perf); - lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); + min_limit_perf = READ_ONCE(cpudata->min_limit_perf); des_perf = cap_perf; if (target_perf < capacity) des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); - min_perf = READ_ONCE(cpudata->lowest_perf); if (_min_perf < capacity) min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); + else + min_perf = cap_perf; - if (min_perf < lowest_nonlinear_perf) - min_perf = lowest_nonlinear_perf; + if (min_perf < min_limit_perf) + min_perf = min_limit_perf; max_perf = cpudata->max_limit_perf; if (max_perf < min_perf) From a09aa952da975c70e2fe095e82ce6fb065c0ca37 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:15 +0000 Subject: [PATCH 098/188] cpufreq/amd-pstate: Remove the redundant des_perf clamping in adjust_perf commit 932da6489669da4d61b711c44af208fa431654fa upstream. des_perf is later on clamped between min_perf and max_perf in amd_pstate_update. So, remove the redundant clamping from amd_pstate_adjust_perf. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20250205112523.201101-5-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 7fa9abb5aaaba..e08ef7751b9ba 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -704,8 +704,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, if (max_perf < min_perf) max_perf = min_perf; - des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); - amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, policy->governor->flags); cpufreq_cpu_put(policy); From 3d06f43451a658f88e6945bcc059c24940f3f87a Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:16 +0000 Subject: [PATCH 099/188] cpufreq/amd-pstate: Pass min/max_limit_perf as min/max_perf to amd_pstate_update commit e9869c836b2a460c48e2d69ae79d786303dbffda upstream. Currently, amd_pstate_update_freq passes the hardware perf limits as min/max_perf to amd_pstate_update, which eventually gets programmed into the min/max_perf fields of the CPPC_REQ register. Instead pass the effective perf limits i.e. min/max_limit_perf values to amd_pstate_update as min/max_perf. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20250205112523.201101-6-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index e08ef7751b9ba..c577836a2d999 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -616,7 +616,7 @@ static int amd_pstate_update_freq(struct cpufreq_policy *policy, { struct cpufreq_freqs freqs; struct amd_cpudata *cpudata = policy->driver_data; - unsigned long max_perf, min_perf, des_perf, cap_perf; + unsigned long des_perf, cap_perf; if (!cpudata->max_freq) return -ENODEV; @@ -625,8 +625,6 @@ static int amd_pstate_update_freq(struct cpufreq_policy *policy, amd_pstate_update_min_max_limit(policy); cap_perf = READ_ONCE(cpudata->highest_perf); - min_perf = READ_ONCE(cpudata->lowest_perf); - max_perf = cap_perf; freqs.old = policy->cur; freqs.new = target_freq; @@ -643,8 +641,9 @@ static int amd_pstate_update_freq(struct cpufreq_policy *policy, if (!fast_switch) cpufreq_freq_transition_begin(policy, &freqs); - amd_pstate_update(cpudata, min_perf, des_perf, - max_perf, fast_switch, policy->governor->flags); + amd_pstate_update(cpudata, cpudata->min_limit_perf, des_perf, + cpudata->max_limit_perf, fast_switch, + policy->governor->flags); if (!fast_switch) cpufreq_freq_transition_end(policy, &freqs, false); From c88d90c6475941152ca8f054704071628f2cc339 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:17 +0000 Subject: [PATCH 100/188] cpufreq/amd-pstate: Convert all perf values to u8 commit 555bbe67a622b297405e246d1868dbda3284bde8 upstream. All perf values are always within 0-255 range, hence convert their datatype to u8 everywhere. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20250205112523.201101-7-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-trace.h | 46 +++++++++++------------ drivers/cpufreq/amd-pstate.c | 60 +++++++++++++++--------------- drivers/cpufreq/amd-pstate.h | 18 ++++----- 3 files changed, 62 insertions(+), 62 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-trace.h b/drivers/cpufreq/amd-pstate-trace.h index 8d692415d9050..f457d4af2c62e 100644 --- a/drivers/cpufreq/amd-pstate-trace.h +++ b/drivers/cpufreq/amd-pstate-trace.h @@ -24,9 +24,9 @@ TRACE_EVENT(amd_pstate_perf, - TP_PROTO(unsigned long min_perf, - unsigned long target_perf, - unsigned long capacity, + TP_PROTO(u8 min_perf, + u8 target_perf, + u8 capacity, u64 freq, u64 mperf, u64 aperf, @@ -47,9 +47,9 @@ TRACE_EVENT(amd_pstate_perf, ), TP_STRUCT__entry( - __field(unsigned long, min_perf) - __field(unsigned long, target_perf) - __field(unsigned long, capacity) + __field(u8, min_perf) + __field(u8, target_perf) + __field(u8, capacity) __field(unsigned long long, freq) __field(unsigned long long, mperf) __field(unsigned long long, aperf) @@ -70,10 +70,10 @@ TRACE_EVENT(amd_pstate_perf, __entry->fast_switch = fast_switch; ), - TP_printk("amd_min_perf=%lu amd_des_perf=%lu amd_max_perf=%lu freq=%llu mperf=%llu aperf=%llu tsc=%llu cpu_id=%u fast_switch=%s", - (unsigned long)__entry->min_perf, - (unsigned long)__entry->target_perf, - (unsigned long)__entry->capacity, + TP_printk("amd_min_perf=%hhu amd_des_perf=%hhu amd_max_perf=%hhu freq=%llu mperf=%llu aperf=%llu tsc=%llu cpu_id=%u fast_switch=%s", + (u8)__entry->min_perf, + (u8)__entry->target_perf, + (u8)__entry->capacity, (unsigned long long)__entry->freq, (unsigned long long)__entry->mperf, (unsigned long long)__entry->aperf, @@ -86,10 +86,10 @@ TRACE_EVENT(amd_pstate_perf, TRACE_EVENT(amd_pstate_epp_perf, TP_PROTO(unsigned int cpu_id, - unsigned int highest_perf, - unsigned int epp, - unsigned int min_perf, - unsigned int max_perf, + u8 highest_perf, + u8 epp, + u8 min_perf, + u8 max_perf, bool boost ), @@ -102,10 +102,10 @@ TRACE_EVENT(amd_pstate_epp_perf, TP_STRUCT__entry( __field(unsigned int, cpu_id) - __field(unsigned int, highest_perf) - __field(unsigned int, epp) - __field(unsigned int, min_perf) - __field(unsigned int, max_perf) + __field(u8, highest_perf) + __field(u8, epp) + __field(u8, min_perf) + __field(u8, max_perf) __field(bool, boost) ), @@ -118,12 +118,12 @@ TRACE_EVENT(amd_pstate_epp_perf, __entry->boost = boost; ), - TP_printk("cpu%u: [%u<->%u]/%u, epp=%u, boost=%u", + TP_printk("cpu%u: [%hhu<->%hhu]/%hhu, epp=%hhu, boost=%u", (unsigned int)__entry->cpu_id, - (unsigned int)__entry->min_perf, - (unsigned int)__entry->max_perf, - (unsigned int)__entry->highest_perf, - (unsigned int)__entry->epp, + (u8)__entry->min_perf, + (u8)__entry->max_perf, + (u8)__entry->highest_perf, + (u8)__entry->epp, (bool)__entry->boost ) ); diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index c577836a2d999..1e79edbbb6c9c 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -187,7 +187,7 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) static DEFINE_MUTEX(amd_pstate_limits_lock); static DEFINE_MUTEX(amd_pstate_driver_lock); -static s16 msr_get_epp(struct amd_cpudata *cpudata) +static u8 msr_get_epp(struct amd_cpudata *cpudata) { u64 value; int ret; @@ -208,7 +208,7 @@ static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) return static_call(amd_pstate_get_epp)(cpudata); } -static s16 shmem_get_epp(struct amd_cpudata *cpudata) +static u8 shmem_get_epp(struct amd_cpudata *cpudata) { u64 epp; int ret; @@ -219,11 +219,11 @@ static s16 shmem_get_epp(struct amd_cpudata *cpudata) return ret; } - return (s16)(epp & 0xff); + return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp); } -static int msr_update_perf(struct amd_cpudata *cpudata, u32 min_perf, - u32 des_perf, u32 max_perf, u32 epp, bool fast_switch) +static int msr_update_perf(struct amd_cpudata *cpudata, u8 min_perf, + u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) { u64 value, prev; @@ -258,15 +258,15 @@ static int msr_update_perf(struct amd_cpudata *cpudata, u32 min_perf, DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); static inline int amd_pstate_update_perf(struct amd_cpudata *cpudata, - u32 min_perf, u32 des_perf, - u32 max_perf, u32 epp, + u8 min_perf, u8 des_perf, + u8 max_perf, u8 epp, bool fast_switch) { return static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, max_perf, epp, fast_switch); } -static int msr_set_epp(struct amd_cpudata *cpudata, u32 epp) +static int msr_set_epp(struct amd_cpudata *cpudata, u8 epp) { u64 value, prev; int ret; @@ -293,12 +293,12 @@ static int msr_set_epp(struct amd_cpudata *cpudata, u32 epp) DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); -static inline int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) +static inline int amd_pstate_set_epp(struct amd_cpudata *cpudata, u8 epp) { return static_call(amd_pstate_set_epp)(cpudata, epp); } -static int shmem_set_epp(struct amd_cpudata *cpudata, u32 epp) +static int shmem_set_epp(struct amd_cpudata *cpudata, u8 epp) { int ret; struct cppc_perf_ctrls perf_ctrls; @@ -321,7 +321,7 @@ static int amd_pstate_set_energy_pref_index(struct cpufreq_policy *policy, int pref_index) { struct amd_cpudata *cpudata = policy->driver_data; - int epp; + u8 epp; if (!pref_index) epp = cpudata->epp_default; @@ -480,8 +480,8 @@ static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) return static_call(amd_pstate_init_perf)(cpudata); } -static int shmem_update_perf(struct amd_cpudata *cpudata, u32 min_perf, - u32 des_perf, u32 max_perf, u32 epp, bool fast_switch) +static int shmem_update_perf(struct amd_cpudata *cpudata, u8 min_perf, + u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) { struct cppc_perf_ctrls perf_ctrls; @@ -532,14 +532,14 @@ static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) return true; } -static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, - u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags) +static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, + u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) { unsigned long max_freq; struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu); - u32 nominal_perf = READ_ONCE(cpudata->nominal_perf); + u8 nominal_perf = READ_ONCE(cpudata->nominal_perf); - des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); + des_perf = clamp_t(u8, des_perf, min_perf, max_perf); max_freq = READ_ONCE(cpudata->max_limit_freq); policy->cur = div_u64(des_perf * max_freq, max_perf); @@ -551,7 +551,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, /* limit the max perf when core performance boost feature is disabled */ if (!cpudata->boost_supported) - max_perf = min_t(unsigned long, nominal_perf, max_perf); + max_perf = min_t(u8, nominal_perf, max_perf); if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, @@ -592,7 +592,8 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) { - u32 max_limit_perf, min_limit_perf, max_perf, max_freq; + u8 max_limit_perf, min_limit_perf, max_perf; + u32 max_freq; struct amd_cpudata *cpudata = policy->driver_data; max_perf = READ_ONCE(cpudata->highest_perf); @@ -616,7 +617,7 @@ static int amd_pstate_update_freq(struct cpufreq_policy *policy, { struct cpufreq_freqs freqs; struct amd_cpudata *cpudata = policy->driver_data; - unsigned long des_perf, cap_perf; + u8 des_perf, cap_perf; if (!cpudata->max_freq) return -ENODEV; @@ -671,8 +672,7 @@ static void amd_pstate_adjust_perf(unsigned int cpu, unsigned long target_perf, unsigned long capacity) { - unsigned long max_perf, min_perf, des_perf, - cap_perf, min_limit_perf; + u8 max_perf, min_perf, des_perf, cap_perf, min_limit_perf; struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); struct amd_cpudata *cpudata; @@ -901,8 +901,8 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) { int ret; u32 min_freq, max_freq; - u32 highest_perf, nominal_perf, nominal_freq; - u32 lowest_nonlinear_perf, lowest_nonlinear_freq; + u8 highest_perf, nominal_perf, lowest_nonlinear_perf; + u32 nominal_freq, lowest_nonlinear_freq; struct cppc_perf_caps cppc_perf; ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); @@ -1109,7 +1109,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, char *buf) { - u32 perf; + u8 perf; struct amd_cpudata *cpudata = policy->driver_data; perf = READ_ONCE(cpudata->highest_perf); @@ -1120,7 +1120,7 @@ static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, char *buf) { - u32 perf; + u8 perf; struct amd_cpudata *cpudata = policy->driver_data; perf = READ_ONCE(cpudata->prefcore_ranking); @@ -1183,7 +1183,7 @@ static ssize_t show_energy_performance_preference( struct cpufreq_policy *policy, char *buf) { struct amd_cpudata *cpudata = policy->driver_data; - int preference; + u8 preference; switch (cpudata->epp_cached) { case AMD_CPPC_EPP_PERFORMANCE: @@ -1545,7 +1545,7 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - u32 epp; + u8 epp; amd_pstate_update_min_max_limit(policy); @@ -1594,7 +1594,7 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) static int amd_pstate_epp_reenable(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - u64 max_perf; + u8 max_perf; int ret; ret = amd_pstate_cppc_enable(true); @@ -1631,7 +1631,7 @@ static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - int min_perf; + u8 min_perf; if (cpudata->suspended) return 0; diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 9747e3be6ceee..19d405c6d805e 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -70,13 +70,13 @@ struct amd_cpudata { struct freq_qos_request req[2]; u64 cppc_req_cached; - u32 highest_perf; - u32 nominal_perf; - u32 lowest_nonlinear_perf; - u32 lowest_perf; - u32 prefcore_ranking; - u32 min_limit_perf; - u32 max_limit_perf; + u8 highest_perf; + u8 nominal_perf; + u8 lowest_nonlinear_perf; + u8 lowest_perf; + u8 prefcore_ranking; + u8 min_limit_perf; + u8 max_limit_perf; u32 min_limit_freq; u32 max_limit_freq; @@ -93,11 +93,11 @@ struct amd_cpudata { bool hw_prefcore; /* EPP feature related attributes*/ - s16 epp_cached; + u8 epp_cached; u32 policy; u64 cppc_cap1_cached; bool suspended; - s16 epp_default; + u8 epp_default; }; /* From 9158c0ac9ce3c6aafe1e1cddbdbac1eed58a9632 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:18 +0000 Subject: [PATCH 101/188] cpufreq/amd-pstate: Modularize perf<->freq conversion commit 620136ced35a9329f4d1ea90e51bee2dfd7ee5b0 upstream. Delegate the perf<->frequency conversion to helper functions to reduce code duplication, and improve readability. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20250205112523.201101-8-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 57 +++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 1e79edbbb6c9c..ae67688caa958 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -143,6 +143,20 @@ static struct quirk_entry quirk_amd_7k62 = { .lowest_freq = 550, }; +static inline u8 freq_to_perf(struct amd_cpudata *cpudata, unsigned int freq_val) +{ + u8 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * cpudata->nominal_perf, + cpudata->nominal_freq); + + return clamp_t(u8, perf_val, cpudata->lowest_perf, cpudata->highest_perf); +} + +static inline u32 perf_to_freq(struct amd_cpudata *cpudata, u8 perf_val) +{ + return DIV_ROUND_UP_ULL((u64)cpudata->nominal_freq * perf_val, + cpudata->nominal_perf); +} + static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) { /** @@ -535,14 +549,12 @@ static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) { - unsigned long max_freq; struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu); u8 nominal_perf = READ_ONCE(cpudata->nominal_perf); des_perf = clamp_t(u8, des_perf, min_perf, max_perf); - max_freq = READ_ONCE(cpudata->max_limit_freq); - policy->cur = div_u64(des_perf * max_freq, max_perf); + policy->cur = perf_to_freq(cpudata, des_perf); if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { min_perf = des_perf; @@ -592,14 +604,11 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) { - u8 max_limit_perf, min_limit_perf, max_perf; - u32 max_freq; + u8 max_limit_perf, min_limit_perf; struct amd_cpudata *cpudata = policy->driver_data; - max_perf = READ_ONCE(cpudata->highest_perf); - max_freq = READ_ONCE(cpudata->max_freq); - max_limit_perf = div_u64(policy->max * max_perf, max_freq); - min_limit_perf = div_u64(policy->min * max_perf, max_freq); + max_limit_perf = freq_to_perf(cpudata, policy->max); + min_limit_perf = freq_to_perf(cpudata, policy->min); if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) min_limit_perf = min(cpudata->nominal_perf, max_limit_perf); @@ -617,21 +626,15 @@ static int amd_pstate_update_freq(struct cpufreq_policy *policy, { struct cpufreq_freqs freqs; struct amd_cpudata *cpudata = policy->driver_data; - u8 des_perf, cap_perf; - - if (!cpudata->max_freq) - return -ENODEV; + u8 des_perf; if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) amd_pstate_update_min_max_limit(policy); - cap_perf = READ_ONCE(cpudata->highest_perf); - freqs.old = policy->cur; freqs.new = target_freq; - des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, - cpudata->max_freq); + des_perf = freq_to_perf(cpudata, target_freq); WARN_ON(fast_switch && !policy->fast_switch_enabled); /* @@ -901,7 +904,6 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) { int ret; u32 min_freq, max_freq; - u8 highest_perf, nominal_perf, lowest_nonlinear_perf; u32 nominal_freq, lowest_nonlinear_freq; struct cppc_perf_caps cppc_perf; @@ -919,16 +921,17 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) else nominal_freq = cppc_perf.nominal_freq; - highest_perf = READ_ONCE(cpudata->highest_perf); - nominal_perf = READ_ONCE(cpudata->nominal_perf); - max_freq = div_u64((u64)highest_perf * nominal_freq, nominal_perf); + min_freq *= 1000; + nominal_freq *= 1000; + + WRITE_ONCE(cpudata->nominal_freq, nominal_freq); + WRITE_ONCE(cpudata->min_freq, min_freq); + + max_freq = perf_to_freq(cpudata, cpudata->highest_perf); + lowest_nonlinear_freq = perf_to_freq(cpudata, cpudata->lowest_nonlinear_perf); - lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); - lowest_nonlinear_freq = div_u64((u64)nominal_freq * lowest_nonlinear_perf, nominal_perf); - WRITE_ONCE(cpudata->min_freq, min_freq * 1000); - WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq * 1000); - WRITE_ONCE(cpudata->nominal_freq, nominal_freq * 1000); - WRITE_ONCE(cpudata->max_freq, max_freq * 1000); + WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); + WRITE_ONCE(cpudata->max_freq, max_freq); /** * Below values need to be initialized correctly, otherwise driver will fail to load From 840f2d851a609256b077f54b3d7bd69982efd0f9 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:19 +0000 Subject: [PATCH 102/188] cpufreq/amd-pstate: Remove the unnecessary cpufreq_update_policy call commit b899434857b0f1ab460b3c126cbed82ab9b52d43 upstream. The update_limits callback is only called in two conditions. * When the preferred core rankings change. In which case, we just need to change the prefcore ranking in the cpudata struct. As there are no changes to any of the perf values, there is no need to call cpufreq_update_policy() * When the _PPC ACPI object changes, i.e. the highest allowed Pstate changes. The _PPC object is only used for a table based cpufreq driver like acpi-cpufreq, hence is irrelevant for CPPC based amd-pstate. Hence, the cpufreq_update_policy() call becomes unnecessary and can be removed. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20250205112523.201101-9-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index ae67688caa958..61eebab29257c 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -853,10 +853,6 @@ static void amd_pstate_update_limits(unsigned int cpu) sched_set_itmt_core_prio((int)cur_high, cpu); } cpufreq_cpu_put(policy); - - if (!highest_perf_changed) - cpufreq_update_policy(cpu); - } /* From dd94deb5fc46738d95a8e9bc762ee1b4f2616f67 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:21 +0000 Subject: [PATCH 103/188] cpufreq/amd-pstate: Add missing NULL ptr check in amd_pstate_update commit 426db24d4db2e4f0d6720aeb7795eafcb9e82640 upstream. Check if policy is NULL before dereferencing it in amd_pstate_update. Fixes: e8f555daacd3 ("cpufreq/amd-pstate: fix setting policy current frequency value") Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20250205112523.201101-11-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 61eebab29257c..4cb2e1b68e56a 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -552,6 +552,9 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu); u8 nominal_perf = READ_ONCE(cpudata->nominal_perf); + if (!policy) + return; + des_perf = clamp_t(u8, des_perf, min_perf, max_perf); policy->cur = perf_to_freq(cpudata, des_perf); From 5bb1008afa126470bafa05444060b2f995f7753f Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:22 +0000 Subject: [PATCH 104/188] cpufreq/amd-pstate: Use scope based cleanup for cpufreq_policy refs commit 97a705dc1a3654d8d2e466433a897be202a7f0ac upstream. There have been instances in past where refcount decrementing is missed while exiting a function. Use automatic scope based cleanup to avoid such errors. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20250205112523.201101-12-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 25 ++++++++----------------- include/linux/cpufreq.h | 3 +++ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 4cb2e1b68e56a..27ee9eab86a59 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -549,7 +549,7 @@ static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) { - struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); u8 nominal_perf = READ_ONCE(cpudata->nominal_perf); if (!policy) @@ -575,8 +575,6 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, } amd_pstate_update_perf(cpudata, min_perf, des_perf, max_perf, 0, fast_switch); - - cpufreq_cpu_put(policy); } static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) @@ -588,7 +586,8 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) * amd-pstate qos_requests. */ if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { - struct cpufreq_policy *policy = cpufreq_cpu_get(policy_data->cpu); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = + cpufreq_cpu_get(policy_data->cpu); struct amd_cpudata *cpudata; if (!policy) @@ -596,7 +595,6 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) cpudata = policy->driver_data; policy_data->min = cpudata->lowest_nonlinear_freq; - cpufreq_cpu_put(policy); } cpufreq_verify_within_cpu_limits(policy_data); @@ -679,7 +677,7 @@ static void amd_pstate_adjust_perf(unsigned int cpu, unsigned long capacity) { u8 max_perf, min_perf, des_perf, cap_perf, min_limit_perf; - struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); struct amd_cpudata *cpudata; if (!policy) @@ -711,7 +709,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, policy->governor->flags); - cpufreq_cpu_put(policy); } static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) @@ -824,28 +821,23 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) static void amd_pstate_update_limits(unsigned int cpu) { - struct cpufreq_policy *policy = NULL; + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); struct amd_cpudata *cpudata; u32 prev_high = 0, cur_high = 0; - int ret; bool highest_perf_changed = false; if (!amd_pstate_prefcore) return; - policy = cpufreq_cpu_get(cpu); if (!policy) return; - cpudata = policy->driver_data; - guard(mutex)(&amd_pstate_driver_lock); - ret = amd_get_highest_perf(cpu, &cur_high); - if (ret) { - cpufreq_cpu_put(policy); + if (amd_get_highest_perf(cpu, &cur_high)) return; - } + + cpudata = policy->driver_data; prev_high = READ_ONCE(cpudata->prefcore_ranking); highest_perf_changed = (prev_high != cur_high); @@ -855,7 +847,6 @@ static void amd_pstate_update_limits(unsigned int cpu) if (cur_high < CPPC_MAX_PERF) sched_set_itmt_core_prio((int)cur_high, cpu); } - cpufreq_cpu_put(policy); } /* diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 2a1324dd3148b..6cdc1820a7aa6 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -216,6 +216,9 @@ static inline struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) static inline void cpufreq_cpu_put(struct cpufreq_policy *policy) { } #endif +/* Scope based cleanup macro for cpufreq_policy kobject reference counting */ +DEFINE_FREE(put_cpufreq_policy, struct cpufreq_policy *, if (_T) cpufreq_cpu_put(_T)) + static inline bool policy_is_inactive(struct cpufreq_policy *policy) { return cpumask_empty(policy->cpus); From 3309fceb541c74e04c375a2090442f77b7c56784 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Wed, 5 Feb 2025 11:25:23 +0000 Subject: [PATCH 105/188] cpufreq/amd-pstate: Remove the unncecessary driver_lock in amd_pstate_update_limits commit 3e93edc58a63cf816e6dc853da8e9b0201bd0298 upstream. There is no need to take a driver wide lock while updating the highest_perf value in the percpu cpudata struct. Hence remove it. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20250205112523.201101-13-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 27ee9eab86a59..85c24329e9f22 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -832,8 +832,6 @@ static void amd_pstate_update_limits(unsigned int cpu) if (!policy) return; - guard(mutex)(&amd_pstate_driver_lock); - if (amd_get_highest_perf(cpu, &cur_high)) return; From 1369054b03a67e85e140f4fb4eb962823765d98a Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Sat, 22 Feb 2025 03:32:22 +0000 Subject: [PATCH 106/188] cpufreq/amd-pstate: Fix the clamping of perf values commit a1d1d8fb653532638cfb3ee0b7e67ebd5327a3d6 upstream. The clamping in freq_to_perf() is broken right now, as we first typecast (read wraparound) the overflowing value into a u8 and then clamp it down. So, use a u32 to store the >255 value in certain edge cases and then clamp it down into a u8. Also, use a "explicit typecast + clamp" instead of just a "clamp_t" as the latter typecasts first and then clamps between the limits, which defeats our purpose. Fixes: 620136ced35a ("cpufreq/amd-pstate: Modularize perf<->freq conversion") Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20250222033221.554976-1-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 85c24329e9f22..ffe831602bcd1 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -145,10 +145,10 @@ static struct quirk_entry quirk_amd_7k62 = { static inline u8 freq_to_perf(struct amd_cpudata *cpudata, unsigned int freq_val) { - u8 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * cpudata->nominal_perf, + u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * cpudata->nominal_perf, cpudata->nominal_freq); - return clamp_t(u8, perf_val, cpudata->lowest_perf, cpudata->highest_perf); + return (u8)clamp(perf_val, cpudata->lowest_perf, cpudata->highest_perf); } static inline u32 perf_to_freq(struct amd_cpudata *cpudata, u8 perf_val) From 7b8b4eaaeaae27d8cb781110ff86b5b02629a58d Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 17 Feb 2025 13:28:51 -0600 Subject: [PATCH 107/188] cpufreq/amd-pstate: Invalidate cppc_req_cached during suspend commit b7a41156588ad03757bf0a2f0e05d6cbcebeaa9e upstream. During resume it's possible the firmware didn't restore the CPPC request MSR but the kernel thinks the values line up. This leads to incorrect performance after resume from suspend. To fix the issue invalidate the cached value at suspend. During resume use the saved values programmed as cached limits. Reviewed-by: Gautham R. Shenoy Reviewed-by: Dhananjay Ugwekar Reported-by: Miroslav Pavleski Closes: https://bugzilla.kernel.org/show_bug.cgi?id=217931 Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index ffe831602bcd1..e3f1e57041a39 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1601,7 +1601,7 @@ static int amd_pstate_epp_reenable(struct cpufreq_policy *policy) max_perf, policy->boost_enabled); } - return amd_pstate_update_perf(cpudata, 0, 0, max_perf, cpudata->epp_cached, false); + return amd_pstate_epp_update_limit(policy); } static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) @@ -1650,6 +1650,9 @@ static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) if (cppc_state != AMD_PSTATE_ACTIVE) return 0; + /* invalidate to ensure it's rewritten during resume */ + cpudata->cppc_req_cached = 0; + /* set this flag to avoid setting core offline*/ cpudata->suspended = true; From b110e5210a8c5e5a6d1320e9a817636ee48496af Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Wed, 11 Dec 2024 14:57:57 -0600 Subject: [PATCH 108/188] cpufreq/amd-pstate: Show a warning when a CPU fails to setup commit a9ba0fd452d82ca0da170eb6291aac01075a17d5 upstream. I came across a system that MSR_AMD_CPPC_CAP1 for some CPUs isn't populated. This is an unexpected behavior that is most likely a BIOS bug. In the event it happens I'd like users to report bugs to properly root cause and get this fixed. Reviewed-by: Gautham R. Shenoy Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index e3f1e57041a39..00b5d2bf0c339 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1024,6 +1024,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) free_cpudata2: freq_qos_remove_request(&cpudata->req[0]); free_cpudata1: + pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); kfree(cpudata); return ret; } @@ -1517,6 +1518,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) return 0; free_cpudata1: + pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); kfree(cpudata); return ret; } From 09c7cecd6cbdaa256eae32618940c2cdd9bc5408 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 23 Jan 2025 16:16:01 -0600 Subject: [PATCH 109/188] cpufreq/amd-pstate: Drop min and max cached frequencies commit a9b9b4c2a4cdd00428d14914e3c18be3856aba71 upstream. Use the perf_to_freq helpers to calculate this on the fly. As the members are no longer cached add an extra check into amd_pstate_epp_update_limit() to avoid unnecessary calls in amd_pstate_update_min_max_limit(). Reviewed-by: Gautham R. Shenoy Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 14 +++++------ drivers/cpufreq/amd-pstate.c | 43 +++++++++------------------------ drivers/cpufreq/amd-pstate.h | 9 ++----- 3 files changed, 20 insertions(+), 46 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 3a0a380c3590c..445278cf40b61 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -214,14 +214,14 @@ static void amd_pstate_ut_check_freq(u32 index) break; cpudata = policy->driver_data; - if (!((cpudata->max_freq >= cpudata->nominal_freq) && + if (!((policy->cpuinfo.max_freq >= cpudata->nominal_freq) && (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) && - (cpudata->lowest_nonlinear_freq > cpudata->min_freq) && - (cpudata->min_freq > 0))) { + (cpudata->lowest_nonlinear_freq > policy->cpuinfo.min_freq) && + (policy->cpuinfo.min_freq > 0))) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n", - __func__, cpu, cpudata->max_freq, cpudata->nominal_freq, - cpudata->lowest_nonlinear_freq, cpudata->min_freq); + __func__, cpu, policy->cpuinfo.max_freq, cpudata->nominal_freq, + cpudata->lowest_nonlinear_freq, policy->cpuinfo.min_freq); goto skip_test; } @@ -233,13 +233,13 @@ static void amd_pstate_ut_check_freq(u32 index) } if (cpudata->boost_supported) { - if ((policy->max == cpudata->max_freq) || + if ((policy->max == policy->cpuinfo.max_freq) || (policy->max == cpudata->nominal_freq)) amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; else { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d policy_max=%d should be equal cpu_max=%d or cpu_nominal=%d !\n", - __func__, cpu, policy->max, cpudata->max_freq, + __func__, cpu, policy->max, policy->cpuinfo.max_freq, cpudata->nominal_freq); goto skip_test; } diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 00b5d2bf0c339..54f219e0d0c92 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -718,7 +718,7 @@ static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) int ret = 0; nominal_freq = READ_ONCE(cpudata->nominal_freq); - max_freq = READ_ONCE(cpudata->max_freq); + max_freq = perf_to_freq(cpudata, READ_ONCE(cpudata->highest_perf)); if (on) policy->cpuinfo.max_freq = max_freq; @@ -913,13 +913,10 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) nominal_freq *= 1000; WRITE_ONCE(cpudata->nominal_freq, nominal_freq); - WRITE_ONCE(cpudata->min_freq, min_freq); max_freq = perf_to_freq(cpudata, cpudata->highest_perf); lowest_nonlinear_freq = perf_to_freq(cpudata, cpudata->lowest_nonlinear_perf); - WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); - WRITE_ONCE(cpudata->max_freq, max_freq); /** * Below values need to be initialized correctly, otherwise driver will fail to load @@ -944,9 +941,9 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { - int min_freq, max_freq, ret; - struct device *dev; struct amd_cpudata *cpudata; + struct device *dev; + int ret; /* * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, @@ -977,17 +974,11 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) if (ret) goto free_cpudata1; - min_freq = READ_ONCE(cpudata->min_freq); - max_freq = READ_ONCE(cpudata->max_freq); - policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); - policy->min = min_freq; - policy->max = max_freq; - - policy->cpuinfo.min_freq = min_freq; - policy->cpuinfo.max_freq = max_freq; + policy->cpuinfo.min_freq = policy->min = perf_to_freq(cpudata, cpudata->lowest_perf); + policy->cpuinfo.max_freq = policy->max = perf_to_freq(cpudata, cpudata->highest_perf); policy->boost_supported = READ_ONCE(cpudata->boost_supported); @@ -1011,9 +1002,6 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) goto free_cpudata2; } - cpudata->max_limit_freq = max_freq; - cpudata->min_limit_freq = min_freq; - policy->driver_data = cpudata; if (!current_pstate_driver->adjust_perf) @@ -1071,14 +1059,10 @@ static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, char *buf) { - int max_freq; struct amd_cpudata *cpudata = policy->driver_data; - max_freq = READ_ONCE(cpudata->max_freq); - if (max_freq < 0) - return max_freq; - return sysfs_emit(buf, "%u\n", max_freq); + return sysfs_emit(buf, "%u\n", perf_to_freq(cpudata, READ_ONCE(cpudata->highest_perf))); } static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, @@ -1436,10 +1420,10 @@ static bool amd_pstate_acpi_pm_profile_undefined(void) static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) { - int min_freq, max_freq, ret; struct amd_cpudata *cpudata; struct device *dev; u64 value; + int ret; /* * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, @@ -1470,19 +1454,13 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (ret) goto free_cpudata1; - min_freq = READ_ONCE(cpudata->min_freq); - max_freq = READ_ONCE(cpudata->max_freq); - - policy->cpuinfo.min_freq = min_freq; - policy->cpuinfo.max_freq = max_freq; + policy->cpuinfo.min_freq = policy->min = perf_to_freq(cpudata, cpudata->lowest_perf); + policy->cpuinfo.max_freq = policy->max = perf_to_freq(cpudata, cpudata->highest_perf); /* It will be updated by governor */ policy->cur = policy->cpuinfo.min_freq; policy->driver_data = cpudata; - policy->min = policy->cpuinfo.min_freq; - policy->max = policy->cpuinfo.max_freq; - policy->boost_supported = READ_ONCE(cpudata->boost_supported); /* @@ -1540,7 +1518,8 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) struct amd_cpudata *cpudata = policy->driver_data; u8 epp; - amd_pstate_update_min_max_limit(policy); + if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) + amd_pstate_update_min_max_limit(policy); if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) epp = 0; diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 19d405c6d805e..0149933692458 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -46,8 +46,6 @@ struct amd_aperf_mperf { * @max_limit_perf: Cached value of the performance corresponding to policy->max * @min_limit_freq: Cached value of policy->min (in khz) * @max_limit_freq: Cached value of policy->max (in khz) - * @max_freq: the frequency (in khz) that mapped to highest_perf - * @min_freq: the frequency (in khz) that mapped to lowest_perf * @nominal_freq: the frequency (in khz) that mapped to nominal_perf * @lowest_nonlinear_freq: the frequency (in khz) that mapped to lowest_nonlinear_perf * @cur: Difference of Aperf/Mperf/tsc count between last and current sample @@ -77,11 +75,8 @@ struct amd_cpudata { u8 prefcore_ranking; u8 min_limit_perf; u8 max_limit_perf; - u32 min_limit_freq; - u32 max_limit_freq; - - u32 max_freq; - u32 min_freq; + u32 min_limit_freq; + u32 max_limit_freq; u32 nominal_freq; u32 lowest_nonlinear_freq; From 5d9f190b924ec004a21613ece7d27f99791d1d3a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 17 Jan 2025 18:34:38 -0600 Subject: [PATCH 110/188] cpufreq/amd-pstate: Move perf values into a union commit 009d1c29a45194212e9310ccd91a19a673908a5c upstream. By storing perf values in a union all the writes and reads can be done atomically, removing the need for some concurrency protections. While making this change, also drop the cached frequency values, using inline helpers to calculate them on demand from perf value. Reviewed-by: Gautham R. Shenoy Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 18 +-- drivers/cpufreq/amd-pstate.c | 213 ++++++++++++++++++-------------- drivers/cpufreq/amd-pstate.h | 51 +++++--- 3 files changed, 162 insertions(+), 120 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 445278cf40b61..5f6a92a816e61 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -129,6 +129,7 @@ static void amd_pstate_ut_check_perf(u32 index) struct cppc_perf_caps cppc_perf; struct cpufreq_policy *policy = NULL; struct amd_cpudata *cpudata = NULL; + union perf_cached cur_perf; for_each_possible_cpu(cpu) { policy = cpufreq_cpu_get(cpu); @@ -162,19 +163,20 @@ static void amd_pstate_ut_check_perf(u32 index) lowest_perf = AMD_CPPC_LOWEST_PERF(cap1); } - if (highest_perf != READ_ONCE(cpudata->highest_perf) && !cpudata->hw_prefcore) { + cur_perf = READ_ONCE(cpudata->perf); + if (highest_perf != cur_perf.highest_perf && !cpudata->hw_prefcore) { pr_err("%s cpu%d highest=%d %d highest perf doesn't match\n", - __func__, cpu, highest_perf, cpudata->highest_perf); + __func__, cpu, highest_perf, cur_perf.highest_perf); goto skip_test; } - if ((nominal_perf != READ_ONCE(cpudata->nominal_perf)) || - (lowest_nonlinear_perf != READ_ONCE(cpudata->lowest_nonlinear_perf)) || - (lowest_perf != READ_ONCE(cpudata->lowest_perf))) { + if (nominal_perf != cur_perf.nominal_perf || + (lowest_nonlinear_perf != cur_perf.lowest_nonlinear_perf) || + (lowest_perf != cur_perf.lowest_perf)) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d nominal=%d %d lowest_nonlinear=%d %d lowest=%d %d, they should be equal!\n", - __func__, cpu, nominal_perf, cpudata->nominal_perf, - lowest_nonlinear_perf, cpudata->lowest_nonlinear_perf, - lowest_perf, cpudata->lowest_perf); + __func__, cpu, nominal_perf, cur_perf.nominal_perf, + lowest_nonlinear_perf, cur_perf.lowest_nonlinear_perf, + lowest_perf, cur_perf.lowest_perf); goto skip_test; } diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 54f219e0d0c92..7c10a7f24705f 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -143,18 +143,17 @@ static struct quirk_entry quirk_amd_7k62 = { .lowest_freq = 550, }; -static inline u8 freq_to_perf(struct amd_cpudata *cpudata, unsigned int freq_val) +static inline u8 freq_to_perf(union perf_cached perf, u32 nominal_freq, unsigned int freq_val) { - u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * cpudata->nominal_perf, - cpudata->nominal_freq); + u32 perf_val = DIV_ROUND_UP_ULL((u64)freq_val * perf.nominal_perf, nominal_freq); - return (u8)clamp(perf_val, cpudata->lowest_perf, cpudata->highest_perf); + return (u8)clamp(perf_val, perf.lowest_perf, perf.highest_perf); } -static inline u32 perf_to_freq(struct amd_cpudata *cpudata, u8 perf_val) +static inline u32 perf_to_freq(union perf_cached perf, u32 nominal_freq, u8 perf_val) { - return DIV_ROUND_UP_ULL((u64)cpudata->nominal_freq * perf_val, - cpudata->nominal_perf); + return DIV_ROUND_UP_ULL((u64)nominal_freq * perf_val, + perf.nominal_perf); } static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) @@ -348,7 +347,9 @@ static int amd_pstate_set_energy_pref_index(struct cpufreq_policy *policy, } if (trace_amd_pstate_epp_perf_enabled()) { - trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, + union perf_cached perf = READ_ONCE(cpudata->perf); + + trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, epp, FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), @@ -426,6 +427,7 @@ static inline int amd_pstate_cppc_enable(bool enable) static int msr_init_perf(struct amd_cpudata *cpudata) { + union perf_cached perf = READ_ONCE(cpudata->perf); u64 cap1, numerator; int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, @@ -437,19 +439,21 @@ static int msr_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - WRITE_ONCE(cpudata->highest_perf, numerator); - WRITE_ONCE(cpudata->max_limit_perf, numerator); - WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); - WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); - WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); + perf.highest_perf = numerator; + perf.max_limit_perf = numerator; + perf.min_limit_perf = AMD_CPPC_LOWEST_PERF(cap1); + perf.nominal_perf = AMD_CPPC_NOMINAL_PERF(cap1); + perf.lowest_nonlinear_perf = AMD_CPPC_LOWNONLIN_PERF(cap1); + perf.lowest_perf = AMD_CPPC_LOWEST_PERF(cap1); + WRITE_ONCE(cpudata->perf, perf); WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1)); - WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1)); return 0; } static int shmem_init_perf(struct amd_cpudata *cpudata) { struct cppc_perf_caps cppc_perf; + union perf_cached perf = READ_ONCE(cpudata->perf); u64 numerator; int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); @@ -460,14 +464,14 @@ static int shmem_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; - WRITE_ONCE(cpudata->highest_perf, numerator); - WRITE_ONCE(cpudata->max_limit_perf, numerator); - WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); - WRITE_ONCE(cpudata->lowest_nonlinear_perf, - cppc_perf.lowest_nonlinear_perf); - WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); + perf.highest_perf = numerator; + perf.max_limit_perf = numerator; + perf.min_limit_perf = cppc_perf.lowest_perf; + perf.nominal_perf = cppc_perf.nominal_perf; + perf.lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; + perf.lowest_perf = cppc_perf.lowest_perf; + WRITE_ONCE(cpudata->perf, perf); WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); - WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf); if (cppc_state == AMD_PSTATE_ACTIVE) return 0; @@ -550,14 +554,14 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) { struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); - u8 nominal_perf = READ_ONCE(cpudata->nominal_perf); + union perf_cached perf = READ_ONCE(cpudata->perf); if (!policy) return; des_perf = clamp_t(u8, des_perf, min_perf, max_perf); - policy->cur = perf_to_freq(cpudata, des_perf); + policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { min_perf = des_perf; @@ -566,7 +570,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, /* limit the max perf when core performance boost feature is disabled */ if (!cpudata->boost_supported) - max_perf = min_t(u8, nominal_perf, max_perf); + max_perf = min_t(u8, perf.nominal_perf, max_perf); if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, @@ -603,39 +607,41 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) return 0; } -static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) +static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) { - u8 max_limit_perf, min_limit_perf; struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); - max_limit_perf = freq_to_perf(cpudata, policy->max); - min_limit_perf = freq_to_perf(cpudata, policy->min); + perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); + perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) - min_limit_perf = min(cpudata->nominal_perf, max_limit_perf); + perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); - WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); - WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); WRITE_ONCE(cpudata->max_limit_freq, policy->max); WRITE_ONCE(cpudata->min_limit_freq, policy->min); - - return 0; + WRITE_ONCE(cpudata->perf, perf); } static int amd_pstate_update_freq(struct cpufreq_policy *policy, unsigned int target_freq, bool fast_switch) { struct cpufreq_freqs freqs; - struct amd_cpudata *cpudata = policy->driver_data; + struct amd_cpudata *cpudata; + union perf_cached perf; u8 des_perf; + cpudata = policy->driver_data; + if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) amd_pstate_update_min_max_limit(policy); + perf = READ_ONCE(cpudata->perf); + freqs.old = policy->cur; freqs.new = target_freq; - des_perf = freq_to_perf(cpudata, target_freq); + des_perf = freq_to_perf(perf, cpudata->nominal_freq, target_freq); WARN_ON(fast_switch && !policy->fast_switch_enabled); /* @@ -646,8 +652,8 @@ static int amd_pstate_update_freq(struct cpufreq_policy *policy, if (!fast_switch) cpufreq_freq_transition_begin(policy, &freqs); - amd_pstate_update(cpudata, cpudata->min_limit_perf, des_perf, - cpudata->max_limit_perf, fast_switch, + amd_pstate_update(cpudata, perf.min_limit_perf, des_perf, + perf.max_limit_perf, fast_switch, policy->governor->flags); if (!fast_switch) @@ -676,9 +682,10 @@ static void amd_pstate_adjust_perf(unsigned int cpu, unsigned long target_perf, unsigned long capacity) { - u8 max_perf, min_perf, des_perf, cap_perf, min_limit_perf; + u8 max_perf, min_perf, des_perf, cap_perf; struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); struct amd_cpudata *cpudata; + union perf_cached perf; if (!policy) return; @@ -688,8 +695,8 @@ static void amd_pstate_adjust_perf(unsigned int cpu, if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) amd_pstate_update_min_max_limit(policy); - cap_perf = READ_ONCE(cpudata->highest_perf); - min_limit_perf = READ_ONCE(cpudata->min_limit_perf); + perf = READ_ONCE(cpudata->perf); + cap_perf = perf.highest_perf; des_perf = cap_perf; if (target_perf < capacity) @@ -700,10 +707,10 @@ static void amd_pstate_adjust_perf(unsigned int cpu, else min_perf = cap_perf; - if (min_perf < min_limit_perf) - min_perf = min_limit_perf; + if (min_perf < perf.min_limit_perf) + min_perf = perf.min_limit_perf; - max_perf = cpudata->max_limit_perf; + max_perf = perf.max_limit_perf; if (max_perf < min_perf) max_perf = min_perf; @@ -714,11 +721,12 @@ static void amd_pstate_adjust_perf(unsigned int cpu, static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) { struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); u32 nominal_freq, max_freq; int ret = 0; nominal_freq = READ_ONCE(cpudata->nominal_freq); - max_freq = perf_to_freq(cpudata, READ_ONCE(cpudata->highest_perf)); + max_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf); if (on) policy->cpuinfo.max_freq = max_freq; @@ -878,44 +886,44 @@ static u32 amd_pstate_get_transition_latency(unsigned int cpu) } /* - * amd_pstate_init_freq: Initialize the max_freq, min_freq, - * nominal_freq and lowest_nonlinear_freq for - * the @cpudata object. + * amd_pstate_init_freq: Initialize the nominal_freq and lowest_nonlinear_freq + * for the @cpudata object. * - * Requires: highest_perf, lowest_perf, nominal_perf and - * lowest_nonlinear_perf members of @cpudata to be - * initialized. + * Requires: all perf members of @cpudata to be initialized. * - * Returns 0 on success, non-zero value on failure. + * Returns 0 on success, non-zero value on failure. */ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) { - int ret; - u32 min_freq, max_freq; - u32 nominal_freq, lowest_nonlinear_freq; + u32 min_freq, max_freq, nominal_freq, lowest_nonlinear_freq; struct cppc_perf_caps cppc_perf; + union perf_cached perf; + int ret; ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); if (ret) return ret; - - if (quirks && quirks->lowest_freq) - min_freq = quirks->lowest_freq; - else - min_freq = cppc_perf.lowest_freq; + perf = READ_ONCE(cpudata->perf); if (quirks && quirks->nominal_freq) nominal_freq = quirks->nominal_freq; else nominal_freq = cppc_perf.nominal_freq; + nominal_freq *= 1000; + + if (quirks && quirks->lowest_freq) { + min_freq = quirks->lowest_freq; + perf.lowest_perf = freq_to_perf(perf, nominal_freq, min_freq); + WRITE_ONCE(cpudata->perf, perf); + } else + min_freq = cppc_perf.lowest_freq; min_freq *= 1000; - nominal_freq *= 1000; WRITE_ONCE(cpudata->nominal_freq, nominal_freq); - max_freq = perf_to_freq(cpudata, cpudata->highest_perf); - lowest_nonlinear_freq = perf_to_freq(cpudata, cpudata->lowest_nonlinear_perf); + max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); + lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); /** @@ -942,6 +950,7 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) static int amd_pstate_cpu_init(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata; + union perf_cached perf; struct device *dev; int ret; @@ -977,8 +986,14 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); - policy->cpuinfo.min_freq = policy->min = perf_to_freq(cpudata, cpudata->lowest_perf); - policy->cpuinfo.max_freq = policy->max = perf_to_freq(cpudata, cpudata->highest_perf); + perf = READ_ONCE(cpudata->perf); + + policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, + cpudata->nominal_freq, + perf.lowest_perf); + policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, + cpudata->nominal_freq, + perf.highest_perf); policy->boost_supported = READ_ONCE(cpudata->boost_supported); @@ -1059,23 +1074,27 @@ static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, char *buf) { - struct amd_cpudata *cpudata = policy->driver_data; + struct amd_cpudata *cpudata; + union perf_cached perf; + cpudata = policy->driver_data; + perf = READ_ONCE(cpudata->perf); - return sysfs_emit(buf, "%u\n", perf_to_freq(cpudata, READ_ONCE(cpudata->highest_perf))); + return sysfs_emit(buf, "%u\n", + perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf)); } static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, char *buf) { - int freq; - struct amd_cpudata *cpudata = policy->driver_data; + struct amd_cpudata *cpudata; + union perf_cached perf; - freq = READ_ONCE(cpudata->lowest_nonlinear_freq); - if (freq < 0) - return freq; + cpudata = policy->driver_data; + perf = READ_ONCE(cpudata->perf); - return sysfs_emit(buf, "%u\n", freq); + return sysfs_emit(buf, "%u\n", + perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_nonlinear_perf)); } /* @@ -1085,12 +1104,11 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, char *buf) { - u8 perf; - struct amd_cpudata *cpudata = policy->driver_data; + struct amd_cpudata *cpudata; - perf = READ_ONCE(cpudata->highest_perf); + cpudata = policy->driver_data; - return sysfs_emit(buf, "%u\n", perf); + return sysfs_emit(buf, "%u\n", cpudata->perf.highest_perf); } static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, @@ -1421,6 +1439,7 @@ static bool amd_pstate_acpi_pm_profile_undefined(void) static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata; + union perf_cached perf; struct device *dev; u64 value; int ret; @@ -1454,8 +1473,15 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (ret) goto free_cpudata1; - policy->cpuinfo.min_freq = policy->min = perf_to_freq(cpudata, cpudata->lowest_perf); - policy->cpuinfo.max_freq = policy->max = perf_to_freq(cpudata, cpudata->highest_perf); + perf = READ_ONCE(cpudata->perf); + + policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, + cpudata->nominal_freq, + perf.lowest_perf); + policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, + cpudata->nominal_freq, + perf.highest_perf); + /* It will be updated by governor */ policy->cur = policy->cpuinfo.min_freq; @@ -1516,6 +1542,7 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf; u8 epp; if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) @@ -1526,15 +1553,16 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) else epp = READ_ONCE(cpudata->epp_cached); + perf = READ_ONCE(cpudata->perf); if (trace_amd_pstate_epp_perf_enabled()) { - trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, epp, - cpudata->min_limit_perf, - cpudata->max_limit_perf, + trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, epp, + perf.min_limit_perf, + perf.max_limit_perf, policy->boost_enabled); } - return amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, - cpudata->max_limit_perf, epp, false); + return amd_pstate_update_perf(cpudata, perf.min_limit_perf, 0U, + perf.max_limit_perf, epp, false); } static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) @@ -1566,20 +1594,18 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) static int amd_pstate_epp_reenable(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - u8 max_perf; + union perf_cached perf = READ_ONCE(cpudata->perf); int ret; ret = amd_pstate_cppc_enable(true); if (ret) pr_err("failed to enable amd pstate during resume, return %d\n", ret); - max_perf = READ_ONCE(cpudata->highest_perf); - if (trace_amd_pstate_epp_perf_enabled()) { - trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, + trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, cpudata->epp_cached, FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), - max_perf, policy->boost_enabled); + perf.highest_perf, policy->boost_enabled); } return amd_pstate_epp_update_limit(policy); @@ -1603,22 +1629,21 @@ static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - u8 min_perf; + union perf_cached perf = READ_ONCE(cpudata->perf); if (cpudata->suspended) return 0; - min_perf = READ_ONCE(cpudata->lowest_perf); - guard(mutex)(&amd_pstate_limits_lock); if (trace_amd_pstate_epp_perf_enabled()) { - trace_amd_pstate_epp_perf(cpudata->cpu, cpudata->highest_perf, + trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, AMD_CPPC_EPP_BALANCE_POWERSAVE, - min_perf, min_perf, policy->boost_enabled); + perf.lowest_perf, perf.lowest_perf, + policy->boost_enabled); } - return amd_pstate_update_perf(cpudata, min_perf, 0, min_perf, + return amd_pstate_update_perf(cpudata, perf.lowest_perf, 0, perf.lowest_perf, AMD_CPPC_EPP_BALANCE_POWERSAVE, false); } diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 0149933692458..83532a0079a81 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -13,6 +13,36 @@ /********************************************************************* * AMD P-state INTERFACE * *********************************************************************/ + +/** + * union perf_cached - A union to cache performance-related data. + * @highest_perf: the maximum performance an individual processor may reach, + * assuming ideal conditions + * For platforms that support the preferred core feature, the highest_perf value maybe + * configured to any value in the range 166-255 by the firmware (because the preferred + * core ranking is encoded in the highest_perf value). To maintain consistency across + * all platforms, we split the highest_perf and preferred core ranking values into + * cpudata->perf.highest_perf and cpudata->prefcore_ranking. + * @nominal_perf: the maximum sustained performance level of the processor, + * assuming ideal operating conditions + * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power + * savings are achieved + * @lowest_perf: the absolute lowest performance level of the processor + * @min_limit_perf: Cached value of the performance corresponding to policy->min + * @max_limit_perf: Cached value of the performance corresponding to policy->max + */ +union perf_cached { + struct { + u8 highest_perf; + u8 nominal_perf; + u8 lowest_nonlinear_perf; + u8 lowest_perf; + u8 min_limit_perf; + u8 max_limit_perf; + }; + u64 val; +}; + /** * struct amd_aperf_mperf * @aperf: actual performance frequency clock count @@ -30,20 +60,9 @@ struct amd_aperf_mperf { * @cpu: CPU number * @req: constraint request to apply * @cppc_req_cached: cached performance request hints - * @highest_perf: the maximum performance an individual processor may reach, - * assuming ideal conditions - * For platforms that do not support the preferred core feature, the - * highest_pef may be configured with 166 or 255, to avoid max frequency - * calculated wrongly. we take the fixed value as the highest_perf. - * @nominal_perf: the maximum sustained performance level of the processor, - * assuming ideal operating conditions - * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power - * savings are achieved - * @lowest_perf: the absolute lowest performance level of the processor + * @perf: cached performance-related data * @prefcore_ranking: the preferred core ranking, the higher value indicates a higher * priority. - * @min_limit_perf: Cached value of the performance corresponding to policy->min - * @max_limit_perf: Cached value of the performance corresponding to policy->max * @min_limit_freq: Cached value of policy->min (in khz) * @max_limit_freq: Cached value of policy->max (in khz) * @nominal_freq: the frequency (in khz) that mapped to nominal_perf @@ -68,13 +87,9 @@ struct amd_cpudata { struct freq_qos_request req[2]; u64 cppc_req_cached; - u8 highest_perf; - u8 nominal_perf; - u8 lowest_nonlinear_perf; - u8 lowest_perf; + union perf_cached perf; + u8 prefcore_ranking; - u8 min_limit_perf; - u8 max_limit_perf; u32 min_limit_freq; u32 max_limit_freq; u32 nominal_freq; From 57e80a72054ed2a60a8bde94b5f59779771b85ab Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 6 Dec 2024 23:20:04 -0600 Subject: [PATCH 111/188] cpufreq/amd-pstate: Overhaul locking commit 6f0b13f16f7a214996a8a125080a6a99bda5d1f7 upstream. amd_pstate_cpu_boost_update() and refresh_frequency_limits() both update the policy state and have nothing to do with the amd-pstate driver itself. A global "limits" lock doesn't make sense because each CPU can have policies changed independently. Each time a CPU changes values they will atomically be written to the per-CPU perf member. Drop per CPU locking cases. The remaining "global" driver lock is used to ensure that only one entity can change driver modes at a given time. [Backport Changes] 1.In the drivers/cpufreq/amd-pstate.c file, within the amd_pstate_set_boost() function, the removal of guard(mutex)(&amd_pstate_driver_lock); was skipped, as this line had already been removed and is not present in the Anolis kernel. Reviewed-by: Gautham R. Shenoy Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 7c10a7f24705f..a472ae16621a2 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -197,7 +197,6 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) return -EINVAL; } -static DEFINE_MUTEX(amd_pstate_limits_lock); static DEFINE_MUTEX(amd_pstate_driver_lock); static u8 msr_get_epp(struct amd_cpudata *cpudata) @@ -1166,8 +1165,6 @@ static ssize_t store_energy_performance_preference( if (ret < 0) return -EINVAL; - guard(mutex)(&amd_pstate_limits_lock); - ret = amd_pstate_set_energy_pref_index(policy, ret); return ret ? ret : count; @@ -1340,8 +1337,10 @@ int amd_pstate_update_status(const char *buf, size_t size) if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) return -EINVAL; - if (mode_state_machine[cppc_state][mode_idx]) + if (mode_state_machine[cppc_state][mode_idx]) { + guard(mutex)(&amd_pstate_driver_lock); return mode_state_machine[cppc_state][mode_idx](mode_idx); + } return 0; } @@ -1362,7 +1361,6 @@ static ssize_t status_store(struct device *a, struct device_attribute *b, char *p = memchr(buf, '\n', count); int ret; - guard(mutex)(&amd_pstate_driver_lock); ret = amd_pstate_update_status(buf, p ? p - buf : count); return ret < 0 ? ret : count; @@ -1634,8 +1632,6 @@ static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) if (cpudata->suspended) return 0; - guard(mutex)(&amd_pstate_limits_lock); - if (trace_amd_pstate_epp_perf_enabled()) { trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, AMD_CPPC_EPP_BALANCE_POWERSAVE, @@ -1675,8 +1671,6 @@ static int amd_pstate_epp_resume(struct cpufreq_policy *policy) struct amd_cpudata *cpudata = policy->driver_data; if (cpudata->suspended) { - guard(mutex)(&amd_pstate_limits_lock); - /* enable amd pstate from suspend state*/ amd_pstate_epp_reenable(policy); From c5e552af3c9806361796ed7e1682ec32cd2ebd63 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 7 Dec 2024 00:28:41 -0600 Subject: [PATCH 112/188] cpufreq/amd-pstate: Drop `cppc_cap1_cached` commit f458cf79d73b144d900e0c274b9eb8a2f3641a0e upstream. The `cppc_cap1_cached` variable isn't used at all, there is no need to read it at initialization for each CPU. Reviewed-by: Gautham R. Shenoy Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 5 ----- drivers/cpufreq/amd-pstate.h | 2 -- 2 files changed, 7 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index a472ae16621a2..f24a0aabea4fb 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1505,11 +1505,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (ret) return ret; WRITE_ONCE(cpudata->cppc_req_cached, value); - - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); - if (ret) - return ret; - WRITE_ONCE(cpudata->cppc_cap1_cached, value); } ret = amd_pstate_set_epp(cpudata, cpudata->epp_default); if (ret) diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 83532a0079a81..1557e1afea79c 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -76,7 +76,6 @@ struct amd_aperf_mperf { * AMD P-State driver supports preferred core featue. * @epp_cached: Cached CPPC energy-performance preference value * @policy: Cpufreq policy value - * @cppc_cap1_cached Cached MSR_AMD_CPPC_CAP1 register value * * The amd_cpudata is key private data for each CPU thread in AMD P-State, and * represents all the attributes and goals that AMD P-State requests at runtime. @@ -105,7 +104,6 @@ struct amd_cpudata { /* EPP feature related attributes*/ u8 epp_cached; u32 policy; - u64 cppc_cap1_cached; bool suspended; u8 epp_default; }; From 44cfb92c52c113f14ec95bfc4539d0a9a1e34e8f Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 7 Dec 2024 23:42:48 -0600 Subject: [PATCH 113/188] cpufreq/amd-pstate-ut: Use _free macro to free put policy commit 93984d3cea8ab5f2631ec5ba491ce4e47218ab7b upstream. Using a scoped cleanup macro simplifies cleanup code. Reviewed-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 5f6a92a816e61..e02672e67380a 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -26,6 +26,7 @@ #include #include #include +#include #include @@ -127,11 +128,12 @@ static void amd_pstate_ut_check_perf(u32 index) u32 highest_perf = 0, nominal_perf = 0, lowest_nonlinear_perf = 0, lowest_perf = 0; u64 cap1 = 0; struct cppc_perf_caps cppc_perf; - struct cpufreq_policy *policy = NULL; struct amd_cpudata *cpudata = NULL; union perf_cached cur_perf; for_each_possible_cpu(cpu) { + struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; + policy = cpufreq_cpu_get(cpu); if (!policy) break; @@ -142,7 +144,7 @@ static void amd_pstate_ut_check_perf(u32 index) if (ret) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cppc_get_perf_caps ret=%d error!\n", __func__, ret); - goto skip_test; + return; } highest_perf = cppc_perf.highest_perf; @@ -154,7 +156,7 @@ static void amd_pstate_ut_check_perf(u32 index) if (ret) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s read CPPC_CAP1 ret=%d error!\n", __func__, ret); - goto skip_test; + return; } highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); @@ -167,7 +169,7 @@ static void amd_pstate_ut_check_perf(u32 index) if (highest_perf != cur_perf.highest_perf && !cpudata->hw_prefcore) { pr_err("%s cpu%d highest=%d %d highest perf doesn't match\n", __func__, cpu, highest_perf, cur_perf.highest_perf); - goto skip_test; + return; } if (nominal_perf != cur_perf.nominal_perf || (lowest_nonlinear_perf != cur_perf.lowest_nonlinear_perf) || @@ -177,7 +179,7 @@ static void amd_pstate_ut_check_perf(u32 index) __func__, cpu, nominal_perf, cur_perf.nominal_perf, lowest_nonlinear_perf, cur_perf.lowest_nonlinear_perf, lowest_perf, cur_perf.lowest_perf); - goto skip_test; + return; } if (!((highest_perf >= nominal_perf) && @@ -188,15 +190,11 @@ static void amd_pstate_ut_check_perf(u32 index) pr_err("%s cpu%d highest=%d >= nominal=%d > lowest_nonlinear=%d > lowest=%d > 0, the formula is incorrect!\n", __func__, cpu, highest_perf, nominal_perf, lowest_nonlinear_perf, lowest_perf); - goto skip_test; + return; } - cpufreq_cpu_put(policy); } amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; - return; -skip_test: - cpufreq_cpu_put(policy); } /* @@ -207,10 +205,11 @@ static void amd_pstate_ut_check_perf(u32 index) static void amd_pstate_ut_check_freq(u32 index) { int cpu = 0; - struct cpufreq_policy *policy = NULL; struct amd_cpudata *cpudata = NULL; for_each_possible_cpu(cpu) { + struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; + policy = cpufreq_cpu_get(cpu); if (!policy) break; @@ -224,14 +223,14 @@ static void amd_pstate_ut_check_freq(u32 index) pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n", __func__, cpu, policy->cpuinfo.max_freq, cpudata->nominal_freq, cpudata->lowest_nonlinear_freq, policy->cpuinfo.min_freq); - goto skip_test; + return; } if (cpudata->lowest_nonlinear_freq != policy->min) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d cpudata_lowest_nonlinear_freq=%d policy_min=%d, they should be equal!\n", __func__, cpu, cpudata->lowest_nonlinear_freq, policy->min); - goto skip_test; + return; } if (cpudata->boost_supported) { @@ -243,20 +242,16 @@ static void amd_pstate_ut_check_freq(u32 index) pr_err("%s cpu%d policy_max=%d should be equal cpu_max=%d or cpu_nominal=%d !\n", __func__, cpu, policy->max, policy->cpuinfo.max_freq, cpudata->nominal_freq); - goto skip_test; + return; } } else { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d must support boost!\n", __func__, cpu); - goto skip_test; + return; } - cpufreq_cpu_put(policy); } amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; - return; -skip_test: - cpufreq_cpu_put(policy); } static int amd_pstate_set_mode(enum amd_pstate_mode mode) From c8fa6b3b1436db8f1976b8baa149e61c60b1c7b1 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 14 Feb 2025 12:59:40 -0600 Subject: [PATCH 114/188] cpufreq/amd-pstate-ut: Allow lowest nonlinear and lowest to be the same commit 66030cc1c533e26bb4dbe8a356090f24dd734801 upstream. Several Ryzen AI processors support the exact same value for lowest nonlinear perf and lowest perf. Loosen up the unit tests to allow this scenario. Reviewed-by: Gautham R. Shenoy Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index e02672e67380a..b3693a4c28d26 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -184,7 +184,7 @@ static void amd_pstate_ut_check_perf(u32 index) if (!((highest_perf >= nominal_perf) && (nominal_perf > lowest_nonlinear_perf) && - (lowest_nonlinear_perf > lowest_perf) && + (lowest_nonlinear_perf >= lowest_perf) && (lowest_perf > 0))) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d highest=%d >= nominal=%d > lowest_nonlinear=%d > lowest=%d > 0, the formula is incorrect!\n", @@ -217,7 +217,7 @@ static void amd_pstate_ut_check_freq(u32 index) if (!((policy->cpuinfo.max_freq >= cpudata->nominal_freq) && (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) && - (cpudata->lowest_nonlinear_freq > policy->cpuinfo.min_freq) && + (cpudata->lowest_nonlinear_freq >= policy->cpuinfo.min_freq) && (policy->cpuinfo.min_freq > 0))) { amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n", From e3a3ce230ceac70957c90329dd4f3adbfc5ef5ab Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 14 Feb 2025 16:31:25 -0600 Subject: [PATCH 115/188] cpufreq/amd-pstate-ut: Drop SUCCESS and FAIL enums commit a7875346c6891063b83cd59dc3bd23113e4debf5 upstream. Enums are effectively used as a boolean and don't show the return value of the failing call. Instead of using enums switch to returning the actual return code from the unit test. Reviewed-by: Gautham R. Shenoy Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 143 ++++++++++++-------------------- 1 file changed, 55 insertions(+), 88 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index b3693a4c28d26..cd9a472e8dc3c 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -32,30 +32,20 @@ #include "amd-pstate.h" -/* - * Abbreviations: - * amd_pstate_ut: used as a shortform for AMD P-State unit test. - * It helps to keep variable names smaller, simpler - */ -enum amd_pstate_ut_result { - AMD_PSTATE_UT_RESULT_PASS, - AMD_PSTATE_UT_RESULT_FAIL, -}; struct amd_pstate_ut_struct { const char *name; - void (*func)(u32 index); - enum amd_pstate_ut_result result; + int (*func)(u32 index); }; /* * Kernel module for testing the AMD P-State unit test */ -static void amd_pstate_ut_acpi_cpc_valid(u32 index); -static void amd_pstate_ut_check_enabled(u32 index); -static void amd_pstate_ut_check_perf(u32 index); -static void amd_pstate_ut_check_freq(u32 index); -static void amd_pstate_ut_check_driver(u32 index); +static int amd_pstate_ut_acpi_cpc_valid(u32 index); +static int amd_pstate_ut_check_enabled(u32 index); +static int amd_pstate_ut_check_perf(u32 index); +static int amd_pstate_ut_check_freq(u32 index); +static int amd_pstate_ut_check_driver(u32 index); static struct amd_pstate_ut_struct amd_pstate_ut_cases[] = { {"amd_pstate_ut_acpi_cpc_valid", amd_pstate_ut_acpi_cpc_valid }, @@ -78,51 +68,46 @@ static bool get_shared_mem(void) /* * check the _CPC object is present in SBIOS. */ -static void amd_pstate_ut_acpi_cpc_valid(u32 index) +static int amd_pstate_ut_acpi_cpc_valid(u32 index) { - if (acpi_cpc_valid()) - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; - else { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + if (!acpi_cpc_valid()) { pr_err("%s the _CPC object is not present in SBIOS!\n", __func__); + return -EINVAL; } + + return 0; } -static void amd_pstate_ut_pstate_enable(u32 index) +/* + * check if amd pstate is enabled + */ +static int amd_pstate_ut_check_enabled(u32 index) { - int ret = 0; u64 cppc_enable = 0; + int ret; + + if (get_shared_mem()) + return 0; ret = rdmsrl_safe(MSR_AMD_CPPC_ENABLE, &cppc_enable); if (ret) { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s rdmsrl_safe MSR_AMD_CPPC_ENABLE ret=%d error!\n", __func__, ret); - return; + return ret; } - if (cppc_enable) - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; - else { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + + if (!cppc_enable) { pr_err("%s amd pstate must be enabled!\n", __func__); + return -EINVAL; } -} -/* - * check if amd pstate is enabled - */ -static void amd_pstate_ut_check_enabled(u32 index) -{ - if (get_shared_mem()) - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; - else - amd_pstate_ut_pstate_enable(index); + return 0; } /* * check if performance values are reasonable. * highest_perf >= nominal_perf > lowest_nonlinear_perf > lowest_perf > 0 */ -static void amd_pstate_ut_check_perf(u32 index) +static int amd_pstate_ut_check_perf(u32 index) { int cpu = 0, ret = 0; u32 highest_perf = 0, nominal_perf = 0, lowest_nonlinear_perf = 0, lowest_perf = 0; @@ -142,9 +127,8 @@ static void amd_pstate_ut_check_perf(u32 index) if (get_shared_mem()) { ret = cppc_get_perf_caps(cpu, &cppc_perf); if (ret) { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cppc_get_perf_caps ret=%d error!\n", __func__, ret); - return; + return ret; } highest_perf = cppc_perf.highest_perf; @@ -154,9 +138,8 @@ static void amd_pstate_ut_check_perf(u32 index) } else { ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); if (ret) { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s read CPPC_CAP1 ret=%d error!\n", __func__, ret); - return; + return ret; } highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); @@ -169,32 +152,30 @@ static void amd_pstate_ut_check_perf(u32 index) if (highest_perf != cur_perf.highest_perf && !cpudata->hw_prefcore) { pr_err("%s cpu%d highest=%d %d highest perf doesn't match\n", __func__, cpu, highest_perf, cur_perf.highest_perf); - return; + return -EINVAL; } if (nominal_perf != cur_perf.nominal_perf || (lowest_nonlinear_perf != cur_perf.lowest_nonlinear_perf) || (lowest_perf != cur_perf.lowest_perf)) { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d nominal=%d %d lowest_nonlinear=%d %d lowest=%d %d, they should be equal!\n", __func__, cpu, nominal_perf, cur_perf.nominal_perf, lowest_nonlinear_perf, cur_perf.lowest_nonlinear_perf, lowest_perf, cur_perf.lowest_perf); - return; + return -EINVAL; } if (!((highest_perf >= nominal_perf) && (nominal_perf > lowest_nonlinear_perf) && (lowest_nonlinear_perf >= lowest_perf) && (lowest_perf > 0))) { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d highest=%d >= nominal=%d > lowest_nonlinear=%d > lowest=%d > 0, the formula is incorrect!\n", __func__, cpu, highest_perf, nominal_perf, lowest_nonlinear_perf, lowest_perf); - return; + return -EINVAL; } } - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; + return 0; } /* @@ -202,7 +183,7 @@ static void amd_pstate_ut_check_perf(u32 index) * max_freq >= nominal_freq > lowest_nonlinear_freq > min_freq > 0 * check max freq when set support boost mode. */ -static void amd_pstate_ut_check_freq(u32 index) +static int amd_pstate_ut_check_freq(u32 index) { int cpu = 0; struct amd_cpudata *cpudata = NULL; @@ -219,39 +200,33 @@ static void amd_pstate_ut_check_freq(u32 index) (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) && (cpudata->lowest_nonlinear_freq >= policy->cpuinfo.min_freq) && (policy->cpuinfo.min_freq > 0))) { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n", __func__, cpu, policy->cpuinfo.max_freq, cpudata->nominal_freq, cpudata->lowest_nonlinear_freq, policy->cpuinfo.min_freq); - return; + return -EINVAL; } if (cpudata->lowest_nonlinear_freq != policy->min) { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d cpudata_lowest_nonlinear_freq=%d policy_min=%d, they should be equal!\n", __func__, cpu, cpudata->lowest_nonlinear_freq, policy->min); - return; + return -EINVAL; } if (cpudata->boost_supported) { - if ((policy->max == policy->cpuinfo.max_freq) || - (policy->max == cpudata->nominal_freq)) - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; - else { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + if ((policy->max != policy->cpuinfo.max_freq) && + (policy->max != cpudata->nominal_freq)) { pr_err("%s cpu%d policy_max=%d should be equal cpu_max=%d or cpu_nominal=%d !\n", __func__, cpu, policy->max, policy->cpuinfo.max_freq, cpudata->nominal_freq); - return; + return -EINVAL; } } else { - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; pr_err("%s cpu%d must support boost!\n", __func__, cpu); - return; + return -EINVAL; } } - amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; + return 0; } static int amd_pstate_set_mode(enum amd_pstate_mode mode) @@ -263,32 +238,28 @@ static int amd_pstate_set_mode(enum amd_pstate_mode mode) return amd_pstate_update_status(mode_str, strlen(mode_str)); } -static void amd_pstate_ut_check_driver(u32 index) +static int amd_pstate_ut_check_driver(u32 index) { enum amd_pstate_mode mode1, mode2 = AMD_PSTATE_DISABLE; - int ret; for (mode1 = AMD_PSTATE_DISABLE; mode1 < AMD_PSTATE_MAX; mode1++) { - ret = amd_pstate_set_mode(mode1); + int ret = amd_pstate_set_mode(mode1); if (ret) - goto out; + return ret; for (mode2 = AMD_PSTATE_DISABLE; mode2 < AMD_PSTATE_MAX; mode2++) { if (mode1 == mode2) continue; ret = amd_pstate_set_mode(mode2); - if (ret) - goto out; + if (ret) { + pr_err("%s: failed to update status for %s->%s\n", __func__, + amd_pstate_get_mode_string(mode1), + amd_pstate_get_mode_string(mode2)); + return ret; + } } } -out: - if (ret) - pr_warn("%s: failed to update status for %s->%s: %d\n", __func__, - amd_pstate_get_mode_string(mode1), - amd_pstate_get_mode_string(mode2), ret); - - amd_pstate_ut_cases[index].result = ret ? - AMD_PSTATE_UT_RESULT_FAIL : - AMD_PSTATE_UT_RESULT_PASS; + + return 0; } static int __init amd_pstate_ut_init(void) @@ -296,16 +267,12 @@ static int __init amd_pstate_ut_init(void) u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases); for (i = 0; i < arr_size; i++) { - amd_pstate_ut_cases[i].func(i); - switch (amd_pstate_ut_cases[i].result) { - case AMD_PSTATE_UT_RESULT_PASS: + int ret = amd_pstate_ut_cases[i].func(i); + + if (ret) + pr_err("%-4d %-20s\t fail: %d!\n", i+1, amd_pstate_ut_cases[i].name, ret); + else pr_info("%-4d %-20s\t success!\n", i+1, amd_pstate_ut_cases[i].name); - break; - case AMD_PSTATE_UT_RESULT_FAIL: - default: - pr_info("%-4d %-20s\t fail!\n", i+1, amd_pstate_ut_cases[i].name); - break; - } } return 0; From 2af7a47a0c8b265909c0ea1e368d475af1f65640 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 14 Feb 2025 16:33:12 -0600 Subject: [PATCH 116/188] cpufreq/amd-pstate-ut: Run on all of the correct CPUs commit 2aac38ac06cb751bd3e672a4fb1eb3073f1866ab upstream. If a CPU is missing a policy or one has been offlined then the unit test is skipped for the rest of the CPUs on the system. Instead; iterate online CPUs and skip any missing policies to allow continuing to test the rest of them. Reviewed-by: Gautham R. Shenoy Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index cd9a472e8dc3c..2ab3017d7a0bb 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -116,12 +116,12 @@ static int amd_pstate_ut_check_perf(u32 index) struct amd_cpudata *cpudata = NULL; union perf_cached cur_perf; - for_each_possible_cpu(cpu) { + for_each_online_cpu(cpu) { struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; policy = cpufreq_cpu_get(cpu); if (!policy) - break; + continue; cpudata = policy->driver_data; if (get_shared_mem()) { @@ -188,12 +188,12 @@ static int amd_pstate_ut_check_freq(u32 index) int cpu = 0; struct amd_cpudata *cpudata = NULL; - for_each_possible_cpu(cpu) { + for_each_online_cpu(cpu) { struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; policy = cpufreq_cpu_get(cpu); if (!policy) - break; + continue; cpudata = policy->driver_data; if (!((policy->cpuinfo.max_freq >= cpudata->nominal_freq) && From 0c2e8c8b4ebe6249314f08200d23de2471eec7ef Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 14 Feb 2025 16:34:23 -0600 Subject: [PATCH 117/188] cpufreq/amd-pstate-ut: Adjust variable scope commit c630458c7a4b990742905e312af20a7c3260ca14 upstream. In amd_pstate_ut_check_freq() and amd_pstate_ut_check_perf() the cpudata variable is only needed in the scope of the for loop. Move it there. Reviewed-by: Gautham R. Shenoy Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 2ab3017d7a0bb..edc1475989e3d 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -113,11 +113,11 @@ static int amd_pstate_ut_check_perf(u32 index) u32 highest_perf = 0, nominal_perf = 0, lowest_nonlinear_perf = 0, lowest_perf = 0; u64 cap1 = 0; struct cppc_perf_caps cppc_perf; - struct amd_cpudata *cpudata = NULL; union perf_cached cur_perf; for_each_online_cpu(cpu) { struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; + struct amd_cpudata *cpudata; policy = cpufreq_cpu_get(cpu); if (!policy) @@ -186,10 +186,10 @@ static int amd_pstate_ut_check_perf(u32 index) static int amd_pstate_ut_check_freq(u32 index) { int cpu = 0; - struct amd_cpudata *cpudata = NULL; for_each_online_cpu(cpu) { struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; + struct amd_cpudata *cpudata; policy = cpufreq_cpu_get(cpu); if (!policy) From de95f6c1b7825c7c77b9b43c5be49b059ce66d19 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sat, 7 Dec 2024 22:38:06 -0600 Subject: [PATCH 118/188] cpufreq/amd-pstate: Replace all AMD_CPPC_* macros with masks commit b4cc466b973590114b4bf49025d362ad9618a23e upstream. Bitfield masks are easier to follow and less error prone. Reviewed-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- arch/x86/include/asm/msr-index.h | 20 +++++++++++--------- arch/x86/kernel/acpi/cppc.c | 4 +++- drivers/cpufreq/amd-pstate-ut.c | 9 +++++---- drivers/cpufreq/amd-pstate.c | 16 ++++++---------- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 5f17c9354e92f..6634830fa77a7 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -706,15 +706,17 @@ #define MSR_AMD_CPPC_REQ 0xc00102b3 #define MSR_AMD_CPPC_STATUS 0xc00102b4 -#define AMD_CPPC_LOWEST_PERF(x) (((x) >> 0) & 0xff) -#define AMD_CPPC_LOWNONLIN_PERF(x) (((x) >> 8) & 0xff) -#define AMD_CPPC_NOMINAL_PERF(x) (((x) >> 16) & 0xff) -#define AMD_CPPC_HIGHEST_PERF(x) (((x) >> 24) & 0xff) - -#define AMD_CPPC_MAX_PERF(x) (((x) & 0xff) << 0) -#define AMD_CPPC_MIN_PERF(x) (((x) & 0xff) << 8) -#define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16) -#define AMD_CPPC_ENERGY_PERF_PREF(x) (((x) & 0xff) << 24) +/* Masks for use with MSR_AMD_CPPC_CAP1 */ +#define AMD_CPPC_LOWEST_PERF_MASK GENMASK(7, 0) +#define AMD_CPPC_LOWNONLIN_PERF_MASK GENMASK(15, 8) +#define AMD_CPPC_NOMINAL_PERF_MASK GENMASK(23, 16) +#define AMD_CPPC_HIGHEST_PERF_MASK GENMASK(31, 24) + +/* Masks for use with MSR_AMD_CPPC_REQ */ +#define AMD_CPPC_MAX_PERF_MASK GENMASK(7, 0) +#define AMD_CPPC_MIN_PERF_MASK GENMASK(15, 8) +#define AMD_CPPC_DES_PERF_MASK GENMASK(23, 16) +#define AMD_CPPC_EPP_PERF_MASK GENMASK(31, 24) /* AMD Performance Counter Global Status and Control MSRs */ #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300 diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index ed0290a8f4de9..7a6615f157a8d 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -4,6 +4,8 @@ * Copyright (c) 2016, Intel Corporation. */ +#include + #include #include #include @@ -149,7 +151,7 @@ int amd_get_highest_perf(unsigned int cpu, u32 *highest_perf) if (ret) goto out; - val = AMD_CPPC_HIGHEST_PERF(val); + val = FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, val); } else { ret = cppc_get_highest_perf(cpu, &val); if (ret) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index edc1475989e3d..e671bc7d15508 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -22,6 +22,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -142,10 +143,10 @@ static int amd_pstate_ut_check_perf(u32 index) return ret; } - highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); - nominal_perf = AMD_CPPC_NOMINAL_PERF(cap1); - lowest_nonlinear_perf = AMD_CPPC_LOWNONLIN_PERF(cap1); - lowest_perf = AMD_CPPC_LOWEST_PERF(cap1); + highest_perf = FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1); + nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); + lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); + lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); } cur_perf = READ_ONCE(cpudata->perf); diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index f24a0aabea4fb..d17996e1d0add 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -90,11 +90,6 @@ static bool cppc_enabled; static bool amd_pstate_prefcore = true; static struct quirk_entry *quirks; -#define AMD_CPPC_MAX_PERF_MASK GENMASK(7, 0) -#define AMD_CPPC_MIN_PERF_MASK GENMASK(15, 8) -#define AMD_CPPC_DES_PERF_MASK GENMASK(23, 16) -#define AMD_CPPC_EPP_PERF_MASK GENMASK(31, 24) - /* * AMD Energy Preference Performance (EPP) * The EPP is used in the CCLK DPM controller to drive @@ -440,12 +435,13 @@ static int msr_init_perf(struct amd_cpudata *cpudata) perf.highest_perf = numerator; perf.max_limit_perf = numerator; - perf.min_limit_perf = AMD_CPPC_LOWEST_PERF(cap1); - perf.nominal_perf = AMD_CPPC_NOMINAL_PERF(cap1); - perf.lowest_nonlinear_perf = AMD_CPPC_LOWNONLIN_PERF(cap1); - perf.lowest_perf = AMD_CPPC_LOWEST_PERF(cap1); + perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); + perf.nominal_perf = FIELD_GET(AMD_CPPC_NOMINAL_PERF_MASK, cap1); + perf.lowest_nonlinear_perf = FIELD_GET(AMD_CPPC_LOWNONLIN_PERF_MASK, cap1); + perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); WRITE_ONCE(cpudata->perf, perf); - WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1)); + WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1)); + return 0; } From 124966cc7e32044727d367a42fcd63eaca54d86d Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 11:54:32 -0600 Subject: [PATCH 119/188] cpufreq/amd-pstate: Cache CPPC request in shared mem case too commit 9f5daa2f2f6dddd2a847d077bfddc7dc0d9dab10 upstream. In order to prevent a potential write for shmem_update_perf() cache the request into the cppc_req_cached variable normally only used for the MSR case. This adds symmetry into the code and potentially avoids extra writes. Reviewed-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index d17996e1d0add..57cbec5dea8e0 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -497,6 +497,8 @@ static int shmem_update_perf(struct amd_cpudata *cpudata, u8 min_perf, u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) { struct cppc_perf_ctrls perf_ctrls; + u64 value, prev; + int ret; if (cppc_state == AMD_PSTATE_ACTIVE) { int ret = shmem_set_epp(cpudata, epp); @@ -505,11 +507,29 @@ static int shmem_update_perf(struct amd_cpudata *cpudata, u8 min_perf, return ret; } + value = prev = READ_ONCE(cpudata->cppc_req_cached); + + value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK | + AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK); + value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf); + value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf); + value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); + value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + + if (value == prev) + return 0; + perf_ctrls.max_perf = max_perf; perf_ctrls.min_perf = min_perf; perf_ctrls.desired_perf = des_perf; - return cppc_set_perf(cpudata->cpu, &perf_ctrls); + ret = cppc_set_perf(cpudata->cpu, &perf_ctrls); + if (ret) + return ret; + + WRITE_ONCE(cpudata->cppc_req_cached, value); + + return 0; } static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) From 172dc5b4d53937f8239a00307a6981b96001d442 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 11:57:38 -0600 Subject: [PATCH 120/188] cpufreq/amd-pstate: Move all EPP tracing into *_update_perf and *_set_epp functions commit 77fbea69b0ffad0e46c7018d07e04b6628482e14 upstream. The EPP tracing is done by the caller today, but this precludes the information about whether the CPPC request has changed. Move it into the update_perf and set_epp functions and include information about whether the request has changed from the last one. amd_pstate_update_perf() and amd_pstate_set_epp() now require the policy as an argument instead of the cpudata. Reviewed-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-trace.h | 13 +++- drivers/cpufreq/amd-pstate.c | 118 +++++++++++++++++------------ 2 files changed, 80 insertions(+), 51 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-trace.h b/drivers/cpufreq/amd-pstate-trace.h index f457d4af2c62e..32e1bdc588c52 100644 --- a/drivers/cpufreq/amd-pstate-trace.h +++ b/drivers/cpufreq/amd-pstate-trace.h @@ -90,7 +90,8 @@ TRACE_EVENT(amd_pstate_epp_perf, u8 epp, u8 min_perf, u8 max_perf, - bool boost + bool boost, + bool changed ), TP_ARGS(cpu_id, @@ -98,7 +99,8 @@ TRACE_EVENT(amd_pstate_epp_perf, epp, min_perf, max_perf, - boost), + boost, + changed), TP_STRUCT__entry( __field(unsigned int, cpu_id) @@ -107,6 +109,7 @@ TRACE_EVENT(amd_pstate_epp_perf, __field(u8, min_perf) __field(u8, max_perf) __field(bool, boost) + __field(bool, changed) ), TP_fast_assign( @@ -116,15 +119,17 @@ TRACE_EVENT(amd_pstate_epp_perf, __entry->min_perf = min_perf; __entry->max_perf = max_perf; __entry->boost = boost; + __entry->changed = changed; ), - TP_printk("cpu%u: [%hhu<->%hhu]/%hhu, epp=%hhu, boost=%u", + TP_printk("cpu%u: [%hhu<->%hhu]/%hhu, epp=%hhu, boost=%u, changed=%u", (unsigned int)__entry->cpu_id, (u8)__entry->min_perf, (u8)__entry->max_perf, (u8)__entry->highest_perf, (u8)__entry->epp, - (bool)__entry->boost + (bool)__entry->boost, + (bool)__entry->changed ) ); diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 57cbec5dea8e0..e0dd4d116b48f 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -229,9 +229,10 @@ static u8 shmem_get_epp(struct amd_cpudata *cpudata) return FIELD_GET(AMD_CPPC_EPP_PERF_MASK, epp); } -static int msr_update_perf(struct amd_cpudata *cpudata, u8 min_perf, +static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) { + struct amd_cpudata *cpudata = policy->driver_data; u64 value, prev; value = prev = READ_ONCE(cpudata->cppc_req_cached); @@ -243,6 +244,18 @@ static int msr_update_perf(struct amd_cpudata *cpudata, u8 min_perf, value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + if (trace_amd_pstate_epp_perf_enabled()) { + union perf_cached perf = READ_ONCE(cpudata->perf); + + trace_amd_pstate_epp_perf(cpudata->cpu, + perf.highest_perf, + epp, + min_perf, + max_perf, + policy->boost_enabled, + value != prev); + } + if (value == prev) return 0; @@ -257,24 +270,26 @@ static int msr_update_perf(struct amd_cpudata *cpudata, u8 min_perf, } WRITE_ONCE(cpudata->cppc_req_cached, value); - WRITE_ONCE(cpudata->epp_cached, epp); + if (epp != cpudata->epp_cached) + WRITE_ONCE(cpudata->epp_cached, epp); return 0; } DEFINE_STATIC_CALL(amd_pstate_update_perf, msr_update_perf); -static inline int amd_pstate_update_perf(struct amd_cpudata *cpudata, +static inline int amd_pstate_update_perf(struct cpufreq_policy *policy, u8 min_perf, u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) { - return static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, + return static_call(amd_pstate_update_perf)(policy, min_perf, des_perf, max_perf, epp, fast_switch); } -static int msr_set_epp(struct amd_cpudata *cpudata, u8 epp) +static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) { + struct amd_cpudata *cpudata = policy->driver_data; u64 value, prev; int ret; @@ -282,6 +297,19 @@ static int msr_set_epp(struct amd_cpudata *cpudata, u8 epp) value &= ~AMD_CPPC_EPP_PERF_MASK; value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + if (trace_amd_pstate_epp_perf_enabled()) { + union perf_cached perf = cpudata->perf; + + trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, + epp, + FIELD_GET(AMD_CPPC_MIN_PERF_MASK, + cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MAX_PERF_MASK, + cpudata->cppc_req_cached), + policy->boost_enabled, + value != prev); + } + if (value == prev) return 0; @@ -300,15 +328,29 @@ static int msr_set_epp(struct amd_cpudata *cpudata, u8 epp) DEFINE_STATIC_CALL(amd_pstate_set_epp, msr_set_epp); -static inline int amd_pstate_set_epp(struct amd_cpudata *cpudata, u8 epp) +static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp) { - return static_call(amd_pstate_set_epp)(cpudata, epp); + return static_call(amd_pstate_set_epp)(policy, epp); } -static int shmem_set_epp(struct amd_cpudata *cpudata, u8 epp) +static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) { - int ret; + struct amd_cpudata *cpudata = policy->driver_data; struct cppc_perf_ctrls perf_ctrls; + int ret; + + if (trace_amd_pstate_epp_perf_enabled()) { + union perf_cached perf = cpudata->perf; + + trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, + epp, + FIELD_GET(AMD_CPPC_MIN_PERF_MASK, + cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MAX_PERF_MASK, + cpudata->cppc_req_cached), + policy->boost_enabled, + epp != cpudata->epp_cached); + } if (epp == cpudata->epp_cached) return 0; @@ -340,17 +382,7 @@ static int amd_pstate_set_energy_pref_index(struct cpufreq_policy *policy, return -EBUSY; } - if (trace_amd_pstate_epp_perf_enabled()) { - union perf_cached perf = READ_ONCE(cpudata->perf); - - trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, - epp, - FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), - FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), - policy->boost_enabled); - } - - return amd_pstate_set_epp(cpudata, epp); + return amd_pstate_set_epp(policy, epp); } static inline int msr_cppc_enable(bool enable) @@ -493,15 +525,16 @@ static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) return static_call(amd_pstate_init_perf)(cpudata); } -static int shmem_update_perf(struct amd_cpudata *cpudata, u8 min_perf, +static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf, u8 des_perf, u8 max_perf, u8 epp, bool fast_switch) { + struct amd_cpudata *cpudata = policy->driver_data; struct cppc_perf_ctrls perf_ctrls; u64 value, prev; int ret; if (cppc_state == AMD_PSTATE_ACTIVE) { - int ret = shmem_set_epp(cpudata, epp); + int ret = shmem_set_epp(policy, epp); if (ret) return ret; @@ -516,6 +549,18 @@ static int shmem_update_perf(struct amd_cpudata *cpudata, u8 min_perf, value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf); value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + if (trace_amd_pstate_epp_perf_enabled()) { + union perf_cached perf = READ_ONCE(cpudata->perf); + + trace_amd_pstate_epp_perf(cpudata->cpu, + perf.highest_perf, + epp, + min_perf, + max_perf, + policy->boost_enabled, + value != prev); + } + if (value == prev) return 0; @@ -593,7 +638,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, cpudata->cpu, fast_switch); } - amd_pstate_update_perf(cpudata, min_perf, des_perf, max_perf, 0, fast_switch); + amd_pstate_update_perf(policy, min_perf, des_perf, max_perf, 0, fast_switch); } static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) @@ -1522,7 +1567,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) return ret; WRITE_ONCE(cpudata->cppc_req_cached, value); } - ret = amd_pstate_set_epp(cpudata, cpudata->epp_default); + ret = amd_pstate_set_epp(policy, cpudata->epp_default); if (ret) return ret; @@ -1563,14 +1608,8 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) epp = READ_ONCE(cpudata->epp_cached); perf = READ_ONCE(cpudata->perf); - if (trace_amd_pstate_epp_perf_enabled()) { - trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, epp, - perf.min_limit_perf, - perf.max_limit_perf, - policy->boost_enabled); - } - return amd_pstate_update_perf(cpudata, perf.min_limit_perf, 0U, + return amd_pstate_update_perf(policy, perf.min_limit_perf, 0U, perf.max_limit_perf, epp, false); } @@ -1602,20 +1641,12 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) static int amd_pstate_epp_reenable(struct cpufreq_policy *policy) { - struct amd_cpudata *cpudata = policy->driver_data; - union perf_cached perf = READ_ONCE(cpudata->perf); int ret; ret = amd_pstate_cppc_enable(true); if (ret) pr_err("failed to enable amd pstate during resume, return %d\n", ret); - if (trace_amd_pstate_epp_perf_enabled()) { - trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, - cpudata->epp_cached, - FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cpudata->cppc_req_cached), - perf.highest_perf, policy->boost_enabled); - } return amd_pstate_epp_update_limit(policy); } @@ -1643,14 +1674,7 @@ static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) if (cpudata->suspended) return 0; - if (trace_amd_pstate_epp_perf_enabled()) { - trace_amd_pstate_epp_perf(cpudata->cpu, perf.highest_perf, - AMD_CPPC_EPP_BALANCE_POWERSAVE, - perf.lowest_perf, perf.lowest_perf, - policy->boost_enabled); - } - - return amd_pstate_update_perf(cpudata, perf.lowest_perf, 0, perf.lowest_perf, + return amd_pstate_update_perf(policy, perf.lowest_perf, 0, perf.lowest_perf, AMD_CPPC_EPP_BALANCE_POWERSAVE, false); } From 872016da92396494bc50e8aaca412f108a3b0466 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 12:02:14 -0600 Subject: [PATCH 121/188] cpufreq/amd-pstate: Update cppc_req_cached for shared mem EPP writes commit 1905fac6f9e08e34135e089704a0733ce711eb83 upstream. On EPP only writes update the cached variable so that the min/max performance controls don't need to be updated again. Reviewed-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index e0dd4d116b48f..fa6addf154378 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -337,6 +337,7 @@ static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) { struct amd_cpudata *cpudata = policy->driver_data; struct cppc_perf_ctrls perf_ctrls; + u64 value; int ret; if (trace_amd_pstate_epp_perf_enabled()) { @@ -363,6 +364,11 @@ static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) } WRITE_ONCE(cpudata->epp_cached, epp); + value = READ_ONCE(cpudata->cppc_req_cached); + value &= ~AMD_CPPC_EPP_PERF_MASK; + value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp); + WRITE_ONCE(cpudata->cppc_req_cached, value); + return ret; } From 1d285e786334f67a230771728113e0f48bdcb1be Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Dec 2024 14:44:15 -0600 Subject: [PATCH 122/188] cpufreq/amd-pstate: Drop debug statements for policy setting commit 93039a60fb28f72196769869aa4b502f1849a373 upstream. There are trace events that exist now for all amd-pstate modes that will output information right before programming to the hardware. This makes the existing debug statements unnecessary remaining overhead. Drop them. Reviewed-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index fa6addf154378..b811b731e1f93 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -668,7 +668,6 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) } cpufreq_verify_within_cpu_limits(policy_data); - pr_debug("policy_max =%d, policy_min=%d\n", policy_data->max, policy_data->min); return 0; } @@ -1627,9 +1626,6 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) if (!policy->cpuinfo.max_freq) return -ENODEV; - pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", - policy->cpuinfo.max_freq, policy->max); - cpudata->policy = policy->policy; ret = amd_pstate_epp_update_limit(policy); From cf402b015cf380159cb55e0504e3f9ea7b2a5dff Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 16 Dec 2024 10:41:23 -0600 Subject: [PATCH 123/188] cpufreq/amd-pstate: Rework CPPC enabling commit 2064543f5ba0d2929e3e9b3a616c3262a57c7925 upstream. The CPPC enable register is configured as "write once". That is any future writes don't actually do anything. Because of this, all the cleanup paths that currently exist for CPPC disable are non-effective. Rework CPPC enable to only enable after all the CAP registers have been read to avoid enabling CPPC on CPUs with invalid _CPC or unpopulated MSRs. As the register is write once, remove all cleanup paths as well. [Backport Changes] 1. The upstream patch was auto-merged successfully. However, during diff review, some changes appeared due to line shifts in the file. These are not actual conflicts and therefore do not constitute a conflict-related deviation. Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 179 +++++++---------------------------- 1 file changed, 35 insertions(+), 144 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b811b731e1f93..f75093be0e7df 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -86,7 +86,6 @@ static struct cpufreq_driver *current_pstate_driver; static struct cpufreq_driver amd_pstate_driver; static struct cpufreq_driver amd_pstate_epp_driver; static int cppc_state = AMD_PSTATE_UNDEFINED; -static bool cppc_enabled; static bool amd_pstate_prefcore = true; static struct quirk_entry *quirks; @@ -372,89 +371,21 @@ static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) return ret; } -static int amd_pstate_set_energy_pref_index(struct cpufreq_policy *policy, - int pref_index) +static inline int msr_cppc_enable(struct cpufreq_policy *policy) { - struct amd_cpudata *cpudata = policy->driver_data; - u8 epp; - - if (!pref_index) - epp = cpudata->epp_default; - else - epp = epp_values[pref_index]; - - if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { - pr_debug("EPP cannot be set under performance policy\n"); - return -EBUSY; - } - - return amd_pstate_set_epp(policy, epp); + return wrmsrl_safe_on_cpu(policy->cpu, MSR_AMD_CPPC_ENABLE, 1); } -static inline int msr_cppc_enable(bool enable) +static int shmem_cppc_enable(struct cpufreq_policy *policy) { - int ret, cpu; - unsigned long logical_proc_id_mask = 0; - - /* - * MSR_AMD_CPPC_ENABLE is write-once, once set it cannot be cleared. - */ - if (!enable) - return 0; - - if (enable == cppc_enabled) - return 0; - - for_each_present_cpu(cpu) { - unsigned long logical_id = topology_logical_package_id(cpu); - - if (test_bit(logical_id, &logical_proc_id_mask)) - continue; - - set_bit(logical_id, &logical_proc_id_mask); - - ret = wrmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_ENABLE, - enable); - if (ret) - return ret; - } - - cppc_enabled = enable; - return 0; -} - -static int shmem_cppc_enable(bool enable) -{ - int cpu, ret = 0; - struct cppc_perf_ctrls perf_ctrls; - - if (enable == cppc_enabled) - return 0; - - for_each_present_cpu(cpu) { - ret = cppc_set_enable(cpu, enable); - if (ret) - return ret; - - /* Enable autonomous mode for EPP */ - if (cppc_state == AMD_PSTATE_ACTIVE) { - /* Set desired perf as zero to allow EPP firmware control */ - perf_ctrls.desired_perf = 0; - ret = cppc_set_perf(cpu, &perf_ctrls); - if (ret) - return ret; - } - } - - cppc_enabled = enable; - return ret; + return cppc_set_enable(policy->cpu, 1); } DEFINE_STATIC_CALL(amd_pstate_cppc_enable, msr_cppc_enable); -static inline int amd_pstate_cppc_enable(bool enable) +static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) { - return static_call(amd_pstate_cppc_enable)(enable); + return static_call(amd_pstate_cppc_enable)(policy); } static int msr_init_perf(struct amd_cpudata *cpudata) @@ -1060,6 +991,10 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) cpudata->nominal_freq, perf.highest_perf); + ret = amd_pstate_cppc_enable(policy); + if (ret) + goto free_cpudata1; + policy->boost_supported = READ_ONCE(cpudata->boost_supported); /* It will be updated by governor */ @@ -1107,28 +1042,6 @@ static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) kfree(cpudata); } -static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) -{ - int ret; - - ret = amd_pstate_cppc_enable(true); - if (ret) - pr_err("failed to enable amd-pstate during resume, return %d\n", ret); - - return ret; -} - -static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) -{ - int ret; - - ret = amd_pstate_cppc_enable(false); - if (ret) - pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); - - return ret; -} - /* Sysfs attributes */ /* @@ -1220,8 +1133,10 @@ static ssize_t show_energy_performance_available_preferences( static ssize_t store_energy_performance_preference( struct cpufreq_policy *policy, const char *buf, size_t count) { + struct amd_cpudata *cpudata = policy->driver_data; char str_preference[21]; ssize_t ret; + u8 epp; ret = sscanf(buf, "%20s", str_preference); if (ret != 1) @@ -1231,7 +1146,17 @@ static ssize_t store_energy_performance_preference( if (ret < 0) return -EINVAL; - ret = amd_pstate_set_energy_pref_index(policy, ret); + if (!ret) + epp = cpudata->epp_default; + else + epp = epp_values[ret]; + + if (epp > 0 && policy->policy == CPUFREQ_POLICY_PERFORMANCE) { + pr_debug("EPP cannot be set under performance policy\n"); + return -EBUSY; + } + + ret = amd_pstate_set_epp(policy, epp); return ret ? ret : count; } @@ -1264,7 +1189,6 @@ static ssize_t show_energy_performance_preference( static void amd_pstate_driver_cleanup(void) { - amd_pstate_cppc_enable(false); cppc_state = AMD_PSTATE_DISABLE; current_pstate_driver = NULL; } @@ -1298,14 +1222,6 @@ static int amd_pstate_register_driver(int mode) cppc_state = mode; - ret = amd_pstate_cppc_enable(true); - if (ret) { - pr_err("failed to enable cppc during amd-pstate driver registration, return %d\n", - ret); - amd_pstate_driver_cleanup(); - return ret; - } - /* at least one CPU supports CPB */ current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); @@ -1545,11 +1461,15 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf); + policy->driver_data = cpudata; + + ret = amd_pstate_cppc_enable(policy); + if (ret) + goto free_cpudata1; /* It will be updated by governor */ policy->cur = policy->cpuinfo.min_freq; - policy->driver_data = cpudata; policy->boost_supported = READ_ONCE(cpudata->boost_supported); @@ -1641,31 +1561,11 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) return 0; } -static int amd_pstate_epp_reenable(struct cpufreq_policy *policy) -{ - int ret; - - ret = amd_pstate_cppc_enable(true); - if (ret) - pr_err("failed to enable amd pstate during resume, return %d\n", ret); - - - return amd_pstate_epp_update_limit(policy); -} - static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) { - struct amd_cpudata *cpudata = policy->driver_data; - int ret; - - pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); + pr_debug("AMD CPU Core %d going online\n", policy->cpu); - ret = amd_pstate_epp_reenable(policy); - if (ret) - return ret; - cpudata->suspended = false; - - return 0; + return amd_pstate_cppc_enable(policy); } static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) @@ -1683,11 +1583,6 @@ static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; - int ret; - - /* avoid suspending when EPP is not enabled */ - if (cppc_state != AMD_PSTATE_ACTIVE) - return 0; /* invalidate to ensure it's rewritten during resume */ cpudata->cppc_req_cached = 0; @@ -1695,11 +1590,6 @@ static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) /* set this flag to avoid setting core offline*/ cpudata->suspended = true; - /* disable CPPC in lowlevel firmware */ - ret = amd_pstate_cppc_enable(false); - if (ret) - pr_err("failed to suspend, return %d\n", ret); - return 0; } @@ -1708,8 +1598,12 @@ static int amd_pstate_epp_resume(struct cpufreq_policy *policy) struct amd_cpudata *cpudata = policy->driver_data; if (cpudata->suspended) { + int ret; + /* enable amd pstate from suspend state*/ - amd_pstate_epp_reenable(policy); + ret = amd_pstate_epp_update_limit(policy); + if (ret) + return ret; cpudata->suspended = false; } @@ -1724,8 +1618,6 @@ static struct cpufreq_driver amd_pstate_driver = { .fast_switch = amd_pstate_fast_switch, .init = amd_pstate_cpu_init, .exit = amd_pstate_cpu_exit, - .suspend = amd_pstate_cpu_suspend, - .resume = amd_pstate_cpu_resume, .set_boost = amd_pstate_set_boost, .update_limits = amd_pstate_update_limits, .name = "amd-pstate", @@ -1892,7 +1784,6 @@ static int __init amd_pstate_init(void) global_attr_free: cpufreq_unregister_driver(current_pstate_driver); - amd_pstate_cppc_enable(false); return ret; } device_initcall(amd_pstate_init); From 07435bc17781f1f308c3b4f887a837322a95ab8a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Sun, 19 Jan 2025 07:05:43 -0600 Subject: [PATCH 124/188] cpufreq/amd-pstate: Stop caching EPP commit 4e16c1175238f2b9d86199b427c5db1a24c9a85f upstream. EPP values are cached in the cpudata structure per CPU. This is needless though because they are also cached in the CPPC request variable. Drop the separate cache for EPP values and always reference the CPPC request variable when needed. Reviewed-by: Dhananjay Ugwekar Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 19 ++++++++++--------- drivers/cpufreq/amd-pstate.h | 1 - 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index f75093be0e7df..65aeb006235c7 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -269,8 +269,6 @@ static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, } WRITE_ONCE(cpudata->cppc_req_cached, value); - if (epp != cpudata->epp_cached) - WRITE_ONCE(cpudata->epp_cached, epp); return 0; } @@ -319,7 +317,6 @@ static int msr_set_epp(struct cpufreq_policy *policy, u8 epp) } /* update both so that msr_update_perf() can effectively check */ - WRITE_ONCE(cpudata->epp_cached, epp); WRITE_ONCE(cpudata->cppc_req_cached, value); return ret; @@ -336,9 +333,12 @@ static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) { struct amd_cpudata *cpudata = policy->driver_data; struct cppc_perf_ctrls perf_ctrls; + u8 epp_cached; u64 value; int ret; + + epp_cached = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); if (trace_amd_pstate_epp_perf_enabled()) { union perf_cached perf = cpudata->perf; @@ -349,10 +349,10 @@ static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), policy->boost_enabled, - epp != cpudata->epp_cached); + epp != epp_cached); } - if (epp == cpudata->epp_cached) + if (epp == epp_cached) return 0; perf_ctrls.energy_perf = epp; @@ -361,7 +361,6 @@ static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) pr_debug("failed to set energy perf value (%d)\n", ret); return ret; } - WRITE_ONCE(cpudata->epp_cached, epp); value = READ_ONCE(cpudata->cppc_req_cached); value &= ~AMD_CPPC_EPP_PERF_MASK; @@ -1165,9 +1164,11 @@ static ssize_t show_energy_performance_preference( struct cpufreq_policy *policy, char *buf) { struct amd_cpudata *cpudata = policy->driver_data; - u8 preference; + u8 preference, epp; + + epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); - switch (cpudata->epp_cached) { + switch (epp) { case AMD_CPPC_EPP_PERFORMANCE: preference = EPP_INDEX_PERFORMANCE; break; @@ -1530,7 +1531,7 @@ static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) epp = 0; else - epp = READ_ONCE(cpudata->epp_cached); + epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); perf = READ_ONCE(cpudata->perf); diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 1557e1afea79c..fbe1c08d3f061 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -102,7 +102,6 @@ struct amd_cpudata { bool hw_prefcore; /* EPP feature related attributes*/ - u8 epp_cached; u32 policy; bool suspended; u8 epp_default; From 386d8a097c2eb5d86e61fa0cd5cc4765cd8ec137 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Wed, 19 Feb 2025 14:08:14 -0600 Subject: [PATCH 125/188] cpufreq/amd-pstate: Drop actions in amd_pstate_epp_cpu_offline() commit efb758c8c803217e58248f03db372c5e23827dae upstream. When the CPU goes offline there is no need to change the CPPC request because the CPU will go into the deepest C-state it supports already. Actually changing the CPPC request when it goes offline messes up the cached values and can lead to the wrong values being restored when it comes back. Instead drop the actions and if the CPU comes back online let amd_pstate_epp_set_policy() restore it to expected values. Reviewed-by: Dhananjay Ugwekar Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 65aeb006235c7..31ec3c550f807 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1571,14 +1571,7 @@ static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) { - struct amd_cpudata *cpudata = policy->driver_data; - union perf_cached perf = READ_ONCE(cpudata->perf); - - if (cpudata->suspended) - return 0; - - return amd_pstate_update_perf(policy, perf.lowest_perf, 0, perf.lowest_perf, - AMD_CPPC_EPP_BALANCE_POWERSAVE, false); + return 0; } static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) From 7f6bcfdb01749d66fd870bbddb0e869db723ef73 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Mon, 7 Apr 2025 08:19:26 +0000 Subject: [PATCH 126/188] cpufreq/amd-pstate: Fix min_limit perf and freq updation for performance governor commit 56a49e19e1aea1374e9ba58cfd40260587bb7355 upstream. The min_limit perf and freq values can get disconnected with performance governor, as we only modify the perf value in the special case. Fix that by modifying the perf and freq values together Fixes: 009d1c29a451 ("cpufreq/amd-pstate: Move perf values into a union") Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20250407081925.850473-1-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Arukonda Rahul Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 31ec3c550f807..58a6ac5a273d9 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -608,13 +608,16 @@ static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) union perf_cached perf = READ_ONCE(cpudata->perf); perf.max_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->max); - perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); + WRITE_ONCE(cpudata->max_limit_freq, policy->max); - if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) + if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); + WRITE_ONCE(cpudata->min_limit_freq, min(cpudata->nominal_freq, cpudata->max_limit_freq)); + } else { + perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); + WRITE_ONCE(cpudata->min_limit_freq, policy->min); + } - WRITE_ONCE(cpudata->max_limit_freq, policy->max); - WRITE_ONCE(cpudata->min_limit_freq, policy->min); WRITE_ONCE(cpudata->perf, perf); } From 92be73ad5026f5ab07693d02c3cc2e425c402847 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 4 Sep 2024 13:44:54 +0200 Subject: [PATCH 127/188] cpufreq: intel_pstate: Set asymmetric CPU capacity on hybrid systems commit 929ebc93ccaa8d183a2ba9f1cf769d1bfb847cca upstream. Make intel_pstate use the HWP_HIGHEST_PERF values from MSR_HWP_CAPABILITIES to set asymmetric CPU capacity information via the previously introduced arch_set_cpu_capacity() on hybrid systems without SMT. Setting asymmetric CPU capacity is generally necessary to allow the scheduler to compute task sizes in a consistent way across all CPUs in a system where they differ by capacity. That, in turn, should help to improve scheduling decisions. It is also necessary for the schedutil cpufreq governor to operate as expected on hybrid systems where tasks migrate between CPUs of different capacities. The underlying observation is that intel_pstate already uses MSR_HWP_CAPABILITIES to get CPU performance information which is exposed by it via sysfs and CPU performance scaling is based on it. Thus using this information for setting asymmetric CPU capacity is consistent with what the driver has been doing already. Moreover, HWP_HIGHEST_PERF reflects the maximum capacity of a given CPU including both the instructions-per-cycle (IPC) factor and the maximum turbo frequency and the units in which that value is expressed are the same for all CPUs in the system, so the maximum capacity ratio between two CPUs can be obtained by computing the ratio of their HWP_HIGHEST_PERF values. Of course, in principle that capacity ratio need not be directly applicable at lower frequencies, so using it for providing the asymmetric CPU capacity information to the scheduler is a rough approximation, but it is as good as it gets. Also, measurements indicate that this approximation is not too bad in practice. If the given system is hybrid and non-SMT, the new code disables ITMT support in the scheduler (because it may get in the way of asymmetric CPU capacity code in the scheduler that automatically gets enabled by setting asymmetric CPU capacity) after initializing all online CPUs and finds the one with the maximum HWP_HIGHEST_PERF value. Next, it computes the capacity number for each (online) CPU by dividing the product of its HWP_HIGHEST_PERF and SCHED_CAPACITY_SCALE by the maximum HWP_HIGHEST_PERF. When a CPU goes offline, its capacity is reset to SCHED_CAPACITY_SCALE and if it is the one with the maximum HWP_HIGHEST_PERF value, the capacity numbers for all of the other online CPUs are recomputed. This also takes care of a cleanup during driver operation mode changes. Analogously, when a new CPU goes online, its capacity number is updated and if its HWP_HIGHEST_PERF value is greater than the current maximum one, the capacity numbers for all of the other online CPUs are recomputed. The case when the driver is notified of a CPU capacity change, either through the HWP interrupt or through an ACPI notification, is handled similarly to the CPU online case above, except that if the target CPU is the current highest-capacity one and its capacity is reduced, the capacity numbers for all of the other online CPUs need to be recomputed either. If the driver's "no_trubo" sysfs attribute is updated, all of the CPU capacity information is computed from scratch to reflect the new turbo status. Signed-off-by: Rafael J. Wysocki Reviewed-by: Ricardo Neri Tested-by: Ricardo Neri # scale invariance Link: https://patch.msgid.link/1979653.PYKUYFuaPT@rjwysocki.net [ rjw: Fixed a typo in the changelog ] [ rjw: Renamed 3 new functions and added a comment ] Signed-off-by: Rafael J. Wysocki Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/intel_pstate.c | 236 ++++++++++++++++++++++++++++++++- 1 file changed, 232 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index b87f3609fbd51..98fd23ef0b08c 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -215,6 +216,7 @@ struct global_params { * @hwp_req_cached: Cached value of the last HWP Request MSR * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR * @last_io_update: Last time when IO wake flag was set + * @capacity_perf: Highest perf used for scale invariance * @sched_flags: Store scheduler flags for possible cross CPU update * @hwp_boost_min: Last HWP boosted min performance * @suspended: Whether or not the driver has been suspended. @@ -253,6 +255,7 @@ struct cpudata { u64 hwp_req_cached; u64 hwp_cap_cached; u64 last_io_update; + unsigned int capacity_perf; unsigned int sched_flags; u32 hwp_boost_min; bool suspended; @@ -295,6 +298,7 @@ static int hwp_mode_bdw __ro_after_init; static bool per_cpu_limits __ro_after_init; static bool hwp_forced __ro_after_init; static bool hwp_boost __read_mostly; +static bool hwp_is_hybrid; static struct cpufreq_driver *intel_pstate_driver __read_mostly; @@ -933,6 +937,139 @@ static struct freq_attr *hwp_cpufreq_attrs[] = { NULL, }; +static struct cpudata *hybrid_max_perf_cpu __read_mostly; +/* + * Protects hybrid_max_perf_cpu, the capacity_perf fields in struct cpudata, + * and the x86 arch scale-invariance information from concurrent updates. + */ +static DEFINE_MUTEX(hybrid_capacity_lock); + +static void hybrid_set_cpu_capacity(struct cpudata *cpu) +{ + arch_set_cpu_capacity(cpu->cpu, cpu->capacity_perf, + hybrid_max_perf_cpu->capacity_perf, + cpu->capacity_perf, + cpu->pstate.max_pstate_physical); + + pr_debug("CPU%d: perf = %u, max. perf = %u, base perf = %d\n", cpu->cpu, + cpu->capacity_perf, hybrid_max_perf_cpu->capacity_perf, + cpu->pstate.max_pstate_physical); +} + +static void hybrid_clear_cpu_capacity(unsigned int cpunum) +{ + arch_set_cpu_capacity(cpunum, 1, 1, 1, 1); +} + +static void hybrid_get_capacity_perf(struct cpudata *cpu) +{ + if (READ_ONCE(global.no_turbo)) { + cpu->capacity_perf = cpu->pstate.max_pstate_physical; + return; + } + + cpu->capacity_perf = HWP_HIGHEST_PERF(READ_ONCE(cpu->hwp_cap_cached)); +} + +static void hybrid_set_capacity_of_cpus(void) +{ + int cpunum; + + for_each_online_cpu(cpunum) { + struct cpudata *cpu = all_cpu_data[cpunum]; + + if (cpu) + hybrid_set_cpu_capacity(cpu); + } +} + +static void hybrid_update_cpu_capacity_scaling(void) +{ + struct cpudata *max_perf_cpu = NULL; + unsigned int max_cap_perf = 0; + int cpunum; + + for_each_online_cpu(cpunum) { + struct cpudata *cpu = all_cpu_data[cpunum]; + + if (!cpu) + continue; + + /* + * During initialization, CPU performance at full capacity needs + * to be determined. + */ + if (!hybrid_max_perf_cpu) + hybrid_get_capacity_perf(cpu); + + /* + * If hybrid_max_perf_cpu is not NULL at this point, it is + * being replaced, so don't take it into account when looking + * for the new one. + */ + if (cpu == hybrid_max_perf_cpu) + continue; + + if (cpu->capacity_perf > max_cap_perf) { + max_cap_perf = cpu->capacity_perf; + max_perf_cpu = cpu; + } + } + + if (max_perf_cpu) { + hybrid_max_perf_cpu = max_perf_cpu; + hybrid_set_capacity_of_cpus(); + } else { + pr_info("Found no CPUs with nonzero maximum performance\n"); + /* Revert to the flat CPU capacity structure. */ + for_each_online_cpu(cpunum) + hybrid_clear_cpu_capacity(cpunum); + } +} + +static void __hybrid_init_cpu_capacity_scaling(void) +{ + hybrid_max_perf_cpu = NULL; + hybrid_update_cpu_capacity_scaling(); +} + +static void hybrid_init_cpu_capacity_scaling(void) +{ + bool disable_itmt = false; + + mutex_lock(&hybrid_capacity_lock); + + /* + * If hybrid_max_perf_cpu is set at this point, the hybrid CPU capacity + * scaling has been enabled already and the driver is just changing the + * operation mode. + */ + if (hybrid_max_perf_cpu) { + __hybrid_init_cpu_capacity_scaling(); + goto unlock; + } + + /* + * On hybrid systems, use asym capacity instead of ITMT, but because + * the capacity of SMT threads is not deterministic even approximately, + * do not do that when SMT is in use. + */ + if (hwp_is_hybrid && !sched_smt_active() && arch_enable_hybrid_capacity_scale()) { + __hybrid_init_cpu_capacity_scaling(); + disable_itmt = true; + } + +unlock: + mutex_unlock(&hybrid_capacity_lock); + + /* + * Disabling ITMT causes sched domains to be rebuilt to disable asym + * packing and enable asym capacity. + */ + if (disable_itmt) + sched_clear_itmt_support(); +} + static void __intel_pstate_get_hwp_cap(struct cpudata *cpu) { u64 cap; @@ -961,6 +1098,43 @@ static void intel_pstate_get_hwp_cap(struct cpudata *cpu) } } +static void hybrid_update_capacity(struct cpudata *cpu) +{ + unsigned int max_cap_perf; + + mutex_lock(&hybrid_capacity_lock); + + if (!hybrid_max_perf_cpu) + goto unlock; + + /* + * The maximum performance of the CPU may have changed, but assume + * that the performance of the other CPUs has not changed. + */ + max_cap_perf = hybrid_max_perf_cpu->capacity_perf; + + intel_pstate_get_hwp_cap(cpu); + + hybrid_get_capacity_perf(cpu); + /* Should hybrid_max_perf_cpu be replaced by this CPU? */ + if (cpu->capacity_perf > max_cap_perf) { + hybrid_max_perf_cpu = cpu; + hybrid_set_capacity_of_cpus(); + goto unlock; + } + + /* If this CPU is hybrid_max_perf_cpu, should it be replaced? */ + if (cpu == hybrid_max_perf_cpu && cpu->capacity_perf < max_cap_perf) { + hybrid_update_cpu_capacity_scaling(); + goto unlock; + } + + hybrid_set_cpu_capacity(cpu); + +unlock: + mutex_unlock(&hybrid_capacity_lock); +} + static void intel_pstate_hwp_set(unsigned int cpu) { struct cpudata *cpu_data = all_cpu_data[cpu]; @@ -1069,6 +1243,22 @@ static void intel_pstate_hwp_offline(struct cpudata *cpu) value |= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE); wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value); + + mutex_lock(&hybrid_capacity_lock); + + if (!hybrid_max_perf_cpu) { + mutex_unlock(&hybrid_capacity_lock); + + return; + } + + if (hybrid_max_perf_cpu == cpu) + hybrid_update_cpu_capacity_scaling(); + + mutex_unlock(&hybrid_capacity_lock); + + /* Reset the capacity of the CPU going offline to the initial value. */ + hybrid_clear_cpu_capacity(cpu->cpu); } #define POWER_CTL_EE_ENABLE 1 @@ -1164,21 +1354,46 @@ static void __intel_pstate_update_max_freq(struct cpudata *cpudata, static void intel_pstate_update_limits(unsigned int cpu) { struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); + struct cpudata *cpudata; if (!policy) return; - __intel_pstate_update_max_freq(all_cpu_data[cpu], policy); + cpudata = all_cpu_data[cpu]; + + __intel_pstate_update_max_freq(cpudata, policy); + + /* Prevent the driver from being unregistered now. */ + mutex_lock(&intel_pstate_driver_lock); cpufreq_cpu_release(policy); + + hybrid_update_capacity(cpudata); + + mutex_unlock(&intel_pstate_driver_lock); } static void intel_pstate_update_limits_for_all(void) { int cpu; - for_each_possible_cpu(cpu) - intel_pstate_update_limits(cpu); + for_each_possible_cpu(cpu) { + struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); + + if (!policy) + continue; + + __intel_pstate_update_max_freq(all_cpu_data[cpu], policy); + + cpufreq_cpu_release(policy); + } + + mutex_lock(&hybrid_capacity_lock); + + if (hybrid_max_perf_cpu) + __hybrid_init_cpu_capacity_scaling(); + + mutex_unlock(&hybrid_capacity_lock); } /************************** sysfs begin ************************/ @@ -1619,6 +1834,13 @@ static void intel_pstate_notify_work(struct work_struct *work) __intel_pstate_update_max_freq(cpudata, policy); cpufreq_cpu_release(policy); + + /* + * The driver will not be unregistered while this function is + * running, so update the capacity without acquiring the driver + * lock. + */ + hybrid_update_capacity(cpudata); } wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_STATUS, 0); @@ -2035,8 +2257,10 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) if (pstate_funcs.get_cpu_scaling) { cpu->pstate.scaling = pstate_funcs.get_cpu_scaling(cpu->cpu); - if (cpu->pstate.scaling != perf_ctl_scaling) + if (cpu->pstate.scaling != perf_ctl_scaling) { intel_pstate_hybrid_hwp_adjust(cpu); + hwp_is_hybrid = true; + } } else { cpu->pstate.scaling = perf_ctl_scaling; } @@ -2704,6 +2928,8 @@ static int intel_pstate_cpu_online(struct cpufreq_policy *policy) */ intel_pstate_hwp_reenable(cpu); cpu->suspended = false; + + hybrid_update_capacity(cpu); } return 0; @@ -3144,6 +3370,8 @@ static int intel_pstate_register_driver(struct cpufreq_driver *driver) global.min_perf_pct = min_perf_pct_min(); + hybrid_init_cpu_capacity_scaling(); + return 0; } From 97ab1a009c612a4afc43a8db1c41cdb99a9763db Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 28 Aug 2024 13:47:25 +0200 Subject: [PATCH 128/188] x86/sched: Add basic support for CPU capacity scaling commit 5a9d10145a54f7a3fb6297c0082bf030e04db3bc upstream. In order be able to compute the sizes of tasks consistently across all CPUs in a hybrid system, it is necessary to provide CPU capacity scaling information to the scheduler via arch_scale_cpu_capacity(). Moreover, the value returned by arch_scale_freq_capacity() for the given CPU must correspond to the arch_scale_cpu_capacity() return value for it, or utilization computations will be inaccurate. Add support for it through per-CPU variables holding the capacity and maximum-to-base frequency ratio (times SCHED_CAPACITY_SCALE) that will be returned by arch_scale_cpu_capacity() and used by scale_freq_tick() to compute arch_freq_scale for the current CPU, respectively. In order to avoid adding measurable overhead for non-hybrid x86 systems, which are the vast majority in the field, whether or not the new hybrid CPU capacity scaling will be in effect is controlled by a static key. This static key is set by calling arch_enable_hybrid_capacity_scale() which also allocates memory for the per-CPU data and initializes it. Next, arch_set_cpu_capacity() is used to set the per-CPU variables mentioned above for each CPU and arch_rebuild_sched_domains() needs to be called for the scheduler to realize that capacity-aware scheduling can be used going forward. Signed-off-by: Rafael J. Wysocki Acked-by: Peter Zijlstra (Intel) Reviewed-by: Ricardo Neri Tested-by: Ricardo Neri # scale invariance Link: https://patch.msgid.link/10523497.nUPlyArG6x@rjwysocki.net [ rjw: Added parens to function kerneldoc comments ] Signed-off-by: Rafael J. Wysocki Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- arch/x86/include/asm/topology.h | 13 +++++ arch/x86/kernel/cpu/aperfmperf.c | 89 +++++++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 86c077f8ee990..3a7c18851016d 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -282,9 +282,22 @@ static inline long arch_scale_freq_capacity(int cpu) } #define arch_scale_freq_capacity arch_scale_freq_capacity +bool arch_enable_hybrid_capacity_scale(void); +void arch_set_cpu_capacity(int cpu, unsigned long cap, unsigned long max_cap, + unsigned long cap_freq, unsigned long base_freq); + +unsigned long arch_scale_cpu_capacity(int cpu); +#define arch_scale_cpu_capacity arch_scale_cpu_capacity + extern void arch_set_max_freq_ratio(bool turbo_disabled); extern void freq_invariance_set_perf_ratio(u64 ratio, bool turbo_disabled); #else +static inline bool arch_enable_hybrid_capacity_scale(void) { return false; } +static inline void arch_set_cpu_capacity(int cpu, unsigned long cap, + unsigned long max_cap, + unsigned long cap_freq, + unsigned long base_freq) { } + static inline void arch_set_max_freq_ratio(bool turbo_disabled) { } static inline void freq_invariance_set_perf_ratio(u64 ratio, bool turbo_disabled) { } #endif diff --git a/arch/x86/kernel/cpu/aperfmperf.c b/arch/x86/kernel/cpu/aperfmperf.c index ab646aee6387e..ed7ebf2d4314f 100644 --- a/arch/x86/kernel/cpu/aperfmperf.c +++ b/arch/x86/kernel/cpu/aperfmperf.c @@ -347,9 +347,89 @@ static DECLARE_WORK(disable_freq_invariance_work, DEFINE_PER_CPU(unsigned long, arch_freq_scale) = SCHED_CAPACITY_SCALE; +static DEFINE_STATIC_KEY_FALSE(arch_hybrid_cap_scale_key); + +struct arch_hybrid_cpu_scale { + unsigned long capacity; + unsigned long freq_ratio; +}; + +static struct arch_hybrid_cpu_scale __percpu *arch_cpu_scale; + +/** + * arch_enable_hybrid_capacity_scale() - Enable hybrid CPU capacity scaling + * + * Allocate memory for per-CPU data used by hybrid CPU capacity scaling, + * initialize it and set the static key controlling its code paths. + * + * Must be called before arch_set_cpu_capacity(). + */ +bool arch_enable_hybrid_capacity_scale(void) +{ + int cpu; + + if (static_branch_unlikely(&arch_hybrid_cap_scale_key)) { + WARN_ONCE(1, "Hybrid CPU capacity scaling already enabled"); + return true; + } + + arch_cpu_scale = alloc_percpu(struct arch_hybrid_cpu_scale); + if (!arch_cpu_scale) + return false; + + for_each_possible_cpu(cpu) { + per_cpu_ptr(arch_cpu_scale, cpu)->capacity = SCHED_CAPACITY_SCALE; + per_cpu_ptr(arch_cpu_scale, cpu)->freq_ratio = arch_max_freq_ratio; + } + + static_branch_enable(&arch_hybrid_cap_scale_key); + + pr_info("Hybrid CPU capacity scaling enabled\n"); + + return true; +} + +/** + * arch_set_cpu_capacity() - Set scale-invariance parameters for a CPU + * @cpu: Target CPU. + * @cap: Capacity of @cpu at its maximum frequency, relative to @max_cap. + * @max_cap: System-wide maximum CPU capacity. + * @cap_freq: Frequency of @cpu corresponding to @cap. + * @base_freq: Frequency of @cpu at which MPERF counts. + * + * The units in which @cap and @max_cap are expressed do not matter, so long + * as they are consistent, because the former is effectively divided by the + * latter. Analogously for @cap_freq and @base_freq. + * + * After calling this function for all CPUs, call arch_rebuild_sched_domains() + * to let the scheduler know that capacity-aware scheduling can be used going + * forward. + */ +void arch_set_cpu_capacity(int cpu, unsigned long cap, unsigned long max_cap, + unsigned long cap_freq, unsigned long base_freq) +{ + if (static_branch_likely(&arch_hybrid_cap_scale_key)) { + WRITE_ONCE(per_cpu_ptr(arch_cpu_scale, cpu)->capacity, + div_u64(cap << SCHED_CAPACITY_SHIFT, max_cap)); + WRITE_ONCE(per_cpu_ptr(arch_cpu_scale, cpu)->freq_ratio, + div_u64(cap_freq << SCHED_CAPACITY_SHIFT, base_freq)); + } else { + WARN_ONCE(1, "Hybrid CPU capacity scaling not enabled"); + } +} + +unsigned long arch_scale_cpu_capacity(int cpu) +{ + if (static_branch_unlikely(&arch_hybrid_cap_scale_key)) + return READ_ONCE(per_cpu_ptr(arch_cpu_scale, cpu)->capacity); + + return SCHED_CAPACITY_SCALE; +} +EXPORT_SYMBOL_GPL(arch_scale_cpu_capacity); + static void scale_freq_tick(u64 acnt, u64 mcnt) { - u64 freq_scale; + u64 freq_scale, freq_ratio; if (!arch_scale_freq_invariant()) return; @@ -357,7 +437,12 @@ static void scale_freq_tick(u64 acnt, u64 mcnt) if (check_shl_overflow(acnt, 2*SCHED_CAPACITY_SHIFT, &acnt)) goto error; - if (check_mul_overflow(mcnt, arch_max_freq_ratio, &mcnt) || !mcnt) + if (static_branch_unlikely(&arch_hybrid_cap_scale_key)) + freq_ratio = READ_ONCE(this_cpu_ptr(arch_cpu_scale)->freq_ratio); + else + freq_ratio = arch_max_freq_ratio; + + if (check_mul_overflow(mcnt, freq_ratio, &mcnt) || !mcnt) goto error; freq_scale = div64_u64(acnt, mcnt); From eb07797840c25dde7e976bd3e3a4c17d3d938495 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:40:43 +0100 Subject: [PATCH 129/188] cpufreq: Consolidate some code in cpufreq_online() commit 387b51709db546033cf9a940860b6614a5cda5b6 upstream. Notice that the policy->cpu update in cpufreq_policy_alloc() can be moved to cpufreq_online() and then it can be carried out under the policy rwsem, along with the clearing of policy->governor (unnecessary in the "new policy" code branch, but also not harmful). If this is done, the bottom parts of the "if (policy)" branches become identical and they can be collapsed and moved below the conditional. Modify the code accordingly which makes it somewhat easier to follow. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/13741234.uLZWGnKmhe@rjwysocki.net Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/cpufreq.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index effa9b1777170..5e225198435ac 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1340,7 +1340,6 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) init_waitqueue_head(&policy->transition_wait); INIT_WORK(&policy->update, handle_update); - policy->cpu = cpu; return policy; err_min_qos_notifier: @@ -1428,17 +1427,18 @@ static int cpufreq_online(unsigned int cpu) /* This is the only online CPU for the policy. Start over. */ new_policy = false; - down_write(&policy->rwsem); - policy->cpu = cpu; - policy->governor = NULL; } else { new_policy = true; policy = cpufreq_policy_alloc(cpu); if (!policy) return -ENOMEM; - down_write(&policy->rwsem); } + down_write(&policy->rwsem); + + policy->cpu = cpu; + policy->governor = NULL; + if (!new_policy && cpufreq_driver->online) { /* Recover policy->cpus using related_cpus */ cpumask_copy(policy->cpus, policy->related_cpus); From bf2546d428b6640b09fabf3be81c1f1d6699ab2f Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:41:49 +0100 Subject: [PATCH 130/188] cpufreq: Split cpufreq_online() commit 68974e3a15b92639396f39d1cd09d5288ba687bc upstream. In preparation for the introduction of cpufreq policy locking guards, move the part of cpufreq_online() that is carried out under the policy rwsem into a separate function called cpufreq_policy_online(). No intentional functional impact. [Backport Changes] 1. The upstream patch was auto-merged successfully. However, during diff review, some changes appeared due to line shifts in the file. These are not actual conflicts and therefore do not constitute a conflict-related deviation. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/3354747.aeNJFYEL58@rjwysocki.net Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/cpufreq.c | 95 ++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 5e225198435ac..e78eb3087515f 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1408,32 +1408,13 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) kfree(policy); } -static int cpufreq_online(unsigned int cpu) +static int cpufreq_policy_online(struct cpufreq_policy *policy, + unsigned int cpu, bool new_policy) { - struct cpufreq_policy *policy; - bool new_policy; unsigned long flags; unsigned int j; int ret; - pr_debug("%s: bringing CPU%u online\n", __func__, cpu); - - /* Check if this CPU already has a policy to manage it */ - policy = per_cpu(cpufreq_cpu_data, cpu); - if (policy) { - WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); - if (!policy_is_inactive(policy)) - return cpufreq_add_policy_cpu(policy, cpu); - - /* This is the only online CPU for the policy. Start over. */ - new_policy = false; - } else { - new_policy = true; - policy = cpufreq_policy_alloc(cpu); - if (!policy) - return -ENOMEM; - } - down_write(&policy->rwsem); policy->cpu = cpu; @@ -1460,7 +1441,7 @@ static int cpufreq_online(unsigned int cpu) if (ret) { pr_debug("%s: %d: initialization failed\n", __func__, __LINE__); - goto out_free_policy; + goto out_clear_policy; } /* @@ -1611,8 +1592,59 @@ static int cpufreq_online(unsigned int cpu) goto out_destroy_policy; } +out_unlock: up_write(&policy->rwsem); + return ret; + +out_destroy_policy: + for_each_cpu(j, policy->real_cpus) + remove_cpu_dev_symlink(policy, j, get_cpu_device(j)); + +out_offline_policy: + if (cpufreq_driver->offline) + cpufreq_driver->offline(policy); + +out_exit_policy: + if (cpufreq_driver->exit) + cpufreq_driver->exit(policy); + +out_clear_policy: + cpumask_clear(policy->cpus); + + goto out_unlock; +} + +static int cpufreq_online(unsigned int cpu) +{ + struct cpufreq_policy *policy; + bool new_policy; + int ret; + + pr_debug("%s: bringing CPU%u online\n", __func__, cpu); + + /* Check if this CPU already has a policy to manage it */ + policy = per_cpu(cpufreq_cpu_data, cpu); + if (policy) { + WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); + if (!policy_is_inactive(policy)) + return cpufreq_add_policy_cpu(policy, cpu); + + /* This is the only online CPU for the policy. Start over. */ + new_policy = false; + } else { + new_policy = true; + policy = cpufreq_policy_alloc(cpu); + if (!policy) + return -ENOMEM; + } + + ret = cpufreq_policy_online(policy, cpu, new_policy); + if (ret) { + cpufreq_policy_free(policy); + return ret; + } + kobject_uevent(&policy->kobj, KOBJ_ADD); /* Callback for handling stuff after policy is ready */ @@ -1639,25 +1671,6 @@ static int cpufreq_online(unsigned int cpu) pr_debug("initialization complete\n"); return 0; - -out_destroy_policy: - for_each_cpu(j, policy->real_cpus) - remove_cpu_dev_symlink(policy, j, get_cpu_device(j)); - -out_offline_policy: - if (cpufreq_driver->offline) - cpufreq_driver->offline(policy); - -out_exit_policy: - if (cpufreq_driver->exit) - cpufreq_driver->exit(policy); - -out_free_policy: - cpumask_clear(policy->cpus); - up_write(&policy->rwsem); - - cpufreq_policy_free(policy); - return ret; } /** From 3e5b0e0562cb8687874a50e4e82fede8ed2e8242 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:42:48 +0100 Subject: [PATCH 131/188] cpufreq: Add and use cpufreq policy locking guards commit 6fec833b9d70c54ceacbf7d07665215fbd0cddef upstream. Introduce "read" and "write" locking guards for cpufreq policies and use them where applicable in the cpufreq core. No intentional functional impact. [Backport Changes] 1. In drivers/cpufreq/cpufreq.c, cpufreq_get() was adapted to the base tree, which already carries commit c9b6238d15bc (upstream 908981d85f86, "cpufreq: Move the check of cpufreq_driver->get into cpufreq_verify_current_freq()"). That later upstream commit relocated the "if (cpufreq_driver->get)" NULL check out of cpufreq_get() and into cpufreq_verify_current_freq(). Since this patch is the earlier upstream change, its inline "if (cpufreq_driver->get)" guard was intentionally not reintroduced; cpufreq_get() calls __cpufreq_get() unconditionally under scoped_guard(), which matches both the base tree and final upstream. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/8518682.T7Z3S40VBb@rjwysocki.net Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/cpufreq.c | 121 +++++++++++++++++--------------------- include/linux/cpufreq.h | 6 ++ 2 files changed, 60 insertions(+), 67 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index e78eb3087515f..cf01ebee458b2 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1033,17 +1033,16 @@ static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) { struct cpufreq_policy *policy = to_policy(kobj); struct freq_attr *fattr = to_attr(attr); - ssize_t ret = -EBUSY; if (!fattr->show) return -EIO; - down_read(&policy->rwsem); + guard(cpufreq_policy_read)(policy); + if (likely(!policy_is_inactive(policy))) - ret = fattr->show(policy, buf); - up_read(&policy->rwsem); + return fattr->show(policy, buf); - return ret; + return -EBUSY; } static ssize_t store(struct kobject *kobj, struct attribute *attr, @@ -1051,17 +1050,16 @@ static ssize_t store(struct kobject *kobj, struct attribute *attr, { struct cpufreq_policy *policy = to_policy(kobj); struct freq_attr *fattr = to_attr(attr); - ssize_t ret = -EBUSY; if (!fattr->store) return -EIO; - down_write(&policy->rwsem); + guard(cpufreq_policy_write)(policy); + if (likely(!policy_is_inactive(policy))) - ret = fattr->store(policy, buf, count); - up_write(&policy->rwsem); + return fattr->store(policy, buf, count); - return ret; + return -EBUSY; } static void cpufreq_sysfs_release(struct kobject *kobj) @@ -1200,7 +1198,8 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cp if (cpumask_test_cpu(cpu, policy->cpus)) return 0; - down_write(&policy->rwsem); + guard(cpufreq_policy_write)(policy); + if (has_target()) cpufreq_stop_governor(policy); @@ -1211,7 +1210,7 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cp if (ret) pr_err("%s: Failed to start governor\n", __func__); } - up_write(&policy->rwsem); + return ret; } @@ -1231,9 +1230,10 @@ static void handle_update(struct work_struct *work) container_of(work, struct cpufreq_policy, update); pr_debug("handle_update for cpu %u called\n", policy->cpu); - down_write(&policy->rwsem); + + guard(cpufreq_policy_write)(policy); + refresh_frequency_limits(policy); - up_write(&policy->rwsem); } static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq, @@ -1259,11 +1259,11 @@ static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) struct kobject *kobj; struct completion *cmp; - down_write(&policy->rwsem); - cpufreq_stats_free_table(policy); - kobj = &policy->kobj; - cmp = &policy->kobj_unregister; - up_write(&policy->rwsem); + scoped_guard(cpufreq_policy_write, policy) { + cpufreq_stats_free_table(policy); + kobj = &policy->kobj; + cmp = &policy->kobj_unregister; + } kobject_put(kobj); /* @@ -1415,7 +1415,7 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, unsigned int j; int ret; - down_write(&policy->rwsem); + guard(cpufreq_policy_write)(policy); policy->cpu = cpu; policy->governor = NULL; @@ -1592,10 +1592,7 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, goto out_destroy_policy; } -out_unlock: - up_write(&policy->rwsem); - - return ret; + return 0; out_destroy_policy: for_each_cpu(j, policy->real_cpus) @@ -1612,7 +1609,7 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, out_clear_policy: cpumask_clear(policy->cpus); - goto out_unlock; + return ret; } static int cpufreq_online(unsigned int cpu) @@ -1759,11 +1756,10 @@ static int cpufreq_offline(unsigned int cpu) return 0; } - down_write(&policy->rwsem); + guard(cpufreq_policy_write)(policy); __cpufreq_offline(cpu, policy); - up_write(&policy->rwsem); return 0; } @@ -1780,33 +1776,29 @@ static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) if (!policy) return; - down_write(&policy->rwsem); + scoped_guard(cpufreq_policy_write, policy) { + if (cpu_online(cpu)) + __cpufreq_offline(cpu, policy); - if (cpu_online(cpu)) - __cpufreq_offline(cpu, policy); + remove_cpu_dev_symlink(policy, cpu, dev); - remove_cpu_dev_symlink(policy, cpu, dev); + if (!cpumask_empty(policy->real_cpus)) + return; - if (!cpumask_empty(policy->real_cpus)) { - up_write(&policy->rwsem); - return; - } + /* + * Unregister cpufreq cooling once all the CPUs of the policy + * are removed. + */ + if (cpufreq_thermal_control_enabled(cpufreq_driver)) { + cpufreq_cooling_unregister(policy->cdev); + policy->cdev = NULL; + } - /* - * Unregister cpufreq cooling once all the CPUs of the policy are - * removed. - */ - if (cpufreq_thermal_control_enabled(cpufreq_driver)) { - cpufreq_cooling_unregister(policy->cdev); - policy->cdev = NULL; + /* We did light-weight exit earlier, do full tear down now */ + if (cpufreq_driver->offline && cpufreq_driver->exit) + cpufreq_driver->exit(policy); } - /* We did light-weight exit earlier, do full tear down now */ - if (cpufreq_driver->offline && cpufreq_driver->exit) - cpufreq_driver->exit(policy); - - up_write(&policy->rwsem); - cpufreq_policy_free(policy); } @@ -1962,14 +1954,15 @@ unsigned int cpufreq_get(unsigned int cpu) struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); unsigned int ret_freq = 0; - if (policy) { - down_read(&policy->rwsem); - ret_freq = __cpufreq_get(policy); - up_read(&policy->rwsem); + if (!policy) + return 0; - cpufreq_cpu_put(policy); + scoped_guard(cpufreq_policy_read, policy) { + ret_freq = __cpufreq_get(policy); } + cpufreq_cpu_put(policy); + return ret_freq; } EXPORT_SYMBOL(cpufreq_get); @@ -2029,9 +2022,9 @@ void cpufreq_suspend(void) for_each_active_policy(policy) { if (has_target()) { - down_write(&policy->rwsem); - cpufreq_stop_governor(policy); - up_write(&policy->rwsem); + scoped_guard(cpufreq_policy_write, policy) { + cpufreq_stop_governor(policy); + } } if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy)) @@ -2072,9 +2065,9 @@ void cpufreq_resume(void) pr_err("%s: Failed to resume driver: %s\n", __func__, cpufreq_driver->name); } else if (has_target()) { - down_write(&policy->rwsem); - ret = cpufreq_start_governor(policy); - up_write(&policy->rwsem); + scoped_guard(cpufreq_policy_write, policy) { + ret = cpufreq_start_governor(policy); + } if (ret) pr_err("%s: Failed to start governor for CPU%u's policy\n", @@ -2442,15 +2435,9 @@ int cpufreq_driver_target(struct cpufreq_policy *policy, unsigned int target_freq, unsigned int relation) { - int ret; + guard(cpufreq_policy_write)(policy); - down_write(&policy->rwsem); - - ret = __cpufreq_driver_target(policy, target_freq, relation); - - up_write(&policy->rwsem); - - return ret; + return __cpufreq_driver_target(policy, target_freq, relation); } EXPORT_SYMBOL_GPL(cpufreq_driver_target); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 6cdc1820a7aa6..66708dcd0ad15 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -173,6 +173,12 @@ struct cpufreq_policy { struct notifier_block nb_max; }; +DEFINE_GUARD(cpufreq_policy_write, struct cpufreq_policy *, + down_write(&_T->rwsem), up_write(&_T->rwsem)) + +DEFINE_GUARD(cpufreq_policy_read, struct cpufreq_policy *, + down_read(&_T->rwsem), up_read(&_T->rwsem)) + /* * Used for passing new cpufreq policy data to the cpufreq driver's ->verify() * callback for sanitization. That callback is only expected to modify the min From 4f140b14d9a931d195b5d0b90a09f5dd68a1c968 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:43:46 +0100 Subject: [PATCH 132/188] cpufreq: intel_pstate: Rearrange max frequency updates handling code commit 973207ae3d7c3c92df4a382df5d7bd695deaa904 upstream. Rename __intel_pstate_update_max_freq() to intel_pstate_update_max_freq() and move the cpufreq policy reference counting and locking into it (and implement the locking with the recently introduced cpufreq policy "write" locking guard). No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Mario Limonciello Acked-by: Srinivas Pandruvada Link: https://patch.msgid.link/2315023.iZASKD2KPV@rjwysocki.net Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/intel_pstate.c | 52 +++++++++++----------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 98fd23ef0b08c..899c4caf30060 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1339,9 +1339,16 @@ static void intel_pstate_update_policies(void) cpufreq_update_policy(cpu); } -static void __intel_pstate_update_max_freq(struct cpudata *cpudata, - struct cpufreq_policy *policy) +static bool intel_pstate_update_max_freq(struct cpudata *cpudata) { + struct cpufreq_policy *policy __free(put_cpufreq_policy); + + policy = cpufreq_cpu_get(cpudata->cpu); + if (!policy) + return false; + + guard(cpufreq_policy_write)(policy); + if (hwp_active) intel_pstate_get_hwp_cap(cpudata); @@ -1349,44 +1356,24 @@ static void __intel_pstate_update_max_freq(struct cpudata *cpudata, cpudata->pstate.max_freq : cpudata->pstate.turbo_freq; refresh_frequency_limits(policy); + + return true; } static void intel_pstate_update_limits(unsigned int cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); - struct cpudata *cpudata; - - if (!policy) - return; - - cpudata = all_cpu_data[cpu]; - - __intel_pstate_update_max_freq(cpudata, policy); - - /* Prevent the driver from being unregistered now. */ - mutex_lock(&intel_pstate_driver_lock); + struct cpudata *cpudata = all_cpu_data[cpu]; - cpufreq_cpu_release(policy); - - hybrid_update_capacity(cpudata); - - mutex_unlock(&intel_pstate_driver_lock); + if (intel_pstate_update_max_freq(cpudata)) + hybrid_update_capacity(cpudata); } static void intel_pstate_update_limits_for_all(void) { int cpu; - for_each_possible_cpu(cpu) { - struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); - - if (!policy) - continue; - - __intel_pstate_update_max_freq(all_cpu_data[cpu], policy); - - cpufreq_cpu_release(policy); - } + for_each_possible_cpu(cpu) + intel_pstate_update_max_freq(all_cpu_data[cpu]); mutex_lock(&hybrid_capacity_lock); @@ -1828,13 +1815,8 @@ static void intel_pstate_notify_work(struct work_struct *work) { struct cpudata *cpudata = container_of(to_delayed_work(work), struct cpudata, hwp_notify_work); - struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpudata->cpu); - - if (policy) { - __intel_pstate_update_max_freq(cpudata, policy); - - cpufreq_cpu_release(policy); + if (intel_pstate_update_max_freq(cpudata)) { /* * The driver will not be unregistered while this function is * running, so update the capacity without acquiring the driver From 1ba46a17e668c45efe18c9d4cbcd203e7de71fa2 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Fri, 28 Mar 2025 21:48:54 +0100 Subject: [PATCH 133/188] cpufreq: Pass policy pointer to ->update_limits() commit eaff6b62d3439ca6ee00dba4f77673a8c37dac20 upstream. Since cpufreq_update_limits() obtains a cpufreq policy pointer for the given CPU and reference counts the corresponding policy object, it may as well pass the policy pointer to the cpufreq driver's ->update_limits() callback which allows that callback to avoid invoking cpufreq_cpu_get() for the same CPU. Accordingly, redefine ->update_limits() to take a policy pointer instead of a CPU number and update both drivers implementing it, intel_pstate and amd-pstate, as needed. Signed-off-by: Rafael J. Wysocki Acked-by: Viresh Kumar Reviewed-by: Mario Limonciello Acked-by: Srinivas Pandruvada Acked-by: Sudeep Holla Tested-by: Sudeep Holla Link: https://patch.msgid.link/8560367.NyiUUSuA9g@rjwysocki.net Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 7 ++----- drivers/cpufreq/cpufreq.c | 2 +- drivers/cpufreq/intel_pstate.c | 29 ++++++++++++++++++----------- include/linux/cpufreq.h | 2 +- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 58a6ac5a273d9..15644d011aad4 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -825,19 +825,16 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) schedule_work(&sched_prefcore_work); } -static void amd_pstate_update_limits(unsigned int cpu) +static void amd_pstate_update_limits(struct cpufreq_policy *policy) { - struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); struct amd_cpudata *cpudata; u32 prev_high = 0, cur_high = 0; bool highest_perf_changed = false; + unsigned int cpu = policy->cpu; if (!amd_pstate_prefcore) return; - if (!policy) - return; - if (amd_get_highest_perf(cpu, &cur_high)) return; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index cf01ebee458b2..d874bb230c1a8 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2797,7 +2797,7 @@ void cpufreq_update_limits(unsigned int cpu) return; if (cpufreq_driver->update_limits) - cpufreq_driver->update_limits(cpu); + cpufreq_driver->update_limits(policy); else cpufreq_update_policy(cpu); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 899c4caf30060..b26f8c3c3e00c 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1339,14 +1339,9 @@ static void intel_pstate_update_policies(void) cpufreq_update_policy(cpu); } -static bool intel_pstate_update_max_freq(struct cpudata *cpudata) +static void __intel_pstate_update_max_freq(struct cpufreq_policy *policy, + struct cpudata *cpudata) { - struct cpufreq_policy *policy __free(put_cpufreq_policy); - - policy = cpufreq_cpu_get(cpudata->cpu); - if (!policy) - return false; - guard(cpufreq_policy_write)(policy); if (hwp_active) @@ -1356,16 +1351,28 @@ static bool intel_pstate_update_max_freq(struct cpudata *cpudata) cpudata->pstate.max_freq : cpudata->pstate.turbo_freq; refresh_frequency_limits(policy); +} + +static bool intel_pstate_update_max_freq(struct cpudata *cpudata) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy); + + policy = cpufreq_cpu_get(cpudata->cpu); + if (!policy) + return false; + + __intel_pstate_update_max_freq(policy, cpudata); return true; } -static void intel_pstate_update_limits(unsigned int cpu) +static void intel_pstate_update_limits(struct cpufreq_policy *policy) { - struct cpudata *cpudata = all_cpu_data[cpu]; + struct cpudata *cpudata = all_cpu_data[policy->cpu]; - if (intel_pstate_update_max_freq(cpudata)) - hybrid_update_capacity(cpudata); + __intel_pstate_update_max_freq(policy, cpudata); + + hybrid_update_capacity(cpudata); } static void intel_pstate_update_limits_for_all(void) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 66708dcd0ad15..a0d52195a52d4 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -393,7 +393,7 @@ struct cpufreq_driver { unsigned int (*get)(unsigned int cpu); /* Called to update policy limits on firmware notifications. */ - void (*update_limits)(unsigned int cpu); + void (*update_limits)(struct cpufreq_policy *policy); /* optional */ int (*bios_limit)(int cpu, unsigned int *limit); From 7a0718fe1b726aa6f232a41412cd4dbbacf1d551 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Fri, 11 Apr 2025 08:14:39 +0000 Subject: [PATCH 134/188] cpufreq/amd-pstate: Enable ITMT support after initializing core rankings commit d87e4026d1b20e4f237b29e0c956ad415f533de2 upstream. When working on dynamic ITMT priority support, it was observed that "asym_prefer_cpu" on AMD systems supporting Preferred Core ranking was always set to the first CPU in the sched group when the system boots up despite another CPU in the group having a higher ITMT ranking. "asym_prefer_cpu" is cached when the sched domain hierarchy is constructed. On AMD systems that support Preferred Core rankings, sched domains are rebuilt when ITMT support is enabled for the first time from amd_pstate*_cpu_init(). Since amd_pstate*_cpu_init() is called to initialize the cpudata for each CPU, the ITMT support is enabled after the first CPU initializes its asym priority but this is too early since other CPUs have not yet initialized their asym priorities and the sched domain is rebuilt only once when the support is toggled on for the first time. Initialize the asym priorities first in amd_pstate*_cpu_init() and then enable ITMT support later in amd_pstate_register_driver() to ensure all CPUs have correctly initialized their asym priorities before sched domain hierarchy is rebuilt. Clear the ITMT support when the amd-pstate driver unregisters since core rankings cannot be trusted unless the update_limits() callback is operational. Remove the delayed work mechanism now that ITMT support is only toggled from the driver init path which is outside the cpuhp critical section. Fixes: f3a052391822 ("cpufreq: amd-pstate: Enable amd-pstate preferred core support") Signed-off-by: K Prateek Nayak Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20250411081439.27652-1-kprateek.nayak@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 15644d011aad4..5fcc8262148a0 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -795,16 +795,6 @@ static void amd_perf_ctl_reset(unsigned int cpu) wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); } -/* - * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks - * due to locking, so queue the work for later. - */ -static void amd_pstste_sched_prefcore_workfn(struct work_struct *work) -{ - sched_set_itmt_support(); -} -static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn); - #define CPPC_MAX_PERF U8_MAX static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) @@ -815,14 +805,8 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) cpudata->hw_prefcore = true; - /* - * The priorities can be set regardless of whether or not - * sched_set_itmt_support(true) has been called and it is valid to - * update them at any time after it has been called. - */ + /* Priorities must be initialized before ITMT support can be toggled on. */ sched_set_itmt_core_prio((int)READ_ONCE(cpudata->prefcore_ranking), cpudata->cpu); - - schedule_work(&sched_prefcore_work); } static void amd_pstate_update_limits(struct cpufreq_policy *policy) @@ -1190,6 +1174,9 @@ static ssize_t show_energy_performance_preference( static void amd_pstate_driver_cleanup(void) { + if (amd_pstate_prefcore) + sched_clear_itmt_support(); + cppc_state = AMD_PSTATE_DISABLE; current_pstate_driver = NULL; } @@ -1232,6 +1219,10 @@ static int amd_pstate_register_driver(int mode) return ret; } + /* Enable ITMT support once all CPUs have initialized their asym priorities. */ + if (amd_pstate_prefcore) + sched_set_itmt_support(); + return 0; } From a477f12a0c4ca28c29c3b81170e5658beae01215 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Wed, 9 Apr 2025 05:34:44 +0000 Subject: [PATCH 135/188] sched/topology: Introduce sched_update_asym_prefer_cpu() commit 0e3f6c3696424fa90d6f512779d617a05a1cf031 upstream. A subset of AMD Processors supporting Preferred Core Rankings also feature the ability to dynamically switch these rankings at runtime to bias load balancing towards or away from the LLC domain with larger cache. To support dynamically updating "sg->asym_prefer_cpu" without needing to rebuild the sched domain, introduce sched_update_asym_prefer_cpu() which recomutes the "asym_prefer_cpu" when the core-ranking of a CPU changes. sched_update_asym_prefer_cpu() swaps the "sg->asym_prefer_cpu" with the CPU whose ranking has changed if the new ranking is greater than that of the "asym_prefer_cpu". If CPU whose ranking has changed is the current "asym_prefer_cpu", it scans the CPUs of the sched groups to find the new "asym_prefer_cpu" and sets it accordingly. get_group() for non-overlapping sched domains returns the sched group for the first CPU in the sched_group_span() which ensures all CPUs in the group see the updated value of "asym_prefer_cpu". Overlapping groups are allocated differently and will require moving the "asym_prefer_cpu" to "sg->sgc" but since the current implementations do not set "SD_ASYM_PACKING" at NUMA domains, skip additional indirection and place a SCHED_WARN_ON() to alert any future users. Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20250409053446.23367-3-kprateek.nayak@amd.com Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- include/linux/sched/topology.h | 6 ++++ kernel/sched/topology.c | 58 ++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index dda055d66fb53..e480493243837 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -208,6 +208,8 @@ struct sched_domain_topology_level { }; extern void __init set_sched_topology(struct sched_domain_topology_level *tl); +extern void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio); + #ifdef CONFIG_SCHED_DEBUG # define SD_INIT_NAME(type) .name = #type @@ -241,6 +243,10 @@ static inline bool cpus_share_resources(int this_cpu, int that_cpu) return true; } +static inline void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio) +{ +} + #endif /* !CONFIG_SMP */ #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 0b398c2387735..7aaa25f15fabd 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1327,6 +1327,64 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) update_group_capacity(sd, cpu); } +#ifdef CONFIG_SMP + +/* Update the "asym_prefer_cpu" when arch_asym_cpu_priority() changes. */ +void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio) +{ + int asym_prefer_cpu = cpu; + struct sched_domain *sd; + + guard(rcu)(); + + for_each_domain(cpu, sd) { + struct sched_group *sg; + int group_cpu; + + if (!(sd->flags & SD_ASYM_PACKING)) + continue; + + /* + * Groups of overlapping domain are replicated per NUMA + * node and will require updating "asym_prefer_cpu" on + * each local copy. + * + * If you are hitting this warning, consider moving + * "sg->asym_prefer_cpu" to "sg->sgc->asym_prefer_cpu" + * which is shared by all the overlapping groups. + */ + WARN_ON_ONCE(sd->flags & SD_OVERLAP); + + sg = sd->groups; + if (cpu != sg->asym_prefer_cpu) { + /* + * Since the parent is a superset of the current group, + * if the cpu is not the "asym_prefer_cpu" at the + * current level, it cannot be the preferred CPU at a + * higher levels either. + */ + if (!sched_asym_prefer(cpu, sg->asym_prefer_cpu)) + return; + + WRITE_ONCE(sg->asym_prefer_cpu, cpu); + continue; + } + + /* Ranking has improved; CPU is still the preferred one. */ + if (new_prio >= old_prio) + continue; + + for_each_cpu(group_cpu, sched_group_span(sg)) { + if (sched_asym_prefer(group_cpu, asym_prefer_cpu)) + asym_prefer_cpu = group_cpu; + } + + WRITE_ONCE(sg->asym_prefer_cpu, asym_prefer_cpu); + } +} + +#endif /* CONFIG_SMP */ + /* * Asymmetric CPU capacity bits */ From edbe38d5faeef1b742b5bdeef7ea68d0e7621a9c Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Wed, 9 Apr 2025 05:34:45 +0000 Subject: [PATCH 136/188] cpufreq/amd-pstate: Update asym_prefer_cpu when core rankings change commit 8157fbc907452aa5674df2de23c1c7305c907006 upstream. A subset of AMD systems supporting Preferred Core rankings can have their rankings changed dynamically at runtime. Update the "sg->asym_prefer_cpu" across the local hierarchy of CPU when the preferred core ranking changes. Signed-off-by: K Prateek Nayak Signed-off-by: Peter Zijlstra (Intel) Acked-by: Mario Limonciello Link: https://lore.kernel.org/r/20250409053446.23367-4-kprateek.nayak@amd.com Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 5fcc8262148a0..cbba0086eb851 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -829,8 +829,10 @@ static void amd_pstate_update_limits(struct cpufreq_policy *policy) if (highest_perf_changed) { WRITE_ONCE(cpudata->prefcore_ranking, cur_high); - if (cur_high < CPPC_MAX_PERF) + if (cur_high < CPPC_MAX_PERF) { sched_set_itmt_core_prio((int)cur_high, cpu); + sched_update_asym_prefer_cpu(cpu, prev_high, cur_high); + } } } From f4799cd2bcfe18297a186e1197932bdde754b11e Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Tue, 15 Apr 2025 08:23:09 +0000 Subject: [PATCH 137/188] cpufreq/amd-pstate: Move max_perf limiting in amd_pstate_update commit afc9506c2a8169e871cfafd9b94e0e631ae8f9cb upstream. Move up the max_perf limiting, so that we clamp the des_perf with the updated max_perf. Signed-off-by: Dhananjay Ugwekar Link: https://lore.kernel.org/r/20250415082308.3341-1-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index cbba0086eb851..a82fd6f23efe4 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -555,6 +555,10 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, if (!policy) return; + /* limit the max perf when core performance boost feature is disabled */ + if (!cpudata->boost_supported) + max_perf = min_t(u8, perf.nominal_perf, max_perf); + des_perf = clamp_t(u8, des_perf, min_perf, max_perf); policy->cur = perf_to_freq(perf, cpudata->nominal_freq, des_perf); @@ -564,10 +568,6 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, des_perf = 0; } - /* limit the max perf when core performance boost feature is disabled */ - if (!cpudata->boost_supported) - max_perf = min_t(u8, perf.nominal_perf, max_perf); - if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, From cba08a915262675787d0acba99dd911a08923f65 Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Mon, 28 Apr 2025 06:25:21 +0000 Subject: [PATCH 138/188] cpufreq/amd-pstate: Add offline, online and suspend callbacks for amd_pstate_driver commit 98b52c6b0b528764b454350a11c53f757304afbc upstream. Rename and use the existing amd_pstate_epp callbacks for amd_pstate driver as well. Remove the debug print in online callback while at it. These callbacks will be needed to support the "Requested CPU Min Frequency" BIOS option. Reviewed-by: Mario Limonciello Signed-off-by: Dhananjay Ugwekar Link: https://lore.kernel.org/r/20250428062520.4997-2-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index a82fd6f23efe4..c18d505befb29 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1555,19 +1555,17 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) return 0; } -static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) +static int amd_pstate_cpu_online(struct cpufreq_policy *policy) { - pr_debug("AMD CPU Core %d going online\n", policy->cpu); - return amd_pstate_cppc_enable(policy); } -static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) +static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) { return 0; } -static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) +static int amd_pstate_suspend(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; @@ -1605,6 +1603,9 @@ static struct cpufreq_driver amd_pstate_driver = { .fast_switch = amd_pstate_fast_switch, .init = amd_pstate_cpu_init, .exit = amd_pstate_cpu_exit, + .online = amd_pstate_cpu_online, + .offline = amd_pstate_cpu_offline, + .suspend = amd_pstate_suspend, .set_boost = amd_pstate_set_boost, .update_limits = amd_pstate_update_limits, .name = "amd-pstate", @@ -1617,9 +1618,9 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .setpolicy = amd_pstate_epp_set_policy, .init = amd_pstate_epp_cpu_init, .exit = amd_pstate_epp_cpu_exit, - .offline = amd_pstate_epp_cpu_offline, - .online = amd_pstate_epp_cpu_online, - .suspend = amd_pstate_epp_suspend, + .offline = amd_pstate_cpu_offline, + .online = amd_pstate_cpu_online, + .suspend = amd_pstate_suspend, .resume = amd_pstate_epp_resume, .update_limits = amd_pstate_update_limits, .set_boost = amd_pstate_set_boost, From 5a9a4fd7788e18d8e1a4f93fd5dc1dd3478c2b2d Mon Sep 17 00:00:00 2001 From: Dhananjay Ugwekar Date: Mon, 28 Apr 2025 07:16:24 +0000 Subject: [PATCH 139/188] cpufreq/amd-pstate: Add support for the "Requested CPU Min frequency" BIOS option commit 608a76b65288698677843d2f22418e03391ef95f upstream. Initialize lower frequency limit to the "Requested CPU Min frequency" BIOS option (if it is set) value as part of the driver->init() callback. The BIOS specified value is passed by the PMFW as min_perf in CPPC_REQ MSR. To ensure that we don't mistake a stale min_perf value in CPPC_REQ value as the "Requested CPU Min frequency" during a kexec wakeup, reset the CPPC_REQ.min_perf value back to the BIOS specified one in the offline, exit and suspend callbacks. amd_pstate_target() and amd_pstate_epp_update_limit() which are invoked as part of the resume() and online() callbacks will take care of restoring the CPPC_REQ back to the correct values. Signed-off-by: Dhananjay Ugwekar Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20250428071623.4309-1-dhananjay.ugwekar@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 82 ++++++++++++++++++++++++++++++------ drivers/cpufreq/amd-pstate.h | 2 + 2 files changed, 71 insertions(+), 13 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index c18d505befb29..a096c4a6abcdd 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -390,7 +390,8 @@ static inline int amd_pstate_cppc_enable(struct cpufreq_policy *policy) static int msr_init_perf(struct amd_cpudata *cpudata) { union perf_cached perf = READ_ONCE(cpudata->perf); - u64 cap1, numerator; + u64 cap1, numerator, cppc_req; + u8 min_perf; int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &cap1); @@ -401,6 +402,22 @@ static int msr_init_perf(struct amd_cpudata *cpudata) if (ret) return ret; + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &cppc_req); + if (ret) + return ret; + + WRITE_ONCE(cpudata->cppc_req_cached, cppc_req); + min_perf = FIELD_GET(AMD_CPPC_MIN_PERF_MASK, cppc_req); + + /* + * Clear out the min_perf part to check if the rest of the MSR is 0, if yes, this is an + * indication that the min_perf value is the one specified through the BIOS option + */ + cppc_req &= ~(AMD_CPPC_MIN_PERF_MASK); + + if (!cppc_req) + perf.bios_min_perf = min_perf; + perf.highest_perf = numerator; perf.max_limit_perf = numerator; perf.min_limit_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); @@ -581,20 +598,26 @@ static int amd_pstate_verify(struct cpufreq_policy_data *policy_data) { /* * Initialize lower frequency limit (i.e.policy->min) with - * lowest_nonlinear_frequency which is the most energy efficient - * frequency. Override the initial value set by cpufreq core and - * amd-pstate qos_requests. + * lowest_nonlinear_frequency or the min frequency (if) specified in BIOS, + * Override the initial value set by cpufreq core and amd-pstate qos_requests. */ if (policy_data->min == FREQ_QOS_MIN_DEFAULT_VALUE) { struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(policy_data->cpu); struct amd_cpudata *cpudata; + union perf_cached perf; if (!policy) return -EINVAL; cpudata = policy->driver_data; - policy_data->min = cpudata->lowest_nonlinear_freq; + perf = READ_ONCE(cpudata->perf); + + if (perf.bios_min_perf) + policy_data->min = perf_to_freq(perf, cpudata->nominal_freq, + perf.bios_min_perf); + else + policy_data->min = cpudata->lowest_nonlinear_freq; } cpufreq_verify_within_cpu_limits(policy_data); @@ -1020,6 +1043,10 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + + /* Reset CPPC_REQ MSR to the BIOS value */ + amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); freq_qos_remove_request(&cpudata->req[1]); freq_qos_remove_request(&cpudata->req[0]); @@ -1415,7 +1442,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) struct amd_cpudata *cpudata; union perf_cached perf; struct device *dev; - u64 value; int ret; /* @@ -1480,12 +1506,6 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) cpudata->epp_default = AMD_CPPC_EPP_BALANCE_PERFORMANCE; } - if (cpu_feature_enabled(X86_FEATURE_CPPC)) { - ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); - if (ret) - return ret; - WRITE_ONCE(cpudata->cppc_req_cached, value); - } ret = amd_pstate_set_epp(policy, cpudata->epp_default); if (ret) return ret; @@ -1505,6 +1525,11 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) struct amd_cpudata *cpudata = policy->driver_data; if (cpudata) { + union perf_cached perf = READ_ONCE(cpudata->perf); + + /* Reset CPPC_REQ MSR to the BIOS value */ + amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); + kfree(cpudata); policy->driver_data = NULL; } @@ -1562,12 +1587,31 @@ static int amd_pstate_cpu_online(struct cpufreq_policy *policy) static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) { - return 0; + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + + /* + * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified + * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the + * limits, epp and desired perf will get reset to the cached values in cpudata struct + */ + return amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); } static int amd_pstate_suspend(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + int ret; + + /* + * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified + * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, + * the limits, epp and desired perf will get reset to the cached values in cpudata struct + */ + ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); + if (ret) + return ret; /* invalidate to ensure it's rewritten during resume */ cpudata->cppc_req_cached = 0; @@ -1578,6 +1622,17 @@ static int amd_pstate_suspend(struct cpufreq_policy *policy) return 0; } +static int amd_pstate_resume(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur); + + /* Set CPPC_REQ to last sane value until the governor updates it */ + return amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, + 0U, false); +} + static int amd_pstate_epp_resume(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; @@ -1606,6 +1661,7 @@ static struct cpufreq_driver amd_pstate_driver = { .online = amd_pstate_cpu_online, .offline = amd_pstate_cpu_offline, .suspend = amd_pstate_suspend, + .resume = amd_pstate_resume, .set_boost = amd_pstate_set_boost, .update_limits = amd_pstate_update_limits, .name = "amd-pstate", diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index fbe1c08d3f061..2f7ae364d3313 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -30,6 +30,7 @@ * @lowest_perf: the absolute lowest performance level of the processor * @min_limit_perf: Cached value of the performance corresponding to policy->min * @max_limit_perf: Cached value of the performance corresponding to policy->max + * @bios_min_perf: Cached perf value corresponding to the "Requested CPU Min Frequency" BIOS option */ union perf_cached { struct { @@ -39,6 +40,7 @@ union perf_cached { u8 lowest_perf; u8 min_limit_perf; u8 max_limit_perf; + u8 bios_min_perf; }; u64 val; }; From fe9910237d3e849f9e6e0b00930f111a1bd4349b Mon Sep 17 00:00:00 2001 From: Swapnil Sapkal Date: Wed, 30 Apr 2025 06:42:06 +0000 Subject: [PATCH 140/188] amd-pstate-ut: Reset amd-pstate driver mode after running selftests commit d26d16438bc5fd5524121244cc133f9872a63210 upstream. In amd-pstate-ut, one of the basic test is to switch between all possible mode combinations. After running this test the mode of the amd-pstate driver is active mode. Store and reset the mode to its original state. Reviewed-by: Gautham R. Shenoy Signed-off-by: Swapnil Sapkal Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20250430064206.7402-1-swapnil.sapkal@amd.com Signed-off-by: Mario Limonciello Signed-off-by: Mukesh Ogare Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 19 ++++++++++++------- drivers/cpufreq/amd-pstate.c | 6 ++++++ drivers/cpufreq/amd-pstate.h | 1 + 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index e671bc7d15508..30835d0e49940 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -242,6 +242,8 @@ static int amd_pstate_set_mode(enum amd_pstate_mode mode) static int amd_pstate_ut_check_driver(u32 index) { enum amd_pstate_mode mode1, mode2 = AMD_PSTATE_DISABLE; + enum amd_pstate_mode orig_mode = amd_pstate_get_status(); + int ret; for (mode1 = AMD_PSTATE_DISABLE; mode1 < AMD_PSTATE_MAX; mode1++) { int ret = amd_pstate_set_mode(mode1); @@ -251,16 +253,19 @@ static int amd_pstate_ut_check_driver(u32 index) if (mode1 == mode2) continue; ret = amd_pstate_set_mode(mode2); - if (ret) { - pr_err("%s: failed to update status for %s->%s\n", __func__, - amd_pstate_get_mode_string(mode1), - amd_pstate_get_mode_string(mode2)); - return ret; - } + if (ret) + goto out; } } - return 0; +out: + if (ret) + pr_warn("%s: failed to update status for %s->%s: %d\n", __func__, + amd_pstate_get_mode_string(mode1), + amd_pstate_get_mode_string(mode2), ret); + + amd_pstate_set_mode(orig_mode); + return ret; } static int __init amd_pstate_ut_init(void) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index a096c4a6abcdd..a76d12a8abf3b 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1328,6 +1328,12 @@ static ssize_t amd_pstate_show_status(char *buf) return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); } +int amd_pstate_get_status(void) +{ + return cppc_state; +} +EXPORT_SYMBOL_GPL(amd_pstate_get_status); + int amd_pstate_update_status(const char *buf, size_t size) { int mode_idx; diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 2f7ae364d3313..cb45fdca27a6c 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -121,6 +121,7 @@ enum amd_pstate_mode { AMD_PSTATE_MAX, }; const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode); +int amd_pstate_get_status(void); int amd_pstate_update_status(const char *buf, size_t size); #endif /* _LINUX_AMD_PSTATE_H */ From 5ed1a9b70d4a36a12ded4e2a3e1c3d55f747696c Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 12 May 2025 23:28:55 +0200 Subject: [PATCH 141/188] cpufreq/amd-pstate: Avoid shadowing ret in amd_pstate_ut_check_driver() commit f6b1eebbdbc75377f98fc9774bb86ffc270dae8e upstream. Clang warns (or errors with CONFIG_WERROR=y): drivers/cpufreq/amd-pstate-ut.c:262:6: error: variable 'ret' is uninitialized when used here [-Werror,-Wuninitialized] 262 | if (ret) | ^~~ ret is declared at the top of the function and in the for loop so the initialization of ret is local to the loop. Remove the declaration in the for loop so that ret is always used initialized. Fixes: d26d16438bc5 ("amd-pstate-ut: Reset amd-pstate driver mode after running selftests") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202505072226.g2QclhaR-lkp@intel.com/ Signed-off-by: Nathan Chancellor Acked-by: Mario Limonciello Link: https://lore.kernel.org/r/20250512-amd-pstate-ut-uninit-ret-v1-1-fcb4104f502e@kernel.org Signed-off-by: Mario Limonciello Signed-off-by: priyanka-mani Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 30835d0e49940..65f9d2bae2d36 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -246,7 +246,7 @@ static int amd_pstate_ut_check_driver(u32 index) int ret; for (mode1 = AMD_PSTATE_DISABLE; mode1 < AMD_PSTATE_MAX; mode1++) { - int ret = amd_pstate_set_mode(mode1); + ret = amd_pstate_set_mode(mode1); if (ret) return ret; for (mode2 = AMD_PSTATE_DISABLE; mode2 < AMD_PSTATE_MAX; mode2++) { From 3b6db3d199940594f3442e9f593a48de516463d5 Mon Sep 17 00:00:00 2001 From: Perry Yuan Date: Sun, 27 Oct 2024 21:02:41 -0500 Subject: [PATCH 142/188] x86/cpufeatures: Add X86_FEATURE_AMD_WORKLOAD_CLASS feature bit commit 0c487010cb4f79e451ac9e7cc47494cb21ac3566 upstream. Add a new feature bit that indicates support for workload-based heuristic feedback to OS for scheduling decisions. When the bit set, threads are classified during runtime into enumerated classes. The classes represent thread performance/power characteristics that may benefit from special scheduling behaviors. Signed-off-by: Perry Yuan Signed-off-by: Mario Limonciello Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20241028020251.8085-4-mario.limonciello@amd.com Signed-off-by: priyanka-mani Signed-off-by: mohanasv2 --- arch/x86/include/asm/cpufeatures.h | 3 ++- arch/x86/kernel/cpu/scattered.c | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 1089c56c647fc..29d8d0fdece84 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -475,7 +475,8 @@ #define X86_FEATURE_BHI_CTRL (21*32+ 2) /* BHI_DIS_S HW control available */ #define X86_FEATURE_CLEAR_BHB_HW (21*32+ 3) /* BHI_DIS_S HW control enabled */ #define X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT (21*32+ 4) /* Clear branch history at vmexit using SW loop */ -#define X86_FEATURE_INDIRECT_THUNK_ITS (21*32 + 5) /* "" Use thunk for indirect branches in lower half of cacheline */ +#define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32 + 7) /* Workload Classification */ +#define X86_FEATURE_INDIRECT_THUNK_ITS (21*32 + 9) /* "" Use thunk for indirect branches in lower half of cacheline */ #define X86_FEATURE_TSA_SQ_NO (21*32+11) /* "" AMD CPU not vulnerable to TSA-SQ */ #define X86_FEATURE_TSA_L1_NO (21*32+12) /* "" AMD CPU not vulnerable to TSA-L1 */ diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 8fad19ec85551..a1eabee0fb6da 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -49,6 +49,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 }, { X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 }, { X86_FEATURE_BMEC, CPUID_EBX, 3, 0x80000020, 0 }, + { X86_FEATURE_AMD_WORKLOAD_CLASS, CPUID_EAX, 22, 0x80000021, 0 }, { X86_FEATURE_TSA_SQ_NO, CPUID_ECX, 1, 0x80000021, 0 }, { X86_FEATURE_TSA_L1_NO, CPUID_ECX, 2, 0x80000021, 0 }, { X86_FEATURE_PERFMON_V2, CPUID_EAX, 0, 0x80000022, 0 }, From 1b21c6eda4f82e48cdf9d6ae2d1a4bcb00946985 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 Jun 2025 15:05:15 -0500 Subject: [PATCH 143/188] cpufreq/amd-pstate: Disable preferred cores on designs with workload classification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit bfea2b3b4f2346510ec45a0c22450e1564f48f88 upstream. On designs that have workload classification, it's preferred that the amd-hfi driver is used to provide hints to the scheduler of which cores to use instead of the amd-pstate driver. Signed-off-by: Mario Limonciello Signed-off-by: Borislav Petkov (AMD) Reviewed-by: Gautham R. Shenoy Reviewed-by: Perry Yuan Acked-by: Ilpo Järvinen Link: https://lore.kernel.org/20250609200518.3616080-11-superm1@kernel.org Signed-off-by: priyanka-mani Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index a76d12a8abf3b..f8ec3bcff8491 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -826,6 +826,13 @@ static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) if (!amd_pstate_prefcore) return; + /* should use amd-hfi instead */ + if (cpu_feature_enabled(X86_FEATURE_AMD_WORKLOAD_CLASS) && + IS_ENABLED(CONFIG_AMD_HFI)) { + amd_pstate_prefcore = false; + return; + } + cpudata->hw_prefcore = true; /* Priorities must be initialized before ITMT support can be toggled on. */ From f46be797326d00e1307aadc4ad949cd2b3dd8ed3 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:45 +0530 Subject: [PATCH 144/188] amd-pstate: Fix memory leak in amd_pstate_epp_cpu_init() commit beda3b363546a423e4e29a7395e04c0ac4ff677e upstream. On failure to set the epp, the function amd_pstate_epp_cpu_init() returns with an error code without freeing the cpudata object that was allocated at the beginning of the function. Ensure that the cpudata object is freed before returning from the function. This memory leak was discovered by Claude Opus 4.6 with the aid of Chris Mason's AI review-prompts (https://github.com/masoncl/review-prompts/tree/main/kernel). Assisted-by: Claude:claude-opus-4.6 review-prompts/linux Fixes: f9a378ff6443 ("cpufreq/amd-pstate: Set different default EPP policy for Epyc and Ryzen") Reviewed-by: Mario Limonciello (AMD) Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: priyanka-mani Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index f8ec3bcff8491..051474a3c4045 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1521,7 +1521,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) ret = amd_pstate_set_epp(policy, cpudata->epp_default); if (ret) - return ret; + goto free_cpudata1; current_pstate_driver->adjust_perf = NULL; From 8c90af599318ae8f8c57c248423a683ce7a6720e Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:46 +0530 Subject: [PATCH 145/188] amd-pstate: Update cppc_req_cached in fast_switch case commit fcc25a291fbdca2c06c2c6602532050873f0c9de upstream. The function msr_update_perf() does not cache the new value that is written to MSR_AMD_CPPC_REQ into the variable cpudata->cppc_req_cached when the update is happening from the fast path. Fix that by caching the value everytime the MSR_AMD_CPPC_REQ gets updated. This issue was discovered by Claude Opus 4.6 with the aid of Chris Mason's AI review-prompts (https://github.com/masoncl/review-prompts/tree/main/kernel). Assisted-by: Claude:claude-opus-4.6 review-prompts/linux Reviewed-by: Mario Limonciello (AMD) Fixes: fff395796917 ("cpufreq/amd-pstate: Always write EPP value when updating perf") Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: priyanka-mani Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 051474a3c4045..cd6d8d95b79db 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -260,7 +260,6 @@ static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, if (fast_switch) { wrmsrl(MSR_AMD_CPPC_REQ, value); - return 0; } else { int ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); From 300b44858278a3f19a1f83d384d8678ef7e0b8ca Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:47 +0530 Subject: [PATCH 146/188] amd-pstate: Make certain freq_attrs conditionally visible commit e67a5b6541831bbf1c40b6042a867a4594ec6b55 upstream. Certain amd_pstate freq_attrs such as amd_pstate_hw_prefcore and amd_pstate_prefcore_ranking are enabled even when preferred core is not supported on the platform. Similarly there are common freq_attrs between the amd-pstate and the amd-pstate-epp drivers (eg: amd_pstate_max_freq, amd_pstate_lowest_nonlinear_freq, etc.) but are duplicated in two different freq_attr structs. Unify all the attributes in a single place and associate each of them with a visibility function that determines whether the attribute should be visible based on the underlying platform support and the current amd_pstate mode. Reviewed-by: Mario Limonciello (AMD) Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: priyanka-mani Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 124 ++++++++++++++++++++++++++--------- 1 file changed, 93 insertions(+), 31 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index cd6d8d95b79db..d38df671da53c 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1207,12 +1207,87 @@ static ssize_t show_energy_performance_preference( return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); } +cpufreq_freq_attr_ro(amd_pstate_max_freq); +cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); + +cpufreq_freq_attr_ro(amd_pstate_highest_perf); +cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); +cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); +cpufreq_freq_attr_rw(energy_performance_preference); +cpufreq_freq_attr_ro(energy_performance_available_preferences); + +struct freq_attr_visibility { + struct freq_attr *attr; + bool (*visibility_fn)(void); +}; + +/* For attributes which are always visible */ +static bool always_visible(void) +{ + return true; +} + +/* Determines whether prefcore related attributes should be visible */ +static bool prefcore_visibility(void) +{ + return amd_pstate_prefcore; +} + +/* Determines whether energy performance preference should be visible */ +static bool epp_visibility(void) +{ + return cppc_state == AMD_PSTATE_ACTIVE; +} + +static struct freq_attr_visibility amd_pstate_attr_visibility[] = { + {&amd_pstate_max_freq, always_visible}, + {&amd_pstate_lowest_nonlinear_freq, always_visible}, + {&amd_pstate_highest_perf, always_visible}, + {&amd_pstate_prefcore_ranking, prefcore_visibility}, + {&amd_pstate_hw_prefcore, prefcore_visibility}, + {&energy_performance_preference, epp_visibility}, + {&energy_performance_available_preferences, epp_visibility}, +}; + +static struct freq_attr **get_freq_attrs(void) +{ + bool attr_visible[ARRAY_SIZE(amd_pstate_attr_visibility)]; + struct freq_attr **attrs; + int i, j, count; + + for (i = 0, count = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { + struct freq_attr_visibility *v = &amd_pstate_attr_visibility[i]; + + attr_visible[i] = v->visibility_fn(); + if (attr_visible[i]) + count++; + } + + /* amd_pstate_{max_freq, lowest_nonlinear_freq, highest_perf} should always be visible */ + BUG_ON(!count); + + attrs = kcalloc(count + 1, sizeof(struct freq_attr *), GFP_KERNEL); + if (!attrs) + return ERR_PTR(-ENOMEM); + + for (i = 0, j = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { + if (!attr_visible[i]) + continue; + + attrs[j++] = amd_pstate_attr_visibility[i].attr; + } + + return attrs; +} + static void amd_pstate_driver_cleanup(void) { if (amd_pstate_prefcore) sched_clear_itmt_support(); cppc_state = AMD_PSTATE_DISABLE; + kfree(current_pstate_driver->attr); + current_pstate_driver->attr = NULL; current_pstate_driver = NULL; } @@ -1237,6 +1312,7 @@ static int amd_pstate_set_driver(int mode_idx) static int amd_pstate_register_driver(int mode) { + struct freq_attr **attr = NULL; int ret; ret = amd_pstate_set_driver(mode); @@ -1245,6 +1321,22 @@ static int amd_pstate_register_driver(int mode) cppc_state = mode; + /* + * Note: It is important to compute the attrs _after_ + * re-initializing the cppc_state. Some attributes become + * visible only when cppc_state is AMD_PSTATE_ACTIVE. + */ + attr = get_freq_attrs(); + if (IS_ERR(attr)) { + ret = (int) PTR_ERR(attr); + pr_err("Couldn't compute freq_attrs for current mode %s [%d]\n", + amd_pstate_get_mode_string(cppc_state), ret); + amd_pstate_driver_cleanup(); + return ret; + } + + current_pstate_driver->attr = attr; + /* at least one CPU supports CPB */ current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); @@ -1387,37 +1479,9 @@ static ssize_t prefcore_show(struct device *dev, return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); } -cpufreq_freq_attr_ro(amd_pstate_max_freq); -cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); - -cpufreq_freq_attr_ro(amd_pstate_highest_perf); -cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); -cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); -cpufreq_freq_attr_rw(energy_performance_preference); -cpufreq_freq_attr_ro(energy_performance_available_preferences); static DEVICE_ATTR_RW(status); static DEVICE_ATTR_RO(prefcore); -static struct freq_attr *amd_pstate_attr[] = { - &amd_pstate_max_freq, - &amd_pstate_lowest_nonlinear_freq, - &amd_pstate_highest_perf, - &amd_pstate_prefcore_ranking, - &amd_pstate_hw_prefcore, - NULL, -}; - -static struct freq_attr *amd_pstate_epp_attr[] = { - &amd_pstate_max_freq, - &amd_pstate_lowest_nonlinear_freq, - &amd_pstate_highest_perf, - &amd_pstate_prefcore_ranking, - &amd_pstate_hw_prefcore, - &energy_performance_preference, - &energy_performance_available_preferences, - NULL, -}; - static struct attribute *pstate_global_attributes[] = { &dev_attr_status.attr, &dev_attr_prefcore.attr, @@ -1677,7 +1741,6 @@ static struct cpufreq_driver amd_pstate_driver = { .set_boost = amd_pstate_set_boost, .update_limits = amd_pstate_update_limits, .name = "amd-pstate", - .attr = amd_pstate_attr, }; static struct cpufreq_driver amd_pstate_epp_driver = { @@ -1693,7 +1756,6 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .update_limits = amd_pstate_update_limits, .set_boost = amd_pstate_set_boost, .name = "amd-pstate-epp", - .attr = amd_pstate_epp_attr, }; /* @@ -1839,7 +1901,7 @@ static int __init amd_pstate_init(void) return ret; global_attr_free: - cpufreq_unregister_driver(current_pstate_driver); + amd_pstate_unregister_driver(0); return ret; } device_initcall(amd_pstate_init); From c15f483e05229da1a66f9b76157b4ceed68976de Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:48 +0530 Subject: [PATCH 147/188] x86/cpufeatures: Add AMD CPPC Performance Priority feature. commit 172100088f9b131b88bcde70724485470c20e7d2 upstream. Some future AMD processors have feature named "CPPC Performance Priority" which lets userspace specify different floor performance levels for different CPUs. The platform firmware takes these different floor performance levels into consideration while throttling the CPUs under power/thermal constraints. The presence of this feature is indicated by bit 16 of the EDX register for CPUID leaf 0x80000007. More details can be found in AMD Publication titled "AMD64 Collaborative Processor Performance Control (CPPC) Performance Priority" Revision 1.10. Define a new feature bit named X86_FEATURE_CPPC_PERF_PRIO to map to CPUID 0x80000007.EDX[16]. [Backport changes] The upstream patch removes empty lines from arch/x86/include/asm/cpufeatures.h and tools/arch/x86/include/asm/cpufeatures.h. Since these empty lines are not present in our kernel, the changes are being skipped. Reviewed-by: Borislav Petkov (AMD) Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: priyanka-mani Signed-off-by: mohanasv2 --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/kernel/cpu/scattered.c | 1 + tools/arch/x86/include/asm/cpufeatures.h | 1 + 3 files changed, 3 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 29d8d0fdece84..cbe324e5197dc 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -413,6 +413,7 @@ /* AMD-defined CPU features, CPUID level 0x80000007 (EBX), word 17 */ #define X86_FEATURE_OVERFLOW_RECOV (17*32+ 0) /* "overflow_recov" MCA overflow recovery support */ #define X86_FEATURE_SUCCOR (17*32+ 1) /* "succor" Uncorrectable error containment and recovery */ +#define X86_FEATURE_CPPC_PERF_PRIO (17*32+ 2) /* CPPC Floor Perf support */ #define X86_FEATURE_SMCA (17*32+ 3) /* "smca" Scalable MCA */ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EDX), word 18 */ diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index a1eabee0fb6da..5adf08a2aca30 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -45,6 +45,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_HW_PSTATE, CPUID_EDX, 7, 0x80000007, 0 }, { X86_FEATURE_CPB, CPUID_EDX, 9, 0x80000007, 0 }, { X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 }, + { X86_FEATURE_CPPC_PERF_PRIO, CPUID_EDX, 16, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, { X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 }, { X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 }, diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index bbea47c0dc095..d0ea6b469ae53 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -401,6 +401,7 @@ /* AMD-defined CPU features, CPUID level 0x80000007 (EBX), word 17 */ #define X86_FEATURE_OVERFLOW_RECOV (17*32+ 0) /* MCA overflow recovery support */ +#define X86_FEATURE_CPPC_PERF_PRIO (17*32+ 2) /* CPPC Floor Perf support */ #define X86_FEATURE_SUCCOR (17*32+ 1) /* Uncorrectable error containment and recovery */ #define X86_FEATURE_SMCA (17*32+ 3) /* Scalable MCA */ From aa0bc36bf92acaab984f7695e8180b01a43b1346 Mon Sep 17 00:00:00 2001 From: Luo Jie Date: Thu, 17 Apr 2025 18:47:08 +0800 Subject: [PATCH 148/188] bitfield: Add FIELD_MODIFY() helper commit a256ae22570ee4c3427fdc703a58a89afee6a332 upstream. Add a helper for replacing the contents of bitfield in memory with the specified value. Even though a helper xxx_replace_bits() is available, it is not well documented, and only reports errors at the run time, which will not be helpful to catch possible overflow errors due to incorrect parameter types used. FIELD_MODIFY(REG_FIELD_C, ®, c) is the wrapper to the code below. reg &= ~REG_FIELD_C; reg |= FIELD_PREP(REG_FIELD_C, c); Yury: trim commit message, align backslashes. Signed-off-by: Luo Jie Signed-off-by: Yury Norov Signed-off-by: priyanka-mani Signed-off-by: mohanasv2 --- include/linux/bitfield.h | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h index ebfa12f695012..d0dfb2d86ec1c 100644 --- a/include/linux/bitfield.h +++ b/include/linux/bitfield.h @@ -8,6 +8,7 @@ #define _LINUX_BITFIELD_H #include +#include #include /* @@ -38,8 +39,7 @@ * FIELD_PREP(REG_FIELD_D, 0x40); * * Modify: - * reg &= ~REG_FIELD_C; - * reg |= FIELD_PREP(REG_FIELD_C, c); + * FIELD_MODIFY(REG_FIELD_C, ®, c); */ #define __bf_shf(x) (__builtin_ffsll(x) - 1) @@ -155,6 +155,23 @@ (typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask)); \ }) +/** + * FIELD_MODIFY() - modify a bitfield element + * @_mask: shifted mask defining the field's length and position + * @_reg_p: pointer to the memory that should be updated + * @_val: value to store in the bitfield + * + * FIELD_MODIFY() modifies the set of bits in @_reg_p specified by @_mask, + * by replacing them with the bitfield value passed in as @_val. + */ +#define FIELD_MODIFY(_mask, _reg_p, _val) \ + ({ \ + typecheck_pointer(_reg_p); \ + __BF_FIELD_CHECK(_mask, *(_reg_p), _val, "FIELD_MODIFY: "); \ + *(_reg_p) &= ~(_mask); \ + *(_reg_p) |= (((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask)); \ + }) + extern void __compiletime_error("value doesn't fit into mask") __field_overflow(void); extern void __compiletime_error("bad bitfield mask") From b85c0f06f44c5c3b9a430bdb2c060745ecf62c80 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:49 +0530 Subject: [PATCH 149/188] amd-pstate: Add support for CPPC_REQ2 and FLOOR_PERF commit 97838281f587a9e98e74b913201f7408214b5999 upstream. Some future AMD processors have feature named "CPPC Performance Priority" which lets userspace specify different floor performance levels for different CPUs. The platform firmware takes these different floor performance levels into consideration while throttling the CPUs under power/thermal constraints. The presence of this feature is indicated by bit 16 of the EDX register for CPUID leaf 0x80000007. More details can be found in AMD Publication titled "AMD64 Collaborative Processor Performance Control (CPPC) Performance Priority" Revision 1.10. The number of distinct floor performance levels supported on the platform will be advertised through the bits 32:39 of the MSR_AMD_CPPC_CAP1. Bits 0:7 of a new MSR MSR_AMD_CPPC_REQ2 (0xc00102b5) will be used to specify the desired floor performance level for that CPU. Add support for the aforementioned MSR_AMD_CPPC_REQ2, and macros for parsing and updating the relevant bits from MSR_AMD_CPPC_CAP1 and MSR_AMD_CPPC_REQ2. On boot if the default value of the MSR_AMD_CPPC_REQ2[7:0] (Floor Perf) is lower than CPPC.lowest_perf, and thus invalid, initialize it to MSR_AMD_CPPC_CAP1.nominal_perf which is a sane default value. Save the boot-time floor_perf during amd_pstate_init_floor_perf(). In a subsequent patch it will be restored in the suspend, offline, and exit paths, mirroring how bios_min_perf is handled for MSR_AMD_CPPC_REQ. Link: https://docs.amd.com/v/u/en-US/69206_1.10_AMD64_CPPC_PUB Reviewed-by: Mario Limonciello (AMD) Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: priyanka-mani Signed-off-by: mohanasv2 --- arch/x86/include/asm/msr-index.h | 5 ++ drivers/cpufreq/amd-pstate.c | 78 +++++++++++++++++++++++++++++++- drivers/cpufreq/amd-pstate.h | 8 ++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 6634830fa77a7..3ce44b43217de 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -705,12 +705,14 @@ #define MSR_AMD_CPPC_CAP2 0xc00102b2 #define MSR_AMD_CPPC_REQ 0xc00102b3 #define MSR_AMD_CPPC_STATUS 0xc00102b4 +#define MSR_AMD_CPPC_REQ2 0xc00102b5 /* Masks for use with MSR_AMD_CPPC_CAP1 */ #define AMD_CPPC_LOWEST_PERF_MASK GENMASK(7, 0) #define AMD_CPPC_LOWNONLIN_PERF_MASK GENMASK(15, 8) #define AMD_CPPC_NOMINAL_PERF_MASK GENMASK(23, 16) #define AMD_CPPC_HIGHEST_PERF_MASK GENMASK(31, 24) +#define AMD_CPPC_FLOOR_PERF_CNT_MASK GENMASK_ULL(39, 32) /* Masks for use with MSR_AMD_CPPC_REQ */ #define AMD_CPPC_MAX_PERF_MASK GENMASK(7, 0) @@ -718,6 +720,9 @@ #define AMD_CPPC_DES_PERF_MASK GENMASK(23, 16) #define AMD_CPPC_EPP_PERF_MASK GENMASK(31, 24) +/* Masks for use with MSR_AMD_CPPC_REQ2 */ +#define AMD_CPPC_FLOOR_PERF_MASK GENMASK(7, 0) + /* AMD Performance Counter Global Status and Control MSRs */ #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300 #define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301 diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index d38df671da53c..99d6ae3576aa6 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -328,6 +328,65 @@ static inline int amd_pstate_set_epp(struct cpufreq_policy *policy, u8 epp) return static_call(amd_pstate_set_epp)(policy, epp); } +static int amd_pstate_set_floor_perf(struct cpufreq_policy *policy, u8 perf) +{ + struct amd_cpudata *cpudata = policy->driver_data; + u64 value, prev; + int ret; + + if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) + return 0; + + value = prev = READ_ONCE(cpudata->cppc_req2_cached); + FIELD_MODIFY(AMD_CPPC_FLOOR_PERF_MASK, &value, perf); + + if (value == prev) + return 0; + + ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, value); + if (ret) { + pr_err("failed to set CPPC REQ2 value. Error (%d)\n", ret); + return ret; + } + + WRITE_ONCE(cpudata->cppc_req2_cached, value); + + return ret; +} + +static int amd_pstate_init_floor_perf(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + u8 floor_perf; + u64 value; + int ret; + + if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) + return 0; + + ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, &value); + if (ret) { + pr_err("failed to read CPPC REQ2 value. Error (%d)\n", ret); + return ret; + } + + WRITE_ONCE(cpudata->cppc_req2_cached, value); + floor_perf = FIELD_GET(AMD_CPPC_FLOOR_PERF_MASK, + cpudata->cppc_req2_cached); + + /* Set a sane value for floor_perf if the default value is invalid */ + if (floor_perf < cpudata->perf.lowest_perf) { + floor_perf = cpudata->perf.nominal_perf; + ret = amd_pstate_set_floor_perf(policy, floor_perf); + if (ret) + return ret; + } + + cpudata->bios_floor_perf = floor_perf; + + return 0; +} + static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp) { struct amd_cpudata *cpudata = policy->driver_data; @@ -425,6 +484,7 @@ static int msr_init_perf(struct amd_cpudata *cpudata) perf.lowest_perf = FIELD_GET(AMD_CPPC_LOWEST_PERF_MASK, cap1); WRITE_ONCE(cpudata->perf, perf); WRITE_ONCE(cpudata->prefcore_ranking, FIELD_GET(AMD_CPPC_HIGHEST_PERF_MASK, cap1)); + WRITE_ONCE(cpudata->floor_perf_cnt, FIELD_GET(AMD_CPPC_FLOOR_PERF_CNT_MASK, cap1)); return 0; } @@ -1005,6 +1065,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) cpudata->nominal_freq, perf.highest_perf); + policy->driver_data = cpudata; ret = amd_pstate_cppc_enable(policy); if (ret) goto free_cpudata1; @@ -1017,6 +1078,12 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) if (cpu_feature_enabled(X86_FEATURE_CPPC)) policy->fast_switch_possible = true; + ret = amd_pstate_init_floor_perf(policy); + if (ret) { + dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); + goto free_cpudata1; + } + ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], FREQ_QOS_MIN, FREQ_QOS_MIN_DEFAULT_VALUE); if (ret < 0) { @@ -1031,7 +1098,6 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) goto free_cpudata2; } - policy->driver_data = cpudata; if (!current_pstate_driver->adjust_perf) current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; @@ -1043,6 +1109,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) free_cpudata1: pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); kfree(cpudata); + policy->driver_data = NULL; return ret; } @@ -1053,6 +1120,7 @@ static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) /* Reset CPPC_REQ MSR to the BIOS value */ amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); + amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); freq_qos_remove_request(&cpudata->req[1]); freq_qos_remove_request(&cpudata->req[0]); @@ -1586,6 +1654,12 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (ret) goto free_cpudata1; + ret = amd_pstate_init_floor_perf(policy); + if (ret) { + dev_err(dev, "Failed to initialize Floor Perf (%d)\n", ret); + goto free_cpudata1; + } + current_pstate_driver->adjust_perf = NULL; return 0; @@ -1593,6 +1667,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) free_cpudata1: pr_warn("Failed to initialize CPU %d: %d\n", policy->cpu, ret); kfree(cpudata); + policy->driver_data = NULL; return ret; } @@ -1605,6 +1680,7 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) /* Reset CPPC_REQ MSR to the BIOS value */ amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); + amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); kfree(cpudata); policy->driver_data = NULL; diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index cb45fdca27a6c..303da70b0afa7 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -62,9 +62,14 @@ struct amd_aperf_mperf { * @cpu: CPU number * @req: constraint request to apply * @cppc_req_cached: cached performance request hints + * @cppc_req2_cached: cached value of MSR_AMD_CPPC_REQ2 * @perf: cached performance-related data * @prefcore_ranking: the preferred core ranking, the higher value indicates a higher * priority. + * @floor_perf_cnt: Cached value of the number of distinct floor + * performance levels supported + * @bios_floor_perf: Cached value of the boot-time floor performance level from + * MSR_AMD_CPPC_REQ2 * @min_limit_freq: Cached value of policy->min (in khz) * @max_limit_freq: Cached value of policy->max (in khz) * @nominal_freq: the frequency (in khz) that mapped to nominal_perf @@ -87,10 +92,13 @@ struct amd_cpudata { struct freq_qos_request req[2]; u64 cppc_req_cached; + u64 cppc_req2_cached; union perf_cached perf; u8 prefcore_ranking; + u8 floor_perf_cnt; + u8 bios_floor_perf; u32 min_limit_freq; u32 max_limit_freq; u32 nominal_freq; From 6b6abc85f57cfb42d8d4d6fb4d7c5d767085b37d Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Tue, 26 Aug 2025 00:27:47 -0500 Subject: [PATCH 150/188] cpufreq/amd-pstate: Fix a regression leading to EPP 0 after resume commit ba3319e5905710abe495b11a1aaf03ebb51d62e2 upstream. During the suspend sequence the cached CPPC request is destroyed with the expectation that it's restored during resume. This assumption broke when the separate cache EPP variable was removed, and then it was broken again by commit 608a76b65288 ("cpufreq/amd-pstate: Add support for the "Requested CPU Min frequency" BIOS option") which explicitly set it to zero during suspend. Remove the invalidation and set the value during the suspend call to update limits so that the cached variable can be used to restore on resume. Fixes: 608a76b65288 ("cpufreq/amd-pstate: Add support for the "Requested CPU Min frequency" BIOS option") Fixes: b7a41156588a ("cpufreq/amd-pstate: Invalidate cppc_req_cached during suspend") Reported-by: goldens Closes: https://community.frame.work/t/increased-power-usage-after-resuming-from-suspend-on-ryzen-7040-kernel-6-15-regression/ Closes: https://bugzilla.redhat.com/show_bug.cgi?id=2391221 Tested-by: goldens Tested-by: Willian Wang Reported-by: Vincent Mauirn Closes: https://bugzilla.kernel.org/show_bug.cgi?id=219981 Tested-by: Alex De Lorenzo Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20250826052747.2240670-1-superm1@kernel.org Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 99d6ae3576aa6..49ccc507dc363 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1761,13 +1761,14 @@ static int amd_pstate_suspend(struct cpufreq_policy *policy) * min_perf value across kexec reboots. If this CPU is just resumed back without kexec, * the limits, epp and desired perf will get reset to the cached values in cpudata struct */ - ret = amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); + ret = amd_pstate_update_perf(policy, perf.bios_min_perf, + FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), + false); if (ret) return ret; - /* invalidate to ensure it's rewritten during resume */ - cpudata->cppc_req_cached = 0; - /* set this flag to avoid setting core offline*/ cpudata->suspended = true; From a93242de9b4c80a2d66c1580c7f30e35f719e8fa Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Tue, 23 Sep 2025 10:29:29 -0500 Subject: [PATCH 151/188] cpufreq/amd-pstate: Fix a regression leading to EPP 0 after hibernate commit 85d7dda5a9f665ea579741ec873a8841f37e8943 upstream. After resuming from S4, all CPUs except the boot CPU have the wrong EPP hint programmed. This is because when the CPUs were offlined the EPP value was reset to 0. This is a similar problem as fixed by commit ba3319e590571 ("cpufreq/amd-pstate: Fix a regression leading to EPP 0 after resume") and the solution is also similar. When offlining rather than reset the values to zero, reset them to match those chosen by the policy. When the CPUs are onlined again these values will be restored. Closes: https://community.frame.work/t/increased-power-usage-after-resuming-from-suspend-on-ryzen-7040-kernel-6-15-regression/74531/20?u=mario_limonciello Fixes: 608a76b65288 ("cpufreq/amd-pstate: Add support for the "Requested CPU Min frequency" BIOS option") Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 49ccc507dc363..9cb3500622f7c 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1747,7 +1747,11 @@ static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the * limits, epp and desired perf will get reset to the cached values in cpudata struct */ - return amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); + return amd_pstate_update_perf(policy, perf.bios_min_perf, + FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), + FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), + false); } static int amd_pstate_suspend(struct cpufreq_policy *policy) From ba11f5df1c65f23129aad88422812b0389ce7a18 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:50 +0530 Subject: [PATCH 152/188] amd-pstate: Add sysfs support for floor_freq and floor_count commit b9f103d0968bc5b33bff1b1eb11c756b2ac07c6c upstream. When Floor Performance feature is supported by the platform, expose two sysfs files: * amd_pstate_floor_freq to allow userspace to request the floor frequency for each CPU. * amd_pstate_floor_count which advertises the number of distinct levels of floor frequencies supported on this platform. Reset the floor_perf to bios_floor_perf in the suspend, offline, and exit paths, and restore the value to the cached user-request floor_freq on the resume and online paths mirroring how bios_min_perf is handled for MSR_AMD_CPPC_REQ. Reviewed-by: Mario Limonciello (AMD) Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 93 +++++++++++++++++++++++++++++++++--- drivers/cpufreq/amd-pstate.h | 2 + 2 files changed, 89 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 9cb3500622f7c..42f9548c187e5 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -382,8 +382,10 @@ static int amd_pstate_init_floor_perf(struct cpufreq_policy *policy) return ret; } - cpudata->bios_floor_perf = floor_perf; + cpudata->bios_floor_perf = floor_perf; + cpudata->floor_freq = perf_to_freq(cpudata->perf, cpudata->nominal_freq, + floor_perf); return 0; } @@ -1275,6 +1277,46 @@ static ssize_t show_energy_performance_preference( return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); } +static ssize_t store_amd_pstate_floor_freq(struct cpufreq_policy *policy, + const char *buf, size_t count) +{ + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + unsigned int freq; + u8 floor_perf; + int ret; + + ret = kstrtouint(buf, 0, &freq); + if (ret) + return ret; + + if (freq < policy->cpuinfo.min_freq || freq > policy->max) + return -EINVAL; + + floor_perf = freq_to_perf(perf, cpudata->nominal_freq, freq); + ret = amd_pstate_set_floor_perf(policy, floor_perf); + + if (!ret) + cpudata->floor_freq = freq; + + return ret ?: count; +} + +static ssize_t show_amd_pstate_floor_freq(struct cpufreq_policy *policy, char *buf) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + return sysfs_emit(buf, "%u\n", cpudata->floor_freq); +} + +static ssize_t show_amd_pstate_floor_count(struct cpufreq_policy *policy, char *buf) +{ + struct amd_cpudata *cpudata = policy->driver_data; + u8 count = cpudata->floor_perf_cnt; + + return sysfs_emit(buf, "%u\n", count); +} + cpufreq_freq_attr_ro(amd_pstate_max_freq); cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); @@ -1283,6 +1325,8 @@ cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); cpufreq_freq_attr_rw(energy_performance_preference); cpufreq_freq_attr_ro(energy_performance_available_preferences); +cpufreq_freq_attr_rw(amd_pstate_floor_freq); +cpufreq_freq_attr_ro(amd_pstate_floor_count); struct freq_attr_visibility { struct freq_attr *attr; @@ -1307,6 +1351,12 @@ static bool epp_visibility(void) return cppc_state == AMD_PSTATE_ACTIVE; } +/* Determines whether amd_pstate_floor_freq related attributes should be visible */ +static bool floor_freq_visibility(void) +{ + return cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO); +} + static struct freq_attr_visibility amd_pstate_attr_visibility[] = { {&amd_pstate_max_freq, always_visible}, {&amd_pstate_lowest_nonlinear_freq, always_visible}, @@ -1315,6 +1365,8 @@ static struct freq_attr_visibility amd_pstate_attr_visibility[] = { {&amd_pstate_hw_prefcore, prefcore_visibility}, {&energy_performance_preference, epp_visibility}, {&energy_performance_available_preferences, epp_visibility}, + {&amd_pstate_floor_freq, floor_freq_visibility}, + {&amd_pstate_floor_count, floor_freq_visibility}, }; static struct freq_attr **get_freq_attrs(void) @@ -1734,24 +1786,39 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) static int amd_pstate_cpu_online(struct cpufreq_policy *policy) { - return amd_pstate_cppc_enable(policy); + struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + u8 cached_floor_perf; + int ret; + + ret = amd_pstate_cppc_enable(policy); + if (ret) + return ret; + + cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); + return amd_pstate_set_floor_perf(policy, cached_floor_perf); } static int amd_pstate_cpu_offline(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; union perf_cached perf = READ_ONCE(cpudata->perf); + int ret; /* * Reset CPPC_REQ MSR to the BIOS value, this will allow us to retain the BIOS specified * min_perf value across kexec reboots. If this CPU is just onlined normally after this, the * limits, epp and desired perf will get reset to the cached values in cpudata struct */ - return amd_pstate_update_perf(policy, perf.bios_min_perf, + ret = amd_pstate_update_perf(policy, perf.bios_min_perf, FIELD_GET(AMD_CPPC_DES_PERF_MASK, cpudata->cppc_req_cached), FIELD_GET(AMD_CPPC_MAX_PERF_MASK, cpudata->cppc_req_cached), FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached), false); + if (ret) + return ret; + + return amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); } static int amd_pstate_suspend(struct cpufreq_policy *policy) @@ -1773,6 +1840,10 @@ static int amd_pstate_suspend(struct cpufreq_policy *policy) if (ret) return ret; + ret = amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); + if (ret) + return ret; + /* set this flag to avoid setting core offline*/ cpudata->suspended = true; @@ -1784,15 +1855,24 @@ static int amd_pstate_resume(struct cpufreq_policy *policy) struct amd_cpudata *cpudata = policy->driver_data; union perf_cached perf = READ_ONCE(cpudata->perf); int cur_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->cur); + u8 cached_floor_perf; + int ret; /* Set CPPC_REQ to last sane value until the governor updates it */ - return amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, - 0U, false); + ret = amd_pstate_update_perf(policy, perf.min_limit_perf, cur_perf, perf.max_limit_perf, + 0U, false); + if (ret) + return ret; + + cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); + return amd_pstate_set_floor_perf(policy, cached_floor_perf); } static int amd_pstate_epp_resume(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; + union perf_cached perf = READ_ONCE(cpudata->perf); + u8 cached_floor_perf; if (cpudata->suspended) { int ret; @@ -1805,7 +1885,8 @@ static int amd_pstate_epp_resume(struct cpufreq_policy *policy) cpudata->suspended = false; } - return 0; + cached_floor_perf = freq_to_perf(perf, cpudata->nominal_freq, cpudata->floor_freq); + return amd_pstate_set_floor_perf(policy, cached_floor_perf); } static struct cpufreq_driver amd_pstate_driver = { diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 303da70b0afa7..453adfb445f8d 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -74,6 +74,7 @@ struct amd_aperf_mperf { * @max_limit_freq: Cached value of policy->max (in khz) * @nominal_freq: the frequency (in khz) that mapped to nominal_perf * @lowest_nonlinear_freq: the frequency (in khz) that mapped to lowest_nonlinear_perf + * @floor_freq: Cached value of the user requested floor_freq * @cur: Difference of Aperf/Mperf/tsc count between last and current sample * @prev: Last Aperf/Mperf/tsc count value read from register * @freq: current cpu frequency value (in khz) @@ -103,6 +104,7 @@ struct amd_cpudata { u32 max_limit_freq; u32 nominal_freq; u32 lowest_nonlinear_freq; + u32 floor_freq; struct amd_aperf_mperf cur; struct amd_aperf_mperf prev; From 2f3873823c8b9712e000160b9cd96605cfe596db Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:51 +0530 Subject: [PATCH 153/188] amd-pstate: Introduce a tracepoint trace_amd_pstate_cppc_req2() commit 30c63f723440f12626a19da5a93e094da29af51e upstream. Introduce a new tracepoint trace_amd_pstate_cppc_req2() to track updates to MSR_AMD_CPPC_REQ2. Invoke this while changing the Floor Perf. Reviewed-by: Mario Limonciello Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-trace.h | 35 ++++++++++++++++++++++++++++++ drivers/cpufreq/amd-pstate.c | 14 +++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-trace.h b/drivers/cpufreq/amd-pstate-trace.h index 32e1bdc588c52..91fa073b2be48 100644 --- a/drivers/cpufreq/amd-pstate-trace.h +++ b/drivers/cpufreq/amd-pstate-trace.h @@ -133,6 +133,41 @@ TRACE_EVENT(amd_pstate_epp_perf, ) ); +TRACE_EVENT(amd_pstate_cppc_req2, + + TP_PROTO(unsigned int cpu_id, + u8 floor_perf, + bool changed, + int err_code + ), + + TP_ARGS(cpu_id, + floor_perf, + changed, + err_code), + + TP_STRUCT__entry( + __field(unsigned int, cpu_id) + __field(u8, floor_perf) + __field(bool, changed) + __field(int, err_code) + ), + + TP_fast_assign( + __entry->cpu_id = cpu_id; + __entry->floor_perf = floor_perf; + __entry->changed = changed; + __entry->err_code = err_code; + ), + + TP_printk("cpu%u: floor_perf=%u, changed=%u (error = %d)", + __entry->cpu_id, + __entry->floor_perf, + __entry->changed, + __entry->err_code + ) +); + #endif /* _AMD_PSTATE_TRACE_H */ /* This part must be outside protection */ diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 42f9548c187e5..231372a0a43b6 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -332,6 +332,7 @@ static int amd_pstate_set_floor_perf(struct cpufreq_policy *policy, u8 perf) { struct amd_cpudata *cpudata = policy->driver_data; u64 value, prev; + bool changed; int ret; if (!cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO)) @@ -340,17 +341,24 @@ static int amd_pstate_set_floor_perf(struct cpufreq_policy *policy, u8 perf) value = prev = READ_ONCE(cpudata->cppc_req2_cached); FIELD_MODIFY(AMD_CPPC_FLOOR_PERF_MASK, &value, perf); - if (value == prev) - return 0; + changed = value != prev; + if (!changed) { + ret = 0; + goto out_trace; + } ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ2, value); if (ret) { + changed = false; pr_err("failed to set CPPC REQ2 value. Error (%d)\n", ret); - return ret; + goto out_trace; } WRITE_ONCE(cpudata->cppc_req2_cached, value); +out_trace: + if (trace_amd_pstate_cppc_req2_enabled()) + trace_amd_pstate_cppc_req2(cpudata->cpu, perf, changed, ret); return ret; } From 584e8aca26f9404efed0f0d6556b0eebc20a8bcb Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:52 +0530 Subject: [PATCH 154/188] amd-pstate-ut: Add module parameter to select testcases commit c6a2b750de13db9103db29c64927bec3919232b5 upstream. Currently when amd-pstate-ut test module is loaded, it runs all the tests from amd_pstate_ut_cases[] array. Add a module parameter named "test_list" that accepts a comma-delimited list of test names, allowing users to run a selected subset of tests. When the parameter is omitted or empty, all tests are run as before. Signed-off-by: Gautham R. Shenoy Reviewed-by: Mario Limonciello (AMD) Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 65f9d2bae2d36..bd711d90a6592 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -33,6 +33,10 @@ #include "amd-pstate.h" +static char *test_list; +module_param(test_list, charp, 0444); +MODULE_PARM_DESC(test_list, + "Comma-delimited list of tests to run (empty means run all tests)"); struct amd_pstate_ut_struct { const char *name; @@ -56,6 +60,25 @@ static struct amd_pstate_ut_struct amd_pstate_ut_cases[] = { {"amd_pstate_ut_check_driver", amd_pstate_ut_check_driver } }; +static bool test_in_list(const char *list, const char *name) +{ + size_t name_len = strlen(name); + const char *p = list; + + while (*p) { + const char *sep = strchr(p, ','); + size_t token_len = sep ? sep - p : strlen(p); + + if (token_len == name_len && !strncmp(p, name, token_len)) + return true; + if (!sep) + break; + p = sep + 1; + } + + return false; +} + static bool get_shared_mem(void) { bool result = false; @@ -273,7 +296,13 @@ static int __init amd_pstate_ut_init(void) u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases); for (i = 0; i < arr_size; i++) { - int ret = amd_pstate_ut_cases[i].func(i); + int ret; + + if (test_list && *test_list && + !test_in_list(test_list, amd_pstate_ut_cases[i].name)) + continue; + + ret = amd_pstate_ut_cases[i].func(i); if (ret) pr_err("%-4d %-20s\t fail: %d!\n", i+1, amd_pstate_ut_cases[i].name, ret); From 0b96702bb2001511134423c7c4eb8f1ffa009ee0 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:53 +0530 Subject: [PATCH 155/188] amd-pstate-ut: Add a testcase to validate the visibility of driver attributes commit 3b90e5a4176acacc6781b9ac84cdc5ac53671eee upstream. amd-pstate driver has per-attribute visibility functions to dynamically control which sysfs freq_attrs are exposed based on the platform capabilities and the current amd_pstate mode. However, there is no test coverage to validate that the driver's live attribute list matches the expected visibility for each mode. Add amd_pstate_ut_check_freq_attrs() to the amd-pstate unit test module. For each enabled mode (passive, active, guided), the test independently derives the expected visibility of each attribute: - Core attributes (max_freq, lowest_nonlinear_freq, highest_perf) are always expected. - Prefcore attributes (prefcore_ranking, hw_prefcore) are expected only when cpudata->hw_prefcore indicates platform support. - EPP attributes (energy_performance_preference, energy_performance_available_preferences) are expected only in active mode. - Floor frequency attributes (floor_freq, floor_count) are expected only when X86_FEATURE_CPPC_PERF_PRIO is present. Compare these independent expectations against the live driver's attr array, catching bugs such as attributes leaking into wrong modes or visibility functions checking incorrect conditions. Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 139 ++++++++++++++++++++++++++++++-- drivers/cpufreq/amd-pstate.c | 8 ++ drivers/cpufreq/amd-pstate.h | 4 + 3 files changed, 146 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index bd711d90a6592..092dd87a01fda 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -23,6 +23,8 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include +#include #include #include #include @@ -51,13 +53,15 @@ static int amd_pstate_ut_check_enabled(u32 index); static int amd_pstate_ut_check_perf(u32 index); static int amd_pstate_ut_check_freq(u32 index); static int amd_pstate_ut_check_driver(u32 index); +static int amd_pstate_ut_check_freq_attrs(u32 index); static struct amd_pstate_ut_struct amd_pstate_ut_cases[] = { - {"amd_pstate_ut_acpi_cpc_valid", amd_pstate_ut_acpi_cpc_valid }, - {"amd_pstate_ut_check_enabled", amd_pstate_ut_check_enabled }, - {"amd_pstate_ut_check_perf", amd_pstate_ut_check_perf }, - {"amd_pstate_ut_check_freq", amd_pstate_ut_check_freq }, - {"amd_pstate_ut_check_driver", amd_pstate_ut_check_driver } + {"amd_pstate_ut_acpi_cpc_valid", amd_pstate_ut_acpi_cpc_valid }, + {"amd_pstate_ut_check_enabled", amd_pstate_ut_check_enabled }, + {"amd_pstate_ut_check_perf", amd_pstate_ut_check_perf }, + {"amd_pstate_ut_check_freq", amd_pstate_ut_check_freq }, + {"amd_pstate_ut_check_driver", amd_pstate_ut_check_driver }, + {"amd_pstate_ut_check_freq_attrs", amd_pstate_ut_check_freq_attrs }, }; static bool test_in_list(const char *list, const char *name) @@ -291,6 +295,131 @@ static int amd_pstate_ut_check_driver(u32 index) return ret; } +enum attr_category { + ATTR_ALWAYS, + ATTR_PREFCORE, + ATTR_EPP, + ATTR_FLOOR_FREQ, +}; + +static const struct { + const char *name; + enum attr_category category; +} expected_freq_attrs[] = { + {"amd_pstate_max_freq", ATTR_ALWAYS}, + {"amd_pstate_lowest_nonlinear_freq", ATTR_ALWAYS}, + {"amd_pstate_highest_perf", ATTR_ALWAYS}, + {"amd_pstate_prefcore_ranking", ATTR_PREFCORE}, + {"amd_pstate_hw_prefcore", ATTR_PREFCORE}, + {"energy_performance_preference", ATTR_EPP}, + {"energy_performance_available_preferences", ATTR_EPP}, + {"amd_pstate_floor_freq", ATTR_FLOOR_FREQ}, + {"amd_pstate_floor_count", ATTR_FLOOR_FREQ}, +}; + +static bool attr_in_driver(struct freq_attr **driver_attrs, const char *name) +{ + int j; + + for (j = 0; driver_attrs[j]; j++) { + if (!strcmp(driver_attrs[j]->attr.name, name)) + return true; + } + return false; +} + +/* + * Verify that for each mode the driver's live ->attr array contains exactly + * the attributes that should be visible. Expected visibility is derived + * independently from hw_prefcore, cpu features, and the current mode — + * not from the driver's own visibility functions. + */ +static int amd_pstate_ut_check_freq_attrs(u32 index) +{ + enum amd_pstate_mode orig_mode = amd_pstate_get_status(); + static const enum amd_pstate_mode modes[] = { + AMD_PSTATE_PASSIVE, AMD_PSTATE_ACTIVE, AMD_PSTATE_GUIDED, + }; + bool has_prefcore, has_floor_freq; + int m, i, ret; + + has_floor_freq = cpu_feature_enabled(X86_FEATURE_CPPC_PERF_PRIO); + + /* + * Determine prefcore support from any online CPU's cpudata. + * hw_prefcore reflects the platform-wide decision made at init. + */ + has_prefcore = false; + for_each_online_cpu(i) { + struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; + struct amd_cpudata *cpudata; + + policy = cpufreq_cpu_get(i); + if (!policy) + continue; + cpudata = policy->driver_data; + has_prefcore = cpudata->hw_prefcore; + break; + } + + for (m = 0; m < ARRAY_SIZE(modes); m++) { + struct freq_attr **driver_attrs; + + ret = amd_pstate_set_mode(modes[m]); + if (ret) + goto out; + + driver_attrs = amd_pstate_get_current_attrs(); + if (!driver_attrs) { + pr_err("%s: no driver attrs in mode %s\n", + __func__, amd_pstate_get_mode_string(modes[m])); + ret = -EINVAL; + goto out; + } + + for (i = 0; i < ARRAY_SIZE(expected_freq_attrs); i++) { + bool expected, found; + + switch (expected_freq_attrs[i].category) { + case ATTR_ALWAYS: + expected = true; + break; + case ATTR_PREFCORE: + expected = has_prefcore; + break; + case ATTR_EPP: + expected = (modes[m] == AMD_PSTATE_ACTIVE); + break; + case ATTR_FLOOR_FREQ: + expected = has_floor_freq; + break; + default: + expected = false; + break; + } + + found = attr_in_driver(driver_attrs, + expected_freq_attrs[i].name); + + if (expected != found) { + pr_err("%s: mode %s: attr %s expected %s but is %s\n", + __func__, + amd_pstate_get_mode_string(modes[m]), + expected_freq_attrs[i].name, + expected ? "visible" : "hidden", + found ? "visible" : "hidden"); + ret = -EINVAL; + goto out; + } + } + } + + ret = 0; +out: + amd_pstate_set_mode(orig_mode); + return ret; +} + static int __init amd_pstate_ut_init(void) { u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases); diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 231372a0a43b6..b02956f33f075 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1377,6 +1377,14 @@ static struct freq_attr_visibility amd_pstate_attr_visibility[] = { {&amd_pstate_floor_count, floor_freq_visibility}, }; +struct freq_attr **amd_pstate_get_current_attrs(void) +{ + if (!current_pstate_driver) + return NULL; + return current_pstate_driver->attr; +} +EXPORT_SYMBOL_GPL(amd_pstate_get_current_attrs); + static struct freq_attr **get_freq_attrs(void) { bool attr_visible[ARRAY_SIZE(amd_pstate_attr_visibility)]; diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 453adfb445f8d..faead0b19a8a4 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -134,4 +134,8 @@ const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode); int amd_pstate_get_status(void); int amd_pstate_update_status(const char *buf, size_t size); +struct freq_attr; + +struct freq_attr **amd_pstate_get_current_attrs(void); + #endif /* _LINUX_AMD_PSTATE_H */ From ebe0684cffbfabc1329723db3fb409cb7dbe2fd9 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:54 +0530 Subject: [PATCH 156/188] Documentation/amd-pstate: List amd_pstate_hw_prefcore sysfs file commit 7e1cf24efba4b59a275e87372267dadcb7fd1850 upstream. Add the missing amd_pstate_hw_prefcore filenames in the sysfs listing example leading to the descriptions of these parameters. Clarify when will the file be visible. Fixes: b96b82d1af7f ("cpufreq: amd-pstate: Add documentation for `amd_pstate_hw_prefcore`") Reviewed-by: Mario Limonciello (AMD) Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index fa6cf11a1ae3f..ff4e63b9e73d3 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -239,6 +239,7 @@ control its functionality at the system level. They are located in the root@hr-test1:/home/ray# ls /sys/devices/system/cpu/cpufreq/policy0/*amd* /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_highest_perf + /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_hw_prefcore /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_lowest_nonlinear_freq /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_max_freq @@ -264,8 +265,9 @@ This attribute is read-only. ``amd_pstate_hw_prefcore`` -Whether the platform supports the preferred core feature and it has been -enabled. This attribute is read-only. +Whether the platform supports the preferred core feature and it has +been enabled. This attribute is read-only. This file is only visible +on platforms which support the preferred core feature. ``amd_pstate_prefcore_ranking`` From 0dfc2a411fbe0f7e92559ed6e4c17083764e29f2 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:55 +0530 Subject: [PATCH 157/188] Documentation/amd-pstate: List amd_pstate_prefcore_ranking sysfs file commit a5bc4c44aeec2920931e17db7f93965fcd69ee2f upstream. Add the missing amd_pstate_prefcore_ranking filenames in the sysfs listing example leading to the descriptions of these parameters. Clarify when will the file be visible. Fixes: 15a2b764ea7c ("amd-pstate: Add missing documentation for `amd_pstate_prefcore_ranking`") Reviewed-by: Mario Limonciello (AMD) Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index ff4e63b9e73d3..842660da76e29 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -242,6 +242,7 @@ control its functionality at the system level. They are located in the /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_hw_prefcore /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_lowest_nonlinear_freq /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_max_freq + /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_prefcore_ranking ``amd_pstate_highest_perf / amd_pstate_max_freq`` @@ -273,7 +274,8 @@ on platforms which support the preferred core feature. The performance ranking of the core. This number doesn't have any unit, but larger numbers are preferred at the time of reading. This can change at -runtime based on platform conditions. This attribute is read-only. +runtime based on platform conditions. This attribute is read-only. This file +is only visible on platforms which support the preferred core feature. ``energy_performance_available_preferences`` From ccba5af90267a561997b4e67e054adcad3555bd0 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 26 Mar 2026 17:17:56 +0530 Subject: [PATCH 158/188] Documentation/amd-pstate: Add documentation for amd_pstate_floor_{freq,count} commit 88d2ca6a68fcc43d65b3b056cb8c481804b100b0 upstream. Add documentation for the sysfs files /sys/devices/system/cpu/cpufreq/policy*/amd_pstate_floor_freq and /sys/devices/system/cpu/cpufreq/policy*/amd_pstate_floor_count. Reviewed-by: Mario Limonciello (AMD) Signed-off-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 842660da76e29..3d68897966d85 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -242,6 +242,8 @@ control its functionality at the system level. They are located in the /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_hw_prefcore /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_lowest_nonlinear_freq /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_max_freq + /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_floor_freq + /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_floor_count /sys/devices/system/cpu/cpufreq/policy0/amd_pstate_prefcore_ranking @@ -277,6 +279,36 @@ larger numbers are preferred at the time of reading. This can change at runtime based on platform conditions. This attribute is read-only. This file is only visible on platforms which support the preferred core feature. +``amd_pstate_floor_freq`` + +The floor frequency associated with each CPU. Userspace can write any +value between ``cpuinfo_min_freq`` and ``scaling_max_freq`` into this +file. When the system is under power or thermal constraints, the +platform firmware will attempt to throttle the CPU frequency to the +value specified in ``amd_pstate_floor_freq`` before throttling it +further. This allows userspace to specify different floor frequencies +to different CPUs. For optimal results, threads of the same core +should have the same floor frequency value. This file is only visible +on platforms that support the CPPC Performance Priority feature. + + +``amd_pstate_floor_count`` + +The number of distinct Floor Performance levels supported by the +platform. For example, if this value is 2, then the number of unique +values obtained from the command ``cat +/sys/devices/system/cpu/cpufreq/policy*/amd_pstate_floor_freq | +sort -n | uniq`` should be at most this number for the behavior +described in ``amd_pstate_floor_freq`` to take effect. A zero value +implies that the platform supports unlimited floor performance levels. +This file is only visible on platforms that support the CPPC +Performance Priority feature. + +**Note**: When ``amd_pstate_floor_count`` is non-zero, the frequency to +which the CPU is throttled under power or thermal constraints is +undefined when the number of unique values of ``amd_pstate_floor_freq`` +across all CPUs in the system exceeds ``amd_pstate_floor_count``. + ``energy_performance_available_preferences`` A list of all the supported EPP preferences that could be used for From 9eb1fe5c7c11b93d7242cd8e45deec041f13326a Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Thu, 26 Mar 2026 14:36:20 -0500 Subject: [PATCH 159/188] cpufreq/amd-pstate: Cache the max frequency in cpudata commit 8cdc494013dfcd48f31eafe19b18fd67c224dd8a upstream. The value of maximum frequency is fixed and never changes. Doing calculations every time based off of perf is unnecessary. Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20260326193620.649441-1-mario.limonciello@amd.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 27 +++++++++------------------ drivers/cpufreq/amd-pstate.h | 2 ++ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b02956f33f075..7162becdadbf7 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -811,15 +811,13 @@ static void amd_pstate_adjust_perf(unsigned int cpu, static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on) { struct amd_cpudata *cpudata = policy->driver_data; - union perf_cached perf = READ_ONCE(cpudata->perf); - u32 nominal_freq, max_freq; + u32 nominal_freq; int ret = 0; nominal_freq = READ_ONCE(cpudata->nominal_freq); - max_freq = perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf); if (on) - policy->cpuinfo.max_freq = max_freq; + policy->cpuinfo.max_freq = cpudata->max_freq; else if (policy->cpuinfo.max_freq > nominal_freq) policy->cpuinfo.max_freq = nominal_freq; @@ -1002,13 +1000,15 @@ static int amd_pstate_init_freq(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->nominal_freq, nominal_freq); + /* max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf */ max_freq = perf_to_freq(perf, nominal_freq, perf.highest_perf); + WRITE_ONCE(cpudata->max_freq, max_freq); + lowest_nonlinear_freq = perf_to_freq(perf, nominal_freq, perf.lowest_nonlinear_perf); WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); /** * Below values need to be initialized correctly, otherwise driver will fail to load - * max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf * lowest_nonlinear_freq is a value between [min_freq, nominal_freq] * Check _CPC in ACPI table objects if any values are incorrect */ @@ -1071,9 +1071,7 @@ static int amd_pstate_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_perf); - policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, - cpudata->nominal_freq, - perf.highest_perf); + policy->cpuinfo.max_freq = policy->max = cpudata->max_freq; policy->driver_data = cpudata; ret = amd_pstate_cppc_enable(policy); @@ -1148,14 +1146,9 @@ static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, char *buf) { - struct amd_cpudata *cpudata; - union perf_cached perf; - - cpudata = policy->driver_data; - perf = READ_ONCE(cpudata->perf); + struct amd_cpudata *cpudata = policy->driver_data; - return sysfs_emit(buf, "%u\n", - perf_to_freq(perf, cpudata->nominal_freq, perf.highest_perf)); + return sysfs_emit(buf, "%u\n", cpudata->max_freq); } static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, @@ -1690,9 +1683,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.min_freq = policy->min = perf_to_freq(perf, cpudata->nominal_freq, perf.lowest_perf); - policy->cpuinfo.max_freq = policy->max = perf_to_freq(perf, - cpudata->nominal_freq, - perf.highest_perf); + policy->cpuinfo.max_freq = policy->max = cpudata->max_freq; policy->driver_data = cpudata; ret = amd_pstate_cppc_enable(policy); diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index faead0b19a8a4..32b8b26ce388f 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -73,6 +73,7 @@ struct amd_aperf_mperf { * @min_limit_freq: Cached value of policy->min (in khz) * @max_limit_freq: Cached value of policy->max (in khz) * @nominal_freq: the frequency (in khz) that mapped to nominal_perf + * @max_freq: in ideal conditions the maximum frequency (in khz) possible frequency * @lowest_nonlinear_freq: the frequency (in khz) that mapped to lowest_nonlinear_perf * @floor_freq: Cached value of the user requested floor_freq * @cur: Difference of Aperf/Mperf/tsc count between last and current sample @@ -103,6 +104,7 @@ struct amd_cpudata { u32 min_limit_freq; u32 max_limit_freq; u32 nominal_freq; + u32 max_freq; u32 lowest_nonlinear_freq; u32 floor_freq; From 4aa543633c80cf87856e72adc69930cb66054c85 Mon Sep 17 00:00:00 2001 From: Ninad Naik Date: Tue, 31 Mar 2026 00:38:55 +0530 Subject: [PATCH 160/188] Documentation: amd-pstate: fix dead links in the reference section commit a362ae6e7e85bca4c870c37085d7793c4beec360 upstream. The links for AMD64 Architecture Programmer's Manual and PPR for AMD Family 19h Model 51h, Revision A1 Processors redirect to a generic page. Update the links to the working ones. Signed-off-by: Ninad Naik Link: https://lore.kernel.org/r/20260330190855.1115304-1-ninadnaik07@gmail.com Acked-by: Mario Limonciello (AMD) Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 3d68897966d85..b2a74c485d506 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -755,13 +755,13 @@ Reference =========== .. [1] AMD64 Architecture Programmer's Manual Volume 2: System Programming, - https://www.amd.com/system/files/TechDocs/24593.pdf + https://docs.amd.com/v/u/en-US/24593_3.44_APM_Vol2 .. [2] Advanced Configuration and Power Interface Specification, https://uefi.org/sites/default/files/resources/ACPI_Spec_6_4_Jan22.pdf .. [3] Processor Programming Reference (PPR) for AMD Family 19h Model 51h, Revision A1 Processors - https://www.amd.com/system/files/TechDocs/56569-A1-PUB.zip + https://docs.amd.com/v/u/en-US/56569-A1-PUB_3.03 .. [4] Linux Kernel Selftests, https://www.kernel.org/doc/html/latest/dev-tools/kselftest.html From a6862ec7922b6e7382b9017fedef851275f6d9d6 Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Thu, 9 Oct 2025 11:17:51 -0500 Subject: [PATCH 161/188] cpufreq/amd-pstate: Use sysfs_match_string() for epp commit 7e17f48667b6707593fc215cbe025157920934f1 upstream. Rather than scanning the buffer and manually matching the string use the sysfs macros. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 7162becdadbf7..75e9d2e540e84 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -119,7 +119,6 @@ static const char * const energy_perf_strings[] = { [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", [EPP_INDEX_POWERSAVE] = "power", - NULL }; static unsigned int epp_values[] = { @@ -1203,16 +1202,15 @@ static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, static ssize_t show_energy_performance_available_preferences( struct cpufreq_policy *policy, char *buf) { - int i = 0; - int offset = 0; + int offset = 0, i; struct amd_cpudata *cpudata = policy->driver_data; if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) return sysfs_emit_at(buf, offset, "%s\n", energy_perf_strings[EPP_INDEX_PERFORMANCE]); - while (energy_perf_strings[i] != NULL) - offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); + for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++) + offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]); offset += sysfs_emit_at(buf, offset, "\n"); @@ -1223,15 +1221,10 @@ static ssize_t store_energy_performance_preference( struct cpufreq_policy *policy, const char *buf, size_t count) { struct amd_cpudata *cpudata = policy->driver_data; - char str_preference[21]; ssize_t ret; u8 epp; - ret = sscanf(buf, "%20s", str_preference); - if (ret != 1) - return -EINVAL; - - ret = match_string(energy_perf_strings, -1, str_preference); + ret = sysfs_match_string(energy_perf_strings, buf); if (ret < 0) return -EINVAL; From 5be7c53deefc53cf04c6856b80e9a588a62bab88 Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Sun, 29 Mar 2026 15:38:07 -0500 Subject: [PATCH 162/188] cpufreq/amd-pstate: Add dynamic energy performance preference commit e30ca6dd5345c5b8ba05f346a8e81105352fe571 upstream. Dynamic energy performance preference changes the EPP profile based on whether the machine is running on AC or DC power. A notification chain from the power supply core is used to adjust EPP values on plug in or plug out events. When enabled, the driver exposes a sysfs toggle for dynamic EPP, blocks manual writes to energy_performance_preference while it "owns" the EPP updates. For non-server systems: * the default EPP for AC mode is `performance`. * the default EPP for DC mode is `balance_performance`. For server systems dynamic EPP is mostly a no-op. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 18 ++- drivers/cpufreq/Kconfig.x86 | 13 +- drivers/cpufreq/amd-pstate.c | 128 +++++++++++++++++++- drivers/cpufreq/amd-pstate.h | 10 +- 4 files changed, 160 insertions(+), 9 deletions(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index b2a74c485d506..3f2e213593c84 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -325,12 +325,28 @@ and user can change current preference according to energy or performance needs Please get all support profiles list from ``energy_performance_available_preferences`` attribute, all the profiles are integer values defined between 0 to 255 when EPP feature is enabled by platform -firmware, if EPP feature is disabled, driver will ignore the written value +firmware, but if the dynamic EPP feature is enabled, driver will block writes. This attribute is read-write. Other performance and frequency values can be read back from ``/sys/devices/system/cpu/cpuX/acpi_cppc/``, see :ref:`cppc_sysfs`. +Dynamic energy performance profile +================================== +The amd-pstate driver supports dynamically selecting the energy performance +profile based on whether the machine is running on AC or DC power. + +Whether this behavior is enabled by default depends on the kernel +config option `CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP`. This behavior can also be overridden +at runtime by the sysfs file ``/sys/devices/system/cpu/cpufreq/policyX/dynamic_epp``. + +When set to enabled, the driver will select a different energy performance +profile when the machine is running on battery or AC power. +When set to disabled, the driver will not change the energy performance profile +based on the power source and will not react to user desired power state. + +Attempting to manually write to the ``energy_performance_preference`` sysfs +file will fail when ``dynamic_epp`` is enabled. ``amd-pstate`` vs ``acpi-cpufreq`` ====================================== diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 16763364a80a0..53e12ae98f68a 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -68,6 +68,18 @@ config X86_AMD_PSTATE_DEFAULT_MODE For details, take a look at: . +config X86_AMD_PSTATE_DYNAMIC_EPP + bool "AMD Processor P-State dynamic EPP support" + depends on X86_AMD_PSTATE + default n + help + Allow the kernel to dynamically change the energy performance + value from events like ACPI platform profile and AC adapter plug + events. + + This feature can also be changed at runtime, this configuration + option only sets the kernel default value behavior. + config X86_AMD_PSTATE_UT tristate "selftest for AMD Processor P-State driver" depends on X86 && ACPI_PROCESSOR @@ -350,4 +362,3 @@ config CPUFREQ_ARCH_CUR_FREQ cpuinfo_avg_freq attribute, by enabling this option, scaling_cur_freq maintains the provision of a counter based frequency, for compatibility reasons. - diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 75e9d2e540e84..7c509c321ada4 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,11 @@ static struct cpufreq_driver amd_pstate_driver; static struct cpufreq_driver amd_pstate_epp_driver; static int cppc_state = AMD_PSTATE_UNDEFINED; static bool amd_pstate_prefcore = true; +#ifdef CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP +static bool dynamic_epp = CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP; +#else +static bool dynamic_epp; +#endif static struct quirk_entry *quirks; /* @@ -1135,6 +1141,73 @@ static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) kfree(cpudata); } +static int amd_pstate_get_balanced_epp(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + if (power_supply_is_system_supplied()) + return cpudata->epp_default_ac; + else + return cpudata->epp_default_dc; +} + +static int amd_pstate_power_supply_notifier(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct amd_cpudata *cpudata = container_of(nb, struct amd_cpudata, power_nb); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); + u8 epp; + int ret; + + if (event != PSY_EVENT_PROP_CHANGED) + return NOTIFY_OK; + + epp = amd_pstate_get_balanced_epp(policy); + + ret = amd_pstate_set_epp(policy, epp); + if (ret) + pr_warn("Failed to set CPU %d EPP %u: %d\n", cpudata->cpu, epp, ret); + + return NOTIFY_OK; +} +static void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + if (cpudata->power_nb.notifier_call) + power_supply_unreg_notifier(&cpudata->power_nb); + cpudata->dynamic_epp = false; +} + +static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + int ret; + u8 epp; + + epp = amd_pstate_get_balanced_epp(policy); + ret = amd_pstate_set_epp(policy, epp); + if (ret) + return ret; + + /* only enable notifier if things will actually change */ + if (cpudata->epp_default_ac != cpudata->epp_default_dc) { + cpudata->power_nb.notifier_call = amd_pstate_power_supply_notifier; + ret = power_supply_reg_notifier(&cpudata->power_nb); + if (ret) + goto cleanup; + } + + cpudata->dynamic_epp = true; + + return 0; + +cleanup: + amd_pstate_clear_dynamic_epp(policy); + + return ret; +} + /* Sysfs attributes */ /* @@ -1224,14 +1297,19 @@ static ssize_t store_energy_performance_preference( ssize_t ret; u8 epp; + if (cpudata->dynamic_epp) { + pr_debug("EPP cannot be set when dynamic EPP is enabled\n"); + return -EBUSY; + } + ret = sysfs_match_string(energy_perf_strings, buf); if (ret < 0) return -EINVAL; - if (!ret) - epp = cpudata->epp_default; - else + if (ret) epp = epp_values[ret]; + else + epp = amd_pstate_get_balanced_epp(policy); if (epp > 0 && policy->policy == CPUFREQ_POLICY_PERFORMANCE) { pr_debug("EPP cannot be set under performance policy\n"); @@ -1239,6 +1317,8 @@ static ssize_t store_energy_performance_preference( } ret = amd_pstate_set_epp(policy, epp); + if (ret) + return ret; return ret ? ret : count; } @@ -1601,12 +1681,42 @@ static ssize_t prefcore_show(struct device *dev, return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); } +static ssize_t dynamic_epp_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%s\n", str_enabled_disabled(dynamic_epp)); +} + +static ssize_t dynamic_epp_store(struct device *a, struct device_attribute *b, + const char *buf, size_t count) +{ + bool enabled; + int ret; + + ret = kstrtobool(buf, &enabled); + if (ret) + return ret; + + if (dynamic_epp == enabled) + return -EINVAL; + + /* reinitialize with desired dynamic EPP value */ + dynamic_epp = enabled; + ret = amd_pstate_change_driver_mode(cppc_state); + if (ret) + dynamic_epp = false; + + return ret ? ret : count; +} + static DEVICE_ATTR_RW(status); static DEVICE_ATTR_RO(prefcore); +static DEVICE_ATTR_RW(dynamic_epp); static struct attribute *pstate_global_attributes[] = { &dev_attr_status.attr, &dev_attr_prefcore.attr, + &dev_attr_dynamic_epp.attr, NULL }; @@ -1696,13 +1806,17 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (amd_pstate_acpi_pm_profile_server() || amd_pstate_acpi_pm_profile_undefined()) { policy->policy = CPUFREQ_POLICY_PERFORMANCE; - cpudata->epp_default = amd_pstate_get_epp(cpudata); + cpudata->epp_default_ac = cpudata->epp_default_dc = amd_pstate_get_epp(cpudata); } else { policy->policy = CPUFREQ_POLICY_POWERSAVE; - cpudata->epp_default = AMD_CPPC_EPP_BALANCE_PERFORMANCE; + cpudata->epp_default_ac = AMD_CPPC_EPP_PERFORMANCE; + cpudata->epp_default_dc = AMD_CPPC_EPP_BALANCE_PERFORMANCE; } - ret = amd_pstate_set_epp(policy, cpudata->epp_default); + if (dynamic_epp) + ret = amd_pstate_set_dynamic_epp(policy); + else + ret = amd_pstate_set_epp(policy, amd_pstate_get_balanced_epp(policy)); if (ret) goto free_cpudata1; @@ -1734,6 +1848,8 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); + if (cpudata->dynamic_epp) + amd_pstate_clear_dynamic_epp(policy); kfree(cpudata); policy->driver_data = NULL; } diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 32b8b26ce388f..d929ae3163b3d 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -85,6 +85,11 @@ struct amd_aperf_mperf { * AMD P-State driver supports preferred core featue. * @epp_cached: Cached CPPC energy-performance preference value * @policy: Cpufreq policy value + * @suspended: If CPU core if offlined + * @epp_default_ac: Default EPP value for AC power source + * @epp_default_dc: Default EPP value for DC power source + * @dynamic_epp: Whether dynamic EPP is enabled + * @power_nb: Notifier block for power events * * The amd_cpudata is key private data for each CPU thread in AMD P-State, and * represents all the attributes and goals that AMD P-State requests at runtime. @@ -118,7 +123,10 @@ struct amd_cpudata { /* EPP feature related attributes*/ u32 policy; bool suspended; - u8 epp_default; + u8 epp_default_ac; + u8 epp_default_dc; + bool dynamic_epp; + struct notifier_block power_nb; }; /* From 33f39bea5b14c5dc199abf653ce757431e531109 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:05:01 +0800 Subject: [PATCH 163/188] Documentation: amd-pstate: introduce amd-pstate preferred core commit 3a004e1fee4ba3d37976c1a9707869acc8d60b55 upstream. Introduce amd-pstate preferred core. check preferred core state set by the kernel parameter: $ cat /sys/devices/system/cpu/amd-pstate/prefcore Tested-by: Oleksandr Natalenko Reviewed-by: Wyes Karny Reviewed-by: Mario Limonciello Reviewed-by: Huang Rui Reviewed-by: Perry Yuan Signed-off-by: Meng Li Signed-off-by: Rafael J. Wysocki Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 59 ++++++++++++++++++++- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 3f2e213593c84..6fff511a104ca 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -363,8 +363,8 @@ platforms. The AMD P-States mechanism is the more performance and energy efficiency frequency management method on AMD processors. -AMD Pstate Driver Operation Modes -================================= +``amd-pstate`` Driver Operation Modes +====================================== ``amd_pstate`` CPPC has 3 operation modes: autonomous (active) mode, non-autonomous (passive) mode and guided autonomous (guided) mode. @@ -416,6 +416,48 @@ is activated. In this mode, driver requests minimum and maximum performance level and the platform autonomously selects a performance level in this range and appropriate to the current workload. +``amd-pstate`` Preferred Core +================================= + +The core frequency is subjected to the process variation in semiconductors. +Not all cores are able to reach the maximum frequency respecting the +infrastructure limits. Consequently, AMD has redefined the concept of +maximum frequency of a part. This means that a fraction of cores can reach +maximum frequency. To find the best process scheduling policy for a given +scenario, OS needs to know the core ordering informed by the platform through +highest performance capability register of the CPPC interface. + +``amd-pstate`` preferred core enables the scheduler to prefer scheduling on +cores that can achieve a higher frequency with lower voltage. The preferred +core rankings can dynamically change based on the workload, platform conditions, +thermals and ageing. + +The priority metric will be initialized by the ``amd-pstate`` driver. The ``amd-pstate`` +driver will also determine whether or not ``amd-pstate`` preferred core is +supported by the platform. + +``amd-pstate`` driver will provide an initial core ordering when the system boots. +The platform uses the CPPC interfaces to communicate the core ranking to the +operating system and scheduler to make sure that OS is choosing the cores +with highest performance firstly for scheduling the process. When ``amd-pstate`` +driver receives a message with the highest performance change, it will +update the core ranking and set the cpu's priority. + +``amd-pstate`` Preferred Core Switch +================================= +Kernel Parameters +----------------- + +``amd-pstate`` peferred core`` has two states: enable and disable. +Enable/disable states can be chosen by different kernel parameters. +Default enable ``amd-pstate`` preferred core. + +``amd_prefcore=disable`` + +For systems that support ``amd-pstate`` preferred core, the core rankings will +always be advertised by the platform. But OS can choose to ignore that via the +kernel parameter ``amd_prefcore=disable``. + User Space Interface in ``sysfs`` - General =========================================== @@ -448,6 +490,19 @@ control its functionality at the system level. They are located in the to the operation mode represented by that string - or to be unregistered in the "disable" case. +``prefcore`` + Preferred core state of the driver: "enabled" or "disabled". + + "enabled" + Enable the ``amd-pstate`` preferred core. + + "disabled" + Disable the ``amd-pstate`` preferred core + + + This attribute is read-only to check the state of preferred core set + by the kernel parameter. + ``cpupower`` tool support for ``amd-pstate`` =============================================== From 1412ff4d2053664a76a2b01507f29e47f7544331 Mon Sep 17 00:00:00 2001 From: Meng Li Date: Fri, 19 Jan 2024 17:05:02 +0800 Subject: [PATCH 164/188] Documentation: introduce amd-pstate preferrd core mode kernel command line options commit dfddf34a3f0d45483f5b3e46c2e7bda173796f1b upstream. amd-pstate driver support enable/disable preferred core. Default enabled on platforms supporting amd-pstate preferred core. Disable amd-pstate preferred core with "amd_prefcore=disable" added to the kernel command line. Signed-off-by: Meng Li Reviewed-by: Mario Limonciello Reviewed-by: Wyes Karny Reviewed-by: Huang Rui Reviewed-by: Perry Yuan Tested-by: Oleksandr Natalenko Signed-off-by: Rafael J. Wysocki Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- Documentation/admin-guide/kernel-parameters.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index e5a7ce494f766..f0743a67301f6 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -363,6 +363,11 @@ selects a performance level in this range and appropriate to the current workload. + amd_prefcore= + [X86] + disable + Disable amd-pstate preferred core. + amijoy.map= [HW,JOY] Amiga joystick support Map of devices attached to JOY0DAT and JOY1DAT Format: , From 0f09e33df2afe6a06ace1f1fcf0ea6d059401637 Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Sun, 29 Mar 2026 15:38:08 -0500 Subject: [PATCH 165/188] cpufreq/amd-pstate: add kernel command line to override dynamic epp commit da8afb1c666a4a966f0ab91dc336df4c855bc7b2 upstream. Add `amd_dynamic_epp=enable` and `amd_dynamic_epp=disable` to override the kernel configuration option `CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP` locally. Signed-off-by: Mario Limonciello (AMD) Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: chaithanyaLagisetty Signed-off-by: mohanasv2 --- Documentation/admin-guide/kernel-parameters.txt | 7 +++++++ Documentation/admin-guide/pm/amd-pstate.rst | 7 +++++++ drivers/cpufreq/amd-pstate.c | 11 +++++++++++ 3 files changed, 25 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index f0743a67301f6..16c422c27694f 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -368,6 +368,13 @@ disable Disable amd-pstate preferred core. + amd_dynamic_epp= + [X86] + disable + Disable amd-pstate dynamic EPP. + enable + Enable amd-pstate dynamic EPP. + amijoy.map= [HW,JOY] Amiga joystick support Map of devices attached to JOY0DAT and JOY1DAT Format: , diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 6fff511a104ca..5eee9c0c59b32 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -458,6 +458,13 @@ For systems that support ``amd-pstate`` preferred core, the core rankings will always be advertised by the platform. But OS can choose to ignore that via the kernel parameter ``amd_prefcore=disable``. +``amd_dynamic_epp`` + +When AMD pstate is in auto mode, dynamic EPP will control whether the kernel +autonomously changes the EPP mode. The default is configured by +``CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP`` but can be explicitly enabled with +``amd_dynamic_epp=enable`` or disabled with ``amd_dynamic_epp=disable``. + User Space Interface in ``sysfs`` - General =========================================== diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 7c509c321ada4..6fab892dae9dc 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -2206,8 +2206,19 @@ static int __init amd_prefcore_param(char *str) return 0; } +static int __init amd_dynamic_epp_param(char *str) +{ + if (!strcmp(str, "disable")) + dynamic_epp = false; + if (!strcmp(str, "enable")) + dynamic_epp = true; + + return 0; +} + early_param("amd_pstate", amd_pstate_param); early_param("amd_prefcore", amd_prefcore_param); +early_param("amd_dynamic_epp", amd_dynamic_epp_param); MODULE_AUTHOR("Huang Rui "); MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); From b1749b2e675881bf8339f273301b7596a5695d5c Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Mon, 16 Mar 2026 08:18:49 +0000 Subject: [PATCH 166/188] cpufreq: Pass the policy to cpufreq_driver->adjust_perf() commit c03791085adcd61fa9b766ab303c7d0941d7378d upstream. cpufreq_cpu_get() can sleep on PREEMPT_RT in presence of concurrent writer(s), however amd-pstate depends on fetching the cpudata via the policy's driver data which necessitates grabbing the reference. Since schedutil governor can call "cpufreq_driver->update_perf()" during sched_tick/enqueue/dequeue with rq_lock held and IRQs disabled, fetching the policy object using the cpufreq_cpu_get() helper in the scheduler fast-path leads to "BUG: scheduling while atomic" on PREEMPT_RT [1]. Pass the cached cpufreq policy object in sg_policy to the update_perf() instead of just the CPU. The CPU can be inferred using "policy->cpu". The lifetime of cpufreq_policy object outlasts that of the governor and the cpufreq driver (allocated when the CPU is onlined and only reclaimed when the CPU is offlined / the CPU device is removed) which makes it safe to be referenced throughout the governor's lifetime. Closes:https://lore.kernel.org/all/20250731092316.3191-1-spasswolf@web.de/ [1] [Backport Changes] Upstream changes to rust/kernel/cpufreq.rs were excluded. The Rust cpufreq abstraction does not yet exist in the Anolis 6.6 kernel tree; only the C cpufreq core (drivers/cpufreq/cpufreq.c, intel_pstate.c, amd-pstate.c), the cpufreq schedutil governor, and include/linux/cpufreq.h hunks are applied. Fixes: 1d215f0319c2 ("cpufreq: amd-pstate: Add fast switch function for AMD P-State") Reported-by: Bert Karwatzki Acked-by: Viresh Kumar Signed-off-by: K Prateek Nayak Acked-by: Gary Guo # Rust Reviewed-by: Gautham R. Shenoy Reviewed-by: Zhongqiu Han Link: https://lore.kernel.org/r/20260316081849.19368-3-kprateek.nayak@amd.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 3 +-- drivers/cpufreq/cpufreq.c | 6 +++--- drivers/cpufreq/intel_pstate.c | 4 ++-- include/linux/cpufreq.h | 4 ++-- kernel/sched/cpufreq_schedutil.c | 5 +++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 6fab892dae9dc..4a0725b75e323 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -772,13 +772,12 @@ static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, return policy->cur; } -static void amd_pstate_adjust_perf(unsigned int cpu, +static void amd_pstate_adjust_perf(struct cpufreq_policy *policy, unsigned long _min_perf, unsigned long target_perf, unsigned long capacity) { u8 max_perf, min_perf, des_perf, cap_perf; - struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); struct amd_cpudata *cpudata; union perf_cached perf; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index d874bb230c1a8..687ff25a6889d 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2264,7 +2264,7 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); /** * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go. - * @cpu: Target CPU. + * @policy: cpufreq policy object of the target CPU. * @min_perf: Minimum (required) performance level (units of @capacity). * @target_perf: Target (desired) performance level (units of @capacity). * @capacity: Capacity of the target CPU. @@ -2283,12 +2283,12 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); * parallel with either ->target() or ->target_index() or ->fast_switch() for * the same CPU. */ -void cpufreq_driver_adjust_perf(unsigned int cpu, +void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, unsigned long capacity) { - cpufreq_driver->adjust_perf(cpu, min_perf, target_perf, capacity); + cpufreq_driver->adjust_perf(policy, min_perf, target_perf, capacity); } /** diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index b26f8c3c3e00c..c27b80d2fafde 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -3147,12 +3147,12 @@ static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy, return target_pstate * cpu->pstate.scaling; } -static void intel_cpufreq_adjust_perf(unsigned int cpunum, +static void intel_cpufreq_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, unsigned long capacity) { - struct cpudata *cpu = all_cpu_data[cpunum]; + struct cpudata *cpu = all_cpu_data[policy->cpu]; u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached); int old_pstate = cpu->pstate.current_pstate; int cap_pstate, min_pstate, max_pstate, target_pstate; diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index a0d52195a52d4..28d68b5a0e2ba 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -364,7 +364,7 @@ struct cpufreq_driver { * conditions) scale invariance can be disabled, which causes the * schedutil governor to fall back to the latter. */ - void (*adjust_perf)(unsigned int cpu, + void (*adjust_perf)(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, unsigned long capacity); @@ -617,7 +617,7 @@ struct cpufreq_governor { /* Pass a target to the cpufreq driver */ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq); -void cpufreq_driver_adjust_perf(unsigned int cpu, +void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, unsigned long capacity); diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 2e6600647d6b5..0d2dcf556eb86 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -458,6 +458,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, unsigned int flags) { struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); + struct sugov_policy *sg_policy = sg_cpu->sg_policy; unsigned long prev_util = sg_cpu->util; unsigned long max_cap; @@ -479,10 +480,10 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util) sg_cpu->util = prev_util; - cpufreq_driver_adjust_perf(sg_cpu->cpu, sg_cpu->bw_min, + cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min, sg_cpu->util, max_cap); - sg_cpu->sg_policy->last_freq_update_time = time; + sg_policy->last_freq_update_time = time; } static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time) From a8cbcb4c877cc95b2884720595326508a497f4ea Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Mon, 16 Mar 2026 08:18:48 +0000 Subject: [PATCH 167/188] cpufreq/amd-pstate: Pass the policy to amd_pstate_update() commit 86d71f1d7686cecebbafb371ad58c6ad7f80a93a upstream. All callers of amd_pstate_update() already have a reference to the cpufreq_policy object. Pass the entire policy object and grab the cpudata using "policy->driver_data" instead of passing the cpudata and unnecessarily grabbing another read-side reference to the cpufreq policy object when it is already available in the caller. No functional changes intended. Reviewed-by: Mario Limonciello (AMD) Acked-by: Viresh Kumar Signed-off-by: K Prateek Nayak Reviewed-by: Gautham R. Shenoy Link: https://lore.kernel.org/r/20260316081849.19368-2-kprateek.nayak@amd.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 4a0725b75e323..34d4a6fff93ee 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -637,15 +637,12 @@ static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) return true; } -static void amd_pstate_update(struct amd_cpudata *cpudata, u8 min_perf, +static void amd_pstate_update(struct cpufreq_policy *policy, u8 min_perf, u8 des_perf, u8 max_perf, bool fast_switch, int gov_flags) { - struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); + struct amd_cpudata *cpudata = policy->driver_data; union perf_cached perf = READ_ONCE(cpudata->perf); - if (!policy) - return; - /* limit the max perf when core performance boost feature is disabled */ if (!cpudata->boost_supported) max_perf = min_t(u8, perf.nominal_perf, max_perf); @@ -747,7 +744,7 @@ static int amd_pstate_update_freq(struct cpufreq_policy *policy, if (!fast_switch) cpufreq_freq_transition_begin(policy, &freqs); - amd_pstate_update(cpudata, perf.min_limit_perf, des_perf, + amd_pstate_update(policy, perf.min_limit_perf, des_perf, perf.max_limit_perf, fast_switch, policy->governor->flags); @@ -808,7 +805,7 @@ static void amd_pstate_adjust_perf(struct cpufreq_policy *policy, if (max_perf < min_perf) max_perf = min_perf; - amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, + amd_pstate_update(policy, min_perf, des_perf, max_perf, true, policy->governor->flags); } From 1c4b7b586c2f3f2205bb4bf3468b9f79e5e90cba Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Thu, 9 Oct 2025 11:17:56 -0500 Subject: [PATCH 168/188] cpufreq/amd-pstate: Add static asserts for EPP indices commit 077f23573d29d063a950e90aa77c8e1f79580147 upstream. In case a new index is introduced add a static assert to make sure that strings and values are updated. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 34d4a6fff93ee..aef06430066b3 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -117,6 +117,7 @@ enum energy_perf_value_index { EPP_INDEX_BALANCE_PERFORMANCE, EPP_INDEX_BALANCE_POWERSAVE, EPP_INDEX_POWERSAVE, + EPP_INDEX_MAX, }; static const char * const energy_perf_strings[] = { @@ -126,6 +127,7 @@ static const char * const energy_perf_strings[] = { [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", [EPP_INDEX_POWERSAVE] = "power", }; +static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); static unsigned int epp_values[] = { [EPP_INDEX_DEFAULT] = 0, @@ -134,6 +136,7 @@ static unsigned int epp_values[] = { [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, }; +static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX); typedef int (*cppc_mode_transition_fn)(int); From 5f8c16d3d17d2130ab5d0facdb02cfad312df7e7 Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Sun, 29 Mar 2026 15:38:10 -0500 Subject: [PATCH 169/188] cpufreq/amd-pstate: Add support for raw EPP writes commit 6927f21852f38db2975b5d5539cbe5241c25a99b upstream. The energy performance preference field of the CPPC request MSR supports values from 0 to 255, but the strings only offer 4 values. The other values are useful for tuning the performance of some workloads. Add support for writing the raw energy performance preference value to the sysfs file. If the last value written was an integer then an integer will be returned. If the last value written was a string then a string will be returned. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- Documentation/admin-guide/pm/amd-pstate.rst | 16 ++++++--- drivers/cpufreq/amd-pstate.c | 36 +++++++++++++++------ drivers/cpufreq/amd-pstate.h | 1 + 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 5eee9c0c59b32..13e2167236889 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -316,16 +316,22 @@ A list of all the supported EPP preferences that could be used for These profiles represent different hints that are provided to the low-level firmware about the user's desired energy vs efficiency tradeoff. ``default`` represents the epp value is set by platform -firmware. This attribute is read-only. +firmware. ``custom`` designates that integer values 0-255 may be written +as well. This attribute is read-only. ``energy_performance_preference`` The current energy performance preference can be read from this attribute. and user can change current preference according to energy or performance needs -Please get all support profiles list from -``energy_performance_available_preferences`` attribute, all the profiles are -integer values defined between 0 to 255 when EPP feature is enabled by platform -firmware, but if the dynamic EPP feature is enabled, driver will block writes. +Coarse named profiles are available in the attribute +``energy_performance_available_preferences``. +Users can also write individual integer values between 0 to 255. +When dynamic EPP is enabled, writes to energy_performance_preference are blocked +even when EPP feature is enabled by platform firmware. Lower epp values shift the bias +towards improved performance while a higher epp value shifts the bias towards +power-savings. The exact impact can change from one platform to the other. +If a valid integer was last written, then a number will be returned on future reads. +If a valid string was last written then a string will be returned on future reads. This attribute is read-write. Other performance and frequency values can be read back from diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index aef06430066b3..18b70f309a44e 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -110,6 +110,7 @@ static struct quirk_entry *quirks; * 2 balance_performance * 3 balance_power * 4 power + * 5 custom (for raw EPP values) */ enum energy_perf_value_index { EPP_INDEX_DEFAULT = 0, @@ -117,6 +118,7 @@ enum energy_perf_value_index { EPP_INDEX_BALANCE_PERFORMANCE, EPP_INDEX_BALANCE_POWERSAVE, EPP_INDEX_POWERSAVE, + EPP_INDEX_CUSTOM, EPP_INDEX_MAX, }; @@ -126,6 +128,7 @@ static const char * const energy_perf_strings[] = { [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", [EPP_INDEX_POWERSAVE] = "power", + [EPP_INDEX_CUSTOM] = "custom", }; static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); @@ -136,7 +139,7 @@ static unsigned int epp_values[] = { [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, }; -static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX); +static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX - 1); typedef int (*cppc_mode_transition_fn)(int); @@ -1294,6 +1297,7 @@ static ssize_t store_energy_performance_preference( { struct amd_cpudata *cpudata = policy->driver_data; ssize_t ret; + bool raw_epp = false; u8 epp; if (cpudata->dynamic_epp) { @@ -1301,14 +1305,21 @@ static ssize_t store_energy_performance_preference( return -EBUSY; } - ret = sysfs_match_string(energy_perf_strings, buf); - if (ret < 0) - return -EINVAL; - - if (ret) - epp = epp_values[ret]; - else - epp = amd_pstate_get_balanced_epp(policy); + /* + * if the value matches a number, use that, otherwise see if + * matches an index in the energy_perf_strings array + */ + ret = kstrtou8(buf, 0, &epp); + raw_epp = !ret; + if (ret) { + ret = sysfs_match_string(energy_perf_strings, buf); + if (ret < 0 || ret == EPP_INDEX_CUSTOM) + return -EINVAL; + if (ret) + epp = epp_values[ret]; + else + epp = amd_pstate_get_balanced_epp(policy); + } if (epp > 0 && policy->policy == CPUFREQ_POLICY_PERFORMANCE) { pr_debug("EPP cannot be set under performance policy\n"); @@ -1319,7 +1330,9 @@ static ssize_t store_energy_performance_preference( if (ret) return ret; - return ret ? ret : count; + cpudata->raw_epp = raw_epp; + + return count; } static ssize_t show_energy_performance_preference( @@ -1330,6 +1343,9 @@ static ssize_t show_energy_performance_preference( epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); + if (cpudata->raw_epp) + return sysfs_emit(buf, "%u\n", epp); + switch (epp) { case AMD_CPPC_EPP_PERFORMANCE: preference = EPP_INDEX_PERFORMANCE; diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index d929ae3163b3d..ee2d83335cd83 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -126,6 +126,7 @@ struct amd_cpudata { u8 epp_default_ac; u8 epp_default_dc; bool dynamic_epp; + bool raw_epp; struct notifier_block power_nb; }; From 3e8a5b842c8a43413235d5f269139c6ddf48b2c6 Mon Sep 17 00:00:00 2001 From: "Mario Limonciello (AMD)" Date: Sun, 29 Mar 2026 15:38:11 -0500 Subject: [PATCH 170/188] cpufreq/amd-pstate-ut: Add a unit test for raw EPP commit 7e173bc310d2b1df018edc66334a5304305889a2 upstream. Ensure that all supported raw EPP values work properly. Export the driver helpers used by the test module so the test can drive raw EPP writes and temporarily disable dynamic EPP while it runs. Reviewed-by: Gautham R. Shenoy Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 109 ++++++++++++++++++++++++++++++++ drivers/cpufreq/amd-pstate.c | 13 ++-- drivers/cpufreq/amd-pstate.h | 4 ++ 3 files changed, 121 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 092dd87a01fda..18dec0caaec65 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -39,6 +40,7 @@ static char *test_list; module_param(test_list, charp, 0444); MODULE_PARM_DESC(test_list, "Comma-delimited list of tests to run (empty means run all tests)"); +DEFINE_FREE(cleanup_page, void *, if (_T) free_page((unsigned long)_T)) struct amd_pstate_ut_struct { const char *name; @@ -52,6 +54,7 @@ static int amd_pstate_ut_acpi_cpc_valid(u32 index); static int amd_pstate_ut_check_enabled(u32 index); static int amd_pstate_ut_check_perf(u32 index); static int amd_pstate_ut_check_freq(u32 index); +static int amd_pstate_ut_epp(u32 index); static int amd_pstate_ut_check_driver(u32 index); static int amd_pstate_ut_check_freq_attrs(u32 index); @@ -60,6 +63,7 @@ static struct amd_pstate_ut_struct amd_pstate_ut_cases[] = { {"amd_pstate_ut_check_enabled", amd_pstate_ut_check_enabled }, {"amd_pstate_ut_check_perf", amd_pstate_ut_check_perf }, {"amd_pstate_ut_check_freq", amd_pstate_ut_check_freq }, + {"amd_pstate_ut_epp", amd_pstate_ut_epp }, {"amd_pstate_ut_check_driver", amd_pstate_ut_check_driver }, {"amd_pstate_ut_check_freq_attrs", amd_pstate_ut_check_freq_attrs }, }; @@ -266,6 +270,111 @@ static int amd_pstate_set_mode(enum amd_pstate_mode mode) return amd_pstate_update_status(mode_str, strlen(mode_str)); } +static int amd_pstate_ut_epp(u32 index) +{ + struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; + char *buf __free(cleanup_page) = NULL; + static const char * const epp_strings[] = { + "performance", + "balance_performance", + "balance_power", + "power", + }; + struct amd_cpudata *cpudata; + enum amd_pstate_mode orig_mode; + bool orig_dynamic_epp; + int ret, cpu = 0; + int i; + u16 epp; + + policy = cpufreq_cpu_get(cpu); + if (!policy) + return -ENODEV; + + cpudata = policy->driver_data; + orig_mode = amd_pstate_get_status(); + orig_dynamic_epp = cpudata->dynamic_epp; + + /* disable dynamic EPP before running test */ + if (cpudata->dynamic_epp) { + pr_debug("Dynamic EPP is enabled, disabling it\n"); + amd_pstate_clear_dynamic_epp(policy); + } + + buf = (char *)__get_free_page(GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = amd_pstate_set_mode(AMD_PSTATE_ACTIVE); + if (ret) + goto out; + + for (epp = 0; epp <= U8_MAX; epp++) { + u8 val; + + /* write all EPP values */ + memset(buf, 0, PAGE_SIZE); + snprintf(buf, PAGE_SIZE, "%d", epp); + ret = store_energy_performance_preference(policy, buf, strlen(buf)); + if (ret < 0) + goto out; + + /* check if the EPP value reads back correctly for raw numbers */ + memset(buf, 0, PAGE_SIZE); + ret = show_energy_performance_preference(policy, buf); + if (ret < 0) + goto out; + strreplace(buf, '\n', '\0'); + ret = kstrtou8(buf, 0, &val); + if (!ret && epp != val) { + pr_err("Raw EPP value mismatch: %d != %d\n", epp, val); + ret = -EINVAL; + goto out; + } + } + + for (i = 0; i < ARRAY_SIZE(epp_strings); i++) { + memset(buf, 0, PAGE_SIZE); + snprintf(buf, PAGE_SIZE, "%s", epp_strings[i]); + ret = store_energy_performance_preference(policy, buf, strlen(buf)); + if (ret < 0) + goto out; + + memset(buf, 0, PAGE_SIZE); + ret = show_energy_performance_preference(policy, buf); + if (ret < 0) + goto out; + strreplace(buf, '\n', '\0'); + + if (strcmp(buf, epp_strings[i])) { + pr_err("String EPP value mismatch: %s != %s\n", buf, epp_strings[i]); + ret = -EINVAL; + goto out; + } + } + + ret = 0; + +out: + if (orig_dynamic_epp) { + int ret2; + + ret2 = amd_pstate_set_mode(AMD_PSTATE_DISABLE); + if (!ret && ret2) + ret = ret2; + } + + if (orig_mode != amd_pstate_get_status()) { + int ret2; + + ret2 = amd_pstate_set_mode(orig_mode); + if (!ret && ret2) + ret = ret2; + } + + return ret; +} + static int amd_pstate_ut_check_driver(u32 index) { enum amd_pstate_mode mode1, mode2 = AMD_PSTATE_DISABLE; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 18b70f309a44e..31dbb7a84f41b 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1172,7 +1172,8 @@ static int amd_pstate_power_supply_notifier(struct notifier_block *nb, return NOTIFY_OK; } -static void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy) + +void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy) { struct amd_cpudata *cpudata = policy->driver_data; @@ -1180,6 +1181,7 @@ static void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy) power_supply_unreg_notifier(&cpudata->power_nb); cpudata->dynamic_epp = false; } +EXPORT_SYMBOL_GPL(amd_pstate_clear_dynamic_epp); static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy) { @@ -1292,8 +1294,8 @@ static ssize_t show_energy_performance_available_preferences( return offset; } -static ssize_t store_energy_performance_preference( - struct cpufreq_policy *policy, const char *buf, size_t count) +ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, + const char *buf, size_t count) { struct amd_cpudata *cpudata = policy->driver_data; ssize_t ret; @@ -1334,9 +1336,9 @@ static ssize_t store_energy_performance_preference( return count; } +EXPORT_SYMBOL_GPL(store_energy_performance_preference); -static ssize_t show_energy_performance_preference( - struct cpufreq_policy *policy, char *buf) +ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf) { struct amd_cpudata *cpudata = policy->driver_data; u8 preference, epp; @@ -1365,6 +1367,7 @@ static ssize_t show_energy_performance_preference( return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); } +EXPORT_SYMBOL_GPL(show_energy_performance_preference); static ssize_t store_amd_pstate_floor_freq(struct cpufreq_policy *policy, const char *buf, size_t count) diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index ee2d83335cd83..b093138b71ed7 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -144,6 +144,10 @@ enum amd_pstate_mode { const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode); int amd_pstate_get_status(void); int amd_pstate_update_status(const char *buf, size_t size); +ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, + const char *buf, size_t count); +ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf); +void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy); struct freq_attr; From ac2aba545bfd868ba9cf96e0885002dccbcb7bac Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 2 Apr 2026 15:56:11 +0530 Subject: [PATCH 171/188] MAINTAINERS: amd-pstate: Step down as maintainer, add Prateek as reviewer commit 9487e2a00e7b3c6f258c5c99953f470eba6fb61d upstream. Mario Limonciello has led amd-pstate maintenance in recent years and has done excellent work. The amd-pstate driver is in good hands with him. I am stepping down as co-maintainer as I move on to other things. Add K Prateek Nayak as a reviewer. He has been actively contributing to the driver including preferred-core and ITMT improvements, and has been helping review amd-pstate patches for a while now. [Backport Changes] The MAINTAINERS entry for the AMD PSTATE DRIVER in this tree does not contain the line "M: Gautham R. Shenoy ", so the hunk that removes it was dropped. Only the upstream change that adds the new reviewer was applied. Signed-off-by: Gautham R. Shenoy Acked-by: K Prateek Nayak Acked-by: Mario Limonciello (AMD) Link: https://lore.kernel.org/r/20260402102611.16519-1-gautham.shenoy@amd.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 2007de0d42b01..7563c6604a99e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1050,6 +1050,7 @@ F: drivers/gpu/drm/amd/pm/ AMD PSTATE DRIVER M: Huang Rui +R: K Prateek Nayak L: linux-pm@vger.kernel.org S: Supported F: Documentation/admin-guide/pm/amd-pstate.rst From e4e4ee0b01056d5fc9a02376905797b4f03235a8 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 4 Nov 2024 19:51:28 +0100 Subject: [PATCH 172/188] cpufreq: intel_pstate: Clear hybrid_max_perf_cpu before driver registration commit a97e293e077a3e8f41e8972e593b34d0052b9e25 upstream. Modify intel_pstate_register_driver() to clear hybrid_max_perf_cpu before calling cpufreq_register_driver(), so that asymmetric CPU capacity scaling is not updated until hybrid_init_cpu_capacity_scaling() runs down the road. This is done in preparation for a subsequent change adding asymmetric CPU capacity computation to the CPU init path to handle CPUs that are initially offline. The information on whether or not hybrid_max_perf_cpu was NULL before it has been cleared is passed to hybrid_init_cpu_capacity_scaling(), so full initialization of CPU capacity scaling can be skipped if it has been carried out already. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Link: https://patch.msgid.link/4616631.LvFx2qVVIh@rjwysocki.net Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/intel_pstate.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index c27b80d2fafde..33ca7d86c3514 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1033,7 +1033,7 @@ static void __hybrid_init_cpu_capacity_scaling(void) hybrid_update_cpu_capacity_scaling(); } -static void hybrid_init_cpu_capacity_scaling(void) +static void hybrid_init_cpu_capacity_scaling(bool refresh) { bool disable_itmt = false; @@ -1044,7 +1044,7 @@ static void hybrid_init_cpu_capacity_scaling(void) * scaling has been enabled already and the driver is just changing the * operation mode. */ - if (hybrid_max_perf_cpu) { + if (refresh) { __hybrid_init_cpu_capacity_scaling(); goto unlock; } @@ -1070,6 +1070,18 @@ static void hybrid_init_cpu_capacity_scaling(void) sched_clear_itmt_support(); } +static bool hybrid_clear_max_perf_cpu(void) +{ + bool ret; + + guard(mutex)(&hybrid_capacity_lock); + + ret = !!hybrid_max_perf_cpu; + hybrid_max_perf_cpu = NULL; + + return ret; +} + static void __intel_pstate_get_hwp_cap(struct cpudata *cpu) { u64 cap; @@ -3338,6 +3350,7 @@ static void intel_pstate_driver_cleanup(void) static int intel_pstate_register_driver(struct cpufreq_driver *driver) { + bool refresh_cpu_cap_scaling; int ret; if (driver == &intel_pstate) @@ -3350,6 +3363,8 @@ static int intel_pstate_register_driver(struct cpufreq_driver *driver) arch_set_max_freq_ratio(global.turbo_disabled); + refresh_cpu_cap_scaling = hybrid_clear_max_perf_cpu(); + intel_pstate_driver = driver; ret = cpufreq_register_driver(intel_pstate_driver); if (ret) { @@ -3359,7 +3374,7 @@ static int intel_pstate_register_driver(struct cpufreq_driver *driver) global.min_perf_pct = min_perf_pct_min(); - hybrid_init_cpu_capacity_scaling(); + hybrid_init_cpu_capacity_scaling(refresh_cpu_cap_scaling); return 0; } From f10f699beeaaaeaaa5d2fc1a8e747cd930756b44 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 4 Nov 2024 16:28:55 -0600 Subject: [PATCH 173/188] ACPI: processor: Move arch_init_invariance_cppc() call later commit b79276dcac9124a79c8cf7cc8fbdd3d4c3c9a7c7 upstream. arch_init_invariance_cppc() is called at the end of acpi_cppc_processor_probe() in order to configure frequency invariance based upon the values from _CPC. This however doesn't work on AMD CPPC shared memory designs that have AMD preferred cores enabled because _CPC needs to be analyzed from all cores to judge if preferred cores are enabled. This issue manifests to users as a warning since commit 21fb59ab4b97 ("ACPI: CPPC: Adjust debug messages in amd_set_max_freq_ratio() to warn"): ``` Could not retrieve highest performance (-19) ``` However the warning isn't the cause of this, it was actually commit 279f838a61f9 ("x86/amd: Detect preferred cores in amd_get_boost_ratio_numerator()") which exposed the issue. To fix this problem, change arch_init_invariance_cppc() into a new weak symbol that is called at the end of acpi_processor_driver_init(). Each architecture that supports it can declare the symbol to override the weak one. Define it for x86, in arch/x86/kernel/acpi/cppc.c, and for all of the architectures using the generic arch_topology.c code. Fixes: 279f838a61f9 ("x86/amd: Detect preferred cores in amd_get_boost_ratio_numerator()") Reported-by: Ivan Shapovalov Closes: https://bugzilla.kernel.org/show_bug.cgi?id=219431 Tested-by: Oleksandr Natalenko Signed-off-by: Mario Limonciello Link: https://patch.msgid.link/20241104222855.3959267-1-superm1@kernel.org [ rjw: Changelog edit ] Signed-off-by: Rafael J. Wysocki Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- arch/arm64/include/asm/topology.h | 4 ---- arch/x86/include/asm/topology.h | 5 ----- arch/x86/kernel/acpi/cppc.c | 7 ++++++- drivers/acpi/cppc_acpi.c | 6 ------ drivers/acpi/processor_driver.c | 9 +++++++++ drivers/base/arch_topology.c | 6 +++++- include/acpi/processor.h | 2 ++ include/linux/arch_topology.h | 4 ---- 8 files changed, 22 insertions(+), 21 deletions(-) diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index a323b109b9c44..fe0b9e389f46b 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -25,10 +25,6 @@ void update_freq_counters_refs(void); #define arch_scale_freq_invariant topology_scale_freq_invariant #define arch_scale_freq_ref topology_get_freq_ref -#ifdef CONFIG_ACPI_CPPC_LIB -#define arch_init_invariance_cppc topology_init_cpu_capacity_cppc -#endif - /* Replace task scheduler's default cpu-invariant accounting */ #define arch_scale_cpu_capacity topology_get_cpu_scale diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 3a7c18851016d..d84d9b6d86781 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -305,9 +305,4 @@ static inline void freq_invariance_set_perf_ratio(u64 ratio, bool turbo_disabled extern void arch_scale_freq_tick(void); #define arch_scale_freq_tick arch_scale_freq_tick -#ifdef CONFIG_ACPI_CPPC_LIB -void init_freq_invariance_cppc(void); -#define arch_init_invariance_cppc init_freq_invariance_cppc -#endif - #endif /* _ASM_X86_TOPOLOGY_H */ diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index 7a6615f157a8d..e774617e1ad5c 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -117,7 +117,7 @@ static void amd_set_max_freq_ratio(void) static DEFINE_MUTEX(freq_invariance_lock); -void init_freq_invariance_cppc(void) +static inline void init_freq_invariance_cppc(void) { static bool init_done; @@ -134,6 +134,11 @@ void init_freq_invariance_cppc(void) mutex_unlock(&freq_invariance_lock); } +void acpi_processor_init_invariance_cppc(void) +{ + init_freq_invariance_cppc(); +} + /* * Get the highest performance register value. * @cpu: CPU from which to get highest performance. diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 16ac219ae6fe5..8208fa9dd5214 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -669,10 +669,6 @@ static int pcc_data_alloc(int pcc_ss_id) * ) */ -#ifndef arch_init_invariance_cppc -static inline void arch_init_invariance_cppc(void) { } -#endif - /** * acpi_cppc_processor_probe - Search for per CPU _CPC objects. * @pr: Ptr to acpi_processor containing this CPU's logical ID. @@ -901,8 +897,6 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) goto out_free; } - arch_init_invariance_cppc(); - kfree(output.pointer); return 0; diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index 4bd16b3f07814..353c30a2d5b2c 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -250,6 +250,9 @@ static struct notifier_block acpi_processor_notifier_block = { .notifier_call = acpi_processor_notifier, }; +void __weak acpi_processor_init_invariance_cppc(void) +{ } + /* * We keep the driver loaded even when ACPI is not running. * This is needed for the powernow-k8 driver, that works even without @@ -283,6 +286,12 @@ static int __init acpi_processor_driver_init(void) NULL, acpi_soft_cpu_dead); acpi_processor_throttling_init(); + + /* + * Frequency invariance calculations on AMD platforms can't be run until + * after acpi_cppc_processor_probe() has been called for all online CPUs + */ + acpi_processor_init_invariance_cppc(); return 0; err: driver_unregister(&acpi_processor_driver); diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 8d6ed7ec70e45..ba9b32cba9a8f 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -368,7 +368,7 @@ void __weak freq_inv_set_max_ratio(int cpu, u64 max_rate) #ifdef CONFIG_ACPI_CPPC_LIB #include -void topology_init_cpu_capacity_cppc(void) +static inline void topology_init_cpu_capacity_cppc(void) { u64 capacity, capacity_scale = 0; struct cppc_perf_caps perf_caps; @@ -419,6 +419,10 @@ void topology_init_cpu_capacity_cppc(void) exit: free_raw_capacity(); } +void acpi_processor_init_invariance_cppc(void) +{ + topology_init_cpu_capacity_cppc(); +} #endif #ifdef CONFIG_CPU_FREQ diff --git a/include/acpi/processor.h b/include/acpi/processor.h index 3f34ebb275253..110203b6429d2 100644 --- a/include/acpi/processor.h +++ b/include/acpi/processor.h @@ -465,4 +465,6 @@ extern int acpi_processor_ffh_lpi_probe(unsigned int cpu); extern int acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi); #endif +void acpi_processor_init_invariance_cppc(void); + #endif diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h index a63d61ca55afc..8591d1a1f4304 100644 --- a/include/linux/arch_topology.h +++ b/include/linux/arch_topology.h @@ -11,10 +11,6 @@ void topology_normalize_cpu_scale(void); int topology_update_cpu_topology(void); -#ifdef CONFIG_ACPI_CPPC_LIB -void topology_init_cpu_capacity_cppc(void); -#endif - struct device_node; bool topology_parse_cpu_capacity(struct device_node *cpu_node, int cpu); From 4dc8bf6507a96334b303eb37595d2ddad64e35bf Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 4 Nov 2024 19:53:53 +0100 Subject: [PATCH 174/188] cpufreq: intel_pstate: Update asym capacity for CPUs that were offline initially commit 92447aa5f6e7fbad9427a3fd1bb9e0679c403206 upstream. Commit 929ebc93ccaa ("cpufreq: intel_pstate: Set asymmetric CPU capacity on hybrid systems") overlooked a corner case in which some CPUs may be offline to start with and brought back online later, after the intel_pstate driver has been registered, so their asymmetric capacity will not be set. Address this by calling hybrid_update_capacity() in the CPU initialization path that is executed instead of the online path for those CPUs. Note that this asymmetric capacity update will be skipped during driver initialization and mode switches because hybrid_max_perf_cpu is NULL in those cases. Fixes: 929ebc93ccaa ("cpufreq: intel_pstate: Set asymmetric CPU capacity on hybrid systems") Signed-off-by: Rafael J. Wysocki Link: https://patch.msgid.link/1913414.tdWV9SEqCh@rjwysocki.net Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/intel_pstate.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 33ca7d86c3514..ac5f58cc0790a 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2265,6 +2265,11 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) } else { cpu->pstate.scaling = perf_ctl_scaling; } + /* + * If the CPU is going online for the first time and it was + * offline initially, asym capacity scaling needs to be updated. + */ + hybrid_update_capacity(cpu); } else { cpu->pstate.scaling = perf_ctl_scaling; cpu->pstate.max_pstate = pstate_funcs.get_max(cpu->cpu); From 41fc5a283fbb287a0cf35b5d0d7b9045f20049b3 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Thu, 7 Nov 2024 13:36:10 +0100 Subject: [PATCH 175/188] cpufreq: intel_pstate: Rearrange locking in hybrid_init_cpu_capacity_scaling() commit 1a1030d10a6335bb5e6cdb24fc9388d3d9bcc1ac upstream. Notice that hybrid_init_cpu_capacity_scaling() only needs to hold hybrid_capacity_lock around __hybrid_init_cpu_capacity_scaling() calls, so introduce a "locked" wrapper around the latter and call it from the former. This allows to drop a local variable and a label that are not needed any more. Also, rename __hybrid_init_cpu_capacity_scaling() to __hybrid_refresh_cpu_capacity_scaling() for consistency. Interestingly enough, this fixes a locking issue introduced by commit 929ebc93ccaa ("cpufreq: intel_pstate: Set asymmetric CPU capacity on hybrid systems") that put an arch_enable_hybrid_capacity_scale() call under hybrid_capacity_lock, which was a mistake because the latter is acquired in CPU hotplug paths and so it cannot be held around cpus_read_lock() calls. Link: https://lore.kernel.org/linux-pm/SJ1PR11MB6129EDBF22F8A90FC3A3EDC8B9582@SJ1PR11MB6129.namprd11.prod.outlook.com/ Fixes: 929ebc93ccaa ("cpufreq: intel_pstate: Set asymmetric CPU capacity on hybrid systems") Signed-off-by: Rafael J. Wysocki Reported-by: "Borah, Chaitanya Kumar" Link: https://patch.msgid.link/12554508.O9o76ZdvQC@rjwysocki.net [ rjw: Changelog update ] Signed-off-by: Rafael J. Wysocki Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/intel_pstate.c | 35 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index ac5f58cc0790a..2c4a1e587edb7 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1027,26 +1027,29 @@ static void hybrid_update_cpu_capacity_scaling(void) } } -static void __hybrid_init_cpu_capacity_scaling(void) +static void __hybrid_refresh_cpu_capacity_scaling(void) { hybrid_max_perf_cpu = NULL; hybrid_update_cpu_capacity_scaling(); } -static void hybrid_init_cpu_capacity_scaling(bool refresh) +static void hybrid_refresh_cpu_capacity_scaling(void) { - bool disable_itmt = false; + guard(mutex)(&hybrid_capacity_lock); - mutex_lock(&hybrid_capacity_lock); + __hybrid_refresh_cpu_capacity_scaling(); +} +static void hybrid_init_cpu_capacity_scaling(bool refresh) +{ /* * If hybrid_max_perf_cpu is set at this point, the hybrid CPU capacity * scaling has been enabled already and the driver is just changing the * operation mode. */ if (refresh) { - __hybrid_init_cpu_capacity_scaling(); - goto unlock; + hybrid_refresh_cpu_capacity_scaling(); + return; } /* @@ -1055,19 +1058,13 @@ static void hybrid_init_cpu_capacity_scaling(bool refresh) * do not do that when SMT is in use. */ if (hwp_is_hybrid && !sched_smt_active() && arch_enable_hybrid_capacity_scale()) { - __hybrid_init_cpu_capacity_scaling(); - disable_itmt = true; - } - -unlock: - mutex_unlock(&hybrid_capacity_lock); - - /* - * Disabling ITMT causes sched domains to be rebuilt to disable asym - * packing and enable asym capacity. - */ - if (disable_itmt) + hybrid_refresh_cpu_capacity_scaling(); + /* + * Disabling ITMT causes sched domains to be rebuilt to disable asym + * packing and enable asym capacity. + */ sched_clear_itmt_support(); + } } static bool hybrid_clear_max_perf_cpu(void) @@ -1397,7 +1394,7 @@ static void intel_pstate_update_limits_for_all(void) mutex_lock(&hybrid_capacity_lock); if (hybrid_max_perf_cpu) - __hybrid_init_cpu_capacity_scaling(); + __hybrid_refresh_cpu_capacity_scaling(); mutex_unlock(&hybrid_capacity_lock); } From 06377eef39faf111d53764da217a3e243a2f3fe2 Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Thu, 21 Aug 2025 09:56:38 +0530 Subject: [PATCH 176/188] cpufreq/amd-pstate: Fix setting of CPPC.min_perf in active mode for performance governor commit 220abf77e7c2835cc63ea8cd7158cf83952640af upstream. In the "active" mode of the amd-pstate driver with performance governor, the CPPC.min_perf is expected to be the nominal_perf. However after commit a9b9b4c2a4cd ("cpufreq/amd-pstate: Drop min and max cached frequencies"), this is not the case when the governor is switched from performance to powersave and back to performance, and the CPPC.min_perf will be equal to the scaling_min_freq that was set for the powersave governor. This is because prior to commit a9b9b4c2a4cd ("cpufreq/amd-pstate: Drop min and max cached frequencies"), amd_pstate_epp_update_limit() would unconditionally call amd_pstate_update_min_max_limit() and the latter function would enforce the CPPC.min_perf constraint when the governor is performance. However, after the aforementioned commit, amd_pstate_update_min_max_limit() is called by amd_pstate_epp_update_limit() only when either the scaling_{min/max}_freq is different from the cached value of cpudata->{min/max}_limit_freq, which wouldn't have changed on a governor transition from powersave to performance, thus missing out on enforcing the CPPC.min_perf constraint for the performance governor. Fix this by invoking amd_pstate_epp_udpate_limit() not only when the {min/max} limits have changed from the cached values, but also when the policy itself has changed. Fixes: a9b9b4c2a4cd ("cpufreq/amd-pstate: Drop min and max cached frequencies") Signed-off-by: Gautham R. Shenoy Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20250821042638.356-1-gautham.shenoy@amd.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 31dbb7a84f41b..70a485a24e430 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1875,13 +1875,15 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) pr_debug("CPU %d exiting\n", policy->cpu); } -static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy) +static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy, bool policy_change) { struct amd_cpudata *cpudata = policy->driver_data; union perf_cached perf; u8 epp; - if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) + if (policy_change || + policy->min != cpudata->min_limit_freq || + policy->max != cpudata->max_limit_freq) amd_pstate_update_min_max_limit(policy); if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) @@ -1905,7 +1907,7 @@ static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) cpudata->policy = policy->policy; - ret = amd_pstate_epp_update_limit(policy); + ret = amd_pstate_epp_update_limit(policy, true); if (ret) return ret; @@ -2012,7 +2014,7 @@ static int amd_pstate_epp_resume(struct cpufreq_policy *policy) int ret; /* enable amd pstate from suspend state*/ - ret = amd_pstate_epp_update_limit(policy); + ret = amd_pstate_epp_update_limit(policy, false); if (ret) return ret; From 50dbacfbad536e1cebf0cefb2392166b079c43bc Mon Sep 17 00:00:00 2001 From: "Gautham R. Shenoy" Date: Fri, 7 Nov 2025 13:11:41 +0530 Subject: [PATCH 177/188] ACPI: CPPC: Detect preferred core availability on online CPUs commit 4fe5934db4a7187d358f1af1b3ef9b6dd59bce58 upstream. Commit 279f838a61f9 ("x86/amd: Detect preferred cores in amd_get_boost_ratio_numerator()") introduced the ability to detect the preferred core on AMD platforms by checking if there at least two distinct highest_perf values. However, it uses for_each_present_cpu() to iterate through all the CPUs in the platform, which is problematic when the kernel is booted with "nosmt=force" commandline option. Hence limit the search to only the online CPUs. Fixes: 279f838a61f9 ("x86/amd: Detect preferred cores in amd_get_boost_ratio_numerator()") Reported-by: Christopher Harris Closes: https://lore.kernel.org/lkml/CAM+eXpdDT7KjLV0AxEwOLkSJ2QtrsvGvjA2cCHvt1d0k2_C4Cw@mail.gmail.com/ Reviewed-by: "Mario Limonciello (AMD) (kernel.org)" Tested-by: Chrisopher Harris Signed-off-by: Gautham R. Shenoy Link: https://patch.msgid.link/20251107074145.2340-2-gautham.shenoy@amd.com Signed-off-by: Rafael J. Wysocki Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- arch/x86/kernel/acpi/cppc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index e774617e1ad5c..ff81320ffaacd 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -201,7 +201,7 @@ int amd_detect_prefcore(bool *detected) break; } - for_each_present_cpu(cpu) { + for_each_online_cpu(cpu) { u32 tmp; int ret; From 39a2c4d692b357cf1b06273adf115834e2917b6b Mon Sep 17 00:00:00 2001 From: Yaxiong Tian Date: Tue, 3 Feb 2026 10:48:52 +0800 Subject: [PATCH 178/188] cpufreq: intel_pstate: Enable asym capacity only when CPU SMT is not possible commit 1fedbb589448bee9f20bb2ed9c850d1d2cf9963c upstream. According to the description in the intel_pstate.rst documentation, Capacity-Aware Scheduling and Energy-Aware Scheduling are only supported on a hybrid processor without SMT. Previously, the system used sched_smt_active() for judgment, which is not a strict condition because users can switch it on or off via /sys at any time. This could lead to incorrect driver settings in certain scenarios. For example, on a CPU that supports SMT, a user can disable SMT via the nosmt parameter to enable asym capacity, and then re-enable SMT via /sys. In such cases, some settings in the driver would no longer be correct. To address this issue, replace sched_smt_active() with cpu_smt_possible(), and only enable asym capacity when CPU SMT is not possible. Fixes: 929ebc93ccaa ("cpufreq: intel_pstate: Set asymmetric CPU capacity on hybrid systems") Signed-off-by: Yaxiong Tian [ rjw: Subject and changelog edits ] Link: https://patch.msgid.link/20260203024852.301066-1-tianyaxiong@kylinos.cn Signed-off-by: Rafael J. Wysocki Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/intel_pstate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 2c4a1e587edb7..fcc29542be4c7 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1057,7 +1057,7 @@ static void hybrid_init_cpu_capacity_scaling(bool refresh) * the capacity of SMT threads is not deterministic even approximately, * do not do that when SMT is in use. */ - if (hwp_is_hybrid && !sched_smt_active() && arch_enable_hybrid_capacity_scale()) { + if (hwp_is_hybrid && !cpu_smt_possible() && arch_enable_hybrid_capacity_scale()) { hybrid_refresh_cpu_capacity_scaling(); /* * Disabling ITMT causes sched domains to be rebuilt to disable asym From 5bf59772093b7ce9af06639ce308a37513b0f014 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Wed, 3 Sep 2025 16:59:47 +0200 Subject: [PATCH 179/188] cpufreq: intel_pstate: Rearrange variable declaration involving __free() commit 5f2ec3536d7933a804d28d2541e4e918ba32587b upstream. Follow cleanup.h recommendations and define and assign a variable in one statement when __free() is used. No intentional functional impact. Signed-off-by: Rafael J. Wysocki Reviewed-by: Krzysztof Kozlowski Acked-by: Viresh Kumar Reviewed-by: Zihuan Zhang Link: https://patch.msgid.link/2251447.irdbgypaU6@rafael.j.wysocki Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/intel_pstate.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index fcc29542be4c7..696ba87afc9a3 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1364,9 +1364,7 @@ static void __intel_pstate_update_max_freq(struct cpufreq_policy *policy, static bool intel_pstate_update_max_freq(struct cpudata *cpudata) { - struct cpufreq_policy *policy __free(put_cpufreq_policy); - - policy = cpufreq_cpu_get(cpudata->cpu); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); if (!policy) return false; From dd1cfb3857abfe78cfe0c6637a3d4d3caa188d30 Mon Sep 17 00:00:00 2001 From: Srinivas Pandruvada Date: Tue, 24 Feb 2026 16:17:52 -0800 Subject: [PATCH 180/188] cpufreq: intel_pstate: Fix crash during turbo disable commit 6b050482ec40569429d963ac52afa878691b04c9 upstream. When the system is booted with kernel command line argument "nosmt" or "maxcpus" to limit the number of CPUs, disabling turbo via: echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo results in a crash: PF: supervisor read access in kernel mode PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: Oops: 0000 [#1] SMP PTI ... RIP: 0010:store_no_turbo+0x100/0x1f0 ... This occurs because for_each_possible_cpu() returns CPUs even if they are not online. For those CPUs, all_cpu_data[] will be NULL. Since commit 973207ae3d7c ("cpufreq: intel_pstate: Rearrange max frequency updates handling code"), all_cpu_data[] is dereferenced even for CPUs which are not online, causing the NULL pointer dereference. To fix that, pass CPU number to intel_pstate_update_max_freq() and use all_cpu_data[] for those CPUs for which there is a valid cpufreq policy. Fixes: 973207ae3d7c ("cpufreq: intel_pstate: Rearrange max frequency updates handling code") Closes: https://bugzilla.kernel.org/show_bug.cgi?id=221068 Signed-off-by: Srinivas Pandruvada Cc: 6.16+ # 6.16+ Link: https://patch.msgid.link/20260225001752.890164-1-srinivas.pandruvada@linux.intel.com Signed-off-by: Rafael J. Wysocki Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/intel_pstate.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 696ba87afc9a3..e9cbf5c33bdfa 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1362,13 +1362,13 @@ static void __intel_pstate_update_max_freq(struct cpufreq_policy *policy, refresh_frequency_limits(policy); } -static bool intel_pstate_update_max_freq(struct cpudata *cpudata) +static bool intel_pstate_update_max_freq(int cpu) { - struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); if (!policy) return false; - __intel_pstate_update_max_freq(policy, cpudata); + __intel_pstate_update_max_freq(policy, all_cpu_data[cpu]); return true; } @@ -1387,7 +1387,7 @@ static void intel_pstate_update_limits_for_all(void) int cpu; for_each_possible_cpu(cpu) - intel_pstate_update_max_freq(all_cpu_data[cpu]); + intel_pstate_update_max_freq(cpu); mutex_lock(&hybrid_capacity_lock); @@ -1830,7 +1830,7 @@ static void intel_pstate_notify_work(struct work_struct *work) struct cpudata *cpudata = container_of(to_delayed_work(work), struct cpudata, hwp_notify_work); - if (intel_pstate_update_max_freq(cpudata)) { + if (intel_pstate_update_max_freq(cpudata->cpu)) { /* * The driver will not be unregistered while this function is * running, so update the capacity without acquiring the driver From 0c825fe9c4d29bc99a2dabca74308e357b1069bf Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Tue, 7 Apr 2026 14:49:49 -0500 Subject: [PATCH 181/188] cpufreq/amd-pstate: Add POWER_SUPPLY select for dynamic EPP commit 679343977588781bd3effba79e9644aee4ee046c upstream. The dynamic EPP feature uses power_supply_reg_notifier() and power_supply_unreg_notifier() but doesn't declare a dependency on POWER_SUPPLY, causing linker errors when POWER_SUPPLY is not enabled. Add POWER_SUPPLY to the selects. Suggested-by: K Prateek Nayak Fixes: e30ca6dd5345 ("cpufreq/amd-pstate: Add dynamic energy performance preference") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202604040742.ySEdkuAa-lkp@intel.com/ Signed-off-by: Mario Limonciello Link: https://patch.msgid.link/20260407194949.310114-1-mario.limonciello@amd.com Signed-off-by: Rafael J. Wysocki Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/Kconfig.x86 | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 53e12ae98f68a..3ee40636071d6 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -40,6 +40,7 @@ config X86_AMD_PSTATE select ACPI_PROCESSOR select ACPI_CPPC_LIB if X86_64 select CPU_FREQ_GOV_SCHEDUTIL if SMP + select POWER_SUPPLY help This driver adds a CPUFreq driver which utilizes a fine grain processor performance frequency control range instead of legacy From fd74d894a06af859db7753f01ba38d22efc0153c Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Fri, 8 May 2026 05:17:43 +0000 Subject: [PATCH 182/188] cpufreq/amd-pstate: Grab "amd_pstate_driver_lock" when toggling dynamic_epp commit 9228169d2ae055ed09a163887fc59a710a5eb73b upstream. Concurrently changing driver mode and dynamic_epp with: echo passive > /sys/devices/system/cpu/amd_pstate/status& echo disable > /sys/devices/system/cpu/amd_pstate/dynamic_epp& hits the WARN_ON_ONCE() in static_key_disable_cpuslocked() and hangs the system since both sysfs writes are trying to do amd_pstate_change_driver_mode() without any synchronization. Grab the "amd_pstate_driver_lock" mutex when modifying "dynamic_epp" to prevent the two paths from racing with each other. Add a lockdep assertion for "amd_pstate_driver_lock" in amd_pstate_change_driver_mode() to formalize the dependency. Since "cppc_mode" is stable under "amd_pstate_driver_lock", only reload the driver when in "AMD_PSTATE_ACTIVE" mode and reject all writes when in passive or guided mode, or if the driver is not loaded, since only active mode operates on EPP. Fixes: e30ca6dd5345 ("cpufreq/amd-pstate: Add dynamic energy performance preference") Reviewed-by: Mario Limonciello Signed-off-by: K Prateek Nayak Link: https://lore.kernel.org/r/20260508051748.10484-2-kprateek.nayak@amd.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 70a485a24e430..4fd915e75473d 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1600,6 +1600,8 @@ static int amd_pstate_change_driver_mode(int mode) { int ret; + lockdep_assert_held(&amd_pstate_driver_lock); + ret = amd_pstate_unregister_driver(0); if (ret) return ret; @@ -1715,6 +1717,13 @@ static ssize_t dynamic_epp_store(struct device *a, struct device_attribute *b, if (ret) return ret; + guard(mutex)(&amd_pstate_driver_lock); + + if (cppc_state != AMD_PSTATE_ACTIVE) { + pr_debug("dynamic_epp can only be toggled in active mode\n"); + return -EINVAL; + } + if (dynamic_epp == enabled) return -EINVAL; From e7b43445d037bafe4c90a2821f6b6584e8a83982 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Fri, 8 May 2026 05:17:45 +0000 Subject: [PATCH 183/188] cpufreq/amd-pstate: Allow writes to dynamic_epp when state isn't modified commit c5eed6ddc757e477f52b3d99bfde9e59975c72ca upstream. Writing the current "dynamic_epp" state to sysfs fails with -EINVAL even though the desired result was achieved. Allow writes to "dynamic_epp" that does not modify the state. Fixes: e30ca6dd5345 ("cpufreq/amd-pstate: Add dynamic energy performance preference") Reviewed-by: Mario Limonciello Signed-off-by: K Prateek Nayak Link: https://lore.kernel.org/r/20260508051748.10484-4-kprateek.nayak@amd.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 4fd915e75473d..b9af6ef758ea4 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1724,8 +1724,9 @@ static ssize_t dynamic_epp_store(struct device *a, struct device_attribute *b, return -EINVAL; } + /* Nothing to do */ if (dynamic_epp == enabled) - return -EINVAL; + return count; /* reinitialize with desired dynamic EPP value */ dynamic_epp = enabled; From 0515a7a64a80249483544f43b20fb6fbee6f43b0 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Fri, 8 May 2026 05:17:46 +0000 Subject: [PATCH 184/188] cpufreq/amd-pstate: Reorder notifier unregistration and floor perf reset commit f3acf7ff113007557538b278ccb0e4ab7ae513ea upstream. An active power supply notifier can race with amd_pstate_epp_cpu_exit() trying to reset the floor perf and can overwrite the floor perf set in MSR_AMD_CPPC_REQ. Unregister the notifier before setting the floor perf to prevent the rare race. Fixes: e30ca6dd5345 ("cpufreq/amd-pstate: Add dynamic energy performance preference") Reviewed-by: Mario Limonciello Signed-off-by: K Prateek Nayak Link: https://lore.kernel.org/r/20260508051748.10484-5-kprateek.nayak@amd.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b9af6ef758ea4..f4dc7a35f4cb9 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1872,12 +1872,13 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) if (cpudata) { union perf_cached perf = READ_ONCE(cpudata->perf); + if (cpudata->dynamic_epp) + amd_pstate_clear_dynamic_epp(policy); + /* Reset CPPC_REQ MSR to the BIOS value */ amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf); - if (cpudata->dynamic_epp) - amd_pstate_clear_dynamic_epp(policy); kfree(cpudata); policy->driver_data = NULL; } From 362d5f49f8366d529a7c82052bb433e0f8ba6a48 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Fri, 8 May 2026 05:17:47 +0000 Subject: [PATCH 185/188] cpufreq/amd-pstate: Use "epp_default_dc" as default when dynamic_epp is disabled commit caa822d312be54e3fe1a3b52c887e0888e149c12 upstream. If "dynamic_epp" is disabled, the driver initialization and the default EPP selection from sysfs currently sets the EPP based on the power supply state of the system at that time but there is no power supply callbacks registered to toggle it when the power supply state changes. This can lead to faster battery drain on platforms that start off while being plugged to the wall but later move to battery power since the EPP stays at AMD_CPPC_EPP_PERFORMANCE. Use "epp_default_dc" as the default EPP selection when dynamic_epp is disabled, restoring older behavior. On servers, this defaults to AMD_CPPC_EPP_PERFORMANCE and on other platforms, it defaults to AMD_CPPC_EPP_BALANCE_PERFORMANCE. Fixes: e30ca6dd5345 ("cpufreq/amd-pstate: Add dynamic energy performance preference") Reviewed-by: Mario Limonciello Signed-off-by: K Prateek Nayak Link: https://lore.kernel.org/r/20260508051748.10484-6-kprateek.nayak@amd.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index f4dc7a35f4cb9..7c01b43731e09 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -1320,7 +1320,7 @@ ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, if (ret) epp = epp_values[ret]; else - epp = amd_pstate_get_balanced_epp(policy); + epp = cpudata->epp_default_dc; } if (epp > 0 && policy->policy == CPUFREQ_POLICY_PERFORMANCE) { @@ -1844,7 +1844,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (dynamic_epp) ret = amd_pstate_set_dynamic_epp(policy); else - ret = amd_pstate_set_epp(policy, amd_pstate_get_balanced_epp(policy)); + ret = amd_pstate_set_epp(policy, cpudata->epp_default_dc); if (ret) goto free_cpudata1; From cbbd57319ad1efbb1dd2346507c6fa5bee133427 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Fri, 8 May 2026 05:17:48 +0000 Subject: [PATCH 186/188] cpufreq/amd-pstate-ut: Drop policy reference before driver switch commit f9f16835d4dc46113c0a72625ffbf61f1aa95e5c upstream. Recent changes to the EPP unit test tries to perform a driver switch with a cpufreq_policy reference held when the driver is loaded into anything but the active mode which leads to a circular dependency and the unit test hanging indefinitely. Drop the reference before driver switch and grab it back once the driver mode is stabilized for the test. The EPP writes are only possible with CPUFREQ_POLICY_POWERSAVE policy. Temporarily switch the cpudata->policy (while holding the write end of the policy->rwsem) to CPUFREQ_POLICY_POWERSAVE and restore the original policy once tests are done. To ensure the final EPP is correct in case the driver started with CPUFREQ_POLICY_PERFORMANCE, EPP performance is tested last. The __free() based cleanup for cpufreq_policy is lost in the process. Reported-by: Kalpana Shetty Fixes: 7e173bc310d2b ("cpufreq/amd-pstate-ut: Add a unit test for raw EPP") Reviewed-by: Mario Limonciello Signed-off-by: K Prateek Nayak Link: https://lore.kernel.org/r/20260508051748.10484-7-kprateek.nayak@amd.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 36 ++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 18dec0caaec65..7933d1c7054f4 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -272,20 +272,21 @@ static int amd_pstate_set_mode(enum amd_pstate_mode mode) static int amd_pstate_ut_epp(u32 index) { - struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL; - char *buf __free(cleanup_page) = NULL; static const char * const epp_strings[] = { - "performance", - "balance_performance", - "balance_power", "power", + "balance_power", + "balance_performance", + "performance", }; - struct amd_cpudata *cpudata; + char *buf __free(cleanup_page) = NULL; + struct cpufreq_policy *policy = NULL; enum amd_pstate_mode orig_mode; + struct amd_cpudata *cpudata; + unsigned long orig_policy; bool orig_dynamic_epp; int ret, cpu = 0; - int i; u16 epp; + int i; policy = cpufreq_cpu_get(cpu); if (!policy) @@ -295,6 +296,10 @@ static int amd_pstate_ut_epp(u32 index) orig_mode = amd_pstate_get_status(); orig_dynamic_epp = cpudata->dynamic_epp; + /* Drop reference before potential driver change. */ + cpufreq_cpu_put(policy); + policy = NULL; + /* disable dynamic EPP before running test */ if (cpudata->dynamic_epp) { pr_debug("Dynamic EPP is enabled, disabling it\n"); @@ -309,6 +314,17 @@ static int amd_pstate_ut_epp(u32 index) if (ret) goto out; + policy = cpufreq_cpu_get(cpu); + if (!policy) { + ret = -ENODEV; + goto out; + } + + down_write(&policy->rwsem); + cpudata = policy->driver_data; + orig_policy = cpudata->policy; + cpudata->policy = CPUFREQ_POLICY_POWERSAVE; + for (epp = 0; epp <= U8_MAX; epp++) { u8 val; @@ -356,6 +372,12 @@ static int amd_pstate_ut_epp(u32 index) ret = 0; out: + if (policy) { + cpudata->policy = orig_policy; + up_write(&policy->rwsem); + cpufreq_cpu_put(policy); + } + if (orig_dynamic_epp) { int ret2; From 05be594b2c005b6d49515b7e903101fbfdc89512 Mon Sep 17 00:00:00 2001 From: K Prateek Nayak Date: Sat, 23 May 2026 05:55:03 +0000 Subject: [PATCH 187/188] cpufreq/amd-pstate-ut: Disable dynamic_epp after the mode switch commit df488cac6140aa04ae52af9b4507d8f99a3762be upstream. Dan reported a possible NULL pointer dereference in amd-pstate-ut.c from static analysis and sure enough, running amd-pstate-ut in active mode with amd_dynamic_epp=enable results in a crash as a reult of the policy reference being set to NULL early, before disabling dynamic EPP. Kalpana also reported seeing amd-pstate-ut error out with -EBUSY for "amd_pstate_ut_epp" test when starting from the passive mode and amd_dynamic_epp=enable in the command line. The reason for the failure is that the command line enables dynamic_epp by default after the mode switch and the modifications to EPP values are blocked when running in dynamic EPP mode. Solution to both problems is to toggle off dynamic_epp *after* the mode switch when the driver grabs the policy reference again since the unit test is in full control of the policy after that point. The final restoration step will reset the dynamic_epp state via mode switch based on the initial conditions of the system. Reported-by: Kalpana Shetty Reported-by: Dan Carpenter Closes: https://lore.kernel.org/linux-pm/ahEq0CvdBX0T7_cO@stanley.mountain/ Fixes: f9f16835d4dc ("cpufreq/amd-pstate-ut: Drop policy reference before driver switch") Signed-off-by: K Prateek Nayak Link: https://patch.msgid.link/20260523055503.7651-1-kprateek.nayak@amd.com Signed-off-by: Rafael J. Wysocki Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate-ut.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 7933d1c7054f4..6ddf2469093c7 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -300,12 +300,6 @@ static int amd_pstate_ut_epp(u32 index) cpufreq_cpu_put(policy); policy = NULL; - /* disable dynamic EPP before running test */ - if (cpudata->dynamic_epp) { - pr_debug("Dynamic EPP is enabled, disabling it\n"); - amd_pstate_clear_dynamic_epp(policy); - } - buf = (char *)__get_free_page(GFP_KERNEL); if (!buf) return -ENOMEM; @@ -325,6 +319,16 @@ static int amd_pstate_ut_epp(u32 index) orig_policy = cpudata->policy; cpudata->policy = CPUFREQ_POLICY_POWERSAVE; + /* + * Disable dynamic EPP before running test. If "orig_dynamic_epp" is + * true, the driver will do a redundant switch at the end and there + * is no need for enabling it again at the end of the test. + */ + if (cpudata->dynamic_epp) { + pr_debug("Dynamic EPP is enabled, disabling it\n"); + amd_pstate_clear_dynamic_epp(policy); + } + for (epp = 0; epp <= U8_MAX; epp++) { u8 val; From b7d53f513b8819d4cb773d72c63aa2db23bdc9a1 Mon Sep 17 00:00:00 2001 From: Zhan Xusheng Date: Tue, 26 May 2026 10:21:31 +0800 Subject: [PATCH 188/188] cpufreq/amd-pstate: drop stale @epp_cached kdoc commit 3cd07ee35a66038fd1a643632bfc057645e07c9a upstream. Commit 4e16c1175238 ("cpufreq/amd-pstate: Stop caching EPP") removed the epp_cached field from struct amd_cpudata in favour of always reading from cppc_req_cached, but the kdoc above the struct still documents @epp_cached. Drop the now-stale @epp_cached entry. Reviewed-by: Mario Limonciello (AMD) Fixes: 4e16c1175238 ("cpufreq/amd-pstate: Stop caching EPP") Signed-off-by: Zhan Xusheng Link: https://lore.kernel.org/r/20260526022131.1302373-1-zhanxusheng@xiaomi.com Signed-off-by: Mario Limonciello (AMD) Signed-off-by: Hemanth Selam Signed-off-by: mohanasv2 --- drivers/cpufreq/amd-pstate.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index b093138b71ed7..e4c53c7881987 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -83,7 +83,6 @@ struct amd_aperf_mperf { * @hw_prefcore: check whether HW supports preferred core featue. * Only when hw_prefcore and early prefcore param are true, * AMD P-State driver supports preferred core featue. - * @epp_cached: Cached CPPC energy-performance preference value * @policy: Cpufreq policy value * @suspended: If CPU core if offlined * @epp_default_ac: Default EPP value for AC power source