diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 2c70b7477516..87dc535913c8 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -934,6 +934,16 @@ config PORTABLE select MMU select OF + +config RISCV_ISA_SSCPUUTIL + bool "Sscucnt extension support for CPU Utilization Counter" + depends on RISCV && GENERIC_ARCH_TOPOLOGY && SMP + default y + help + Adds support for the Sscucnt (CPU Utilization Counter) extension + which provides hardware counters for scheduler frequency invariance. + If you do not know what to do here, say Y. + menu "Power management options" source "kernel/power/Kconfig" diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 39234a148d47..44ebdf4f92df 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -259,6 +259,8 @@ #define CSR_HPMCOUNTER29 0xc1d #define CSR_HPMCOUNTER30 0xc1e #define CSR_HPMCOUNTER31 0xc1f +#define CSR_CORECYC 0xc30 +#define CSR_ACTTIME 0xc31 #define CSR_CYCLEH 0xc80 #define CSR_TIMEH 0xc81 #define CSR_INSTRETH 0xc82 @@ -291,6 +293,8 @@ #define CSR_HPMCOUNTER29H 0xc9d #define CSR_HPMCOUNTER30H 0xc9e #define CSR_HPMCOUNTER31H 0xc9f +#define CSR_CORECYCH 0xcb0 +#define CSR_ACTTIMEH 0xcb1 #define CSR_SSCOUNTOVF 0xda0 diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index e23775c763be..4aaa024b6146 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -59,6 +59,7 @@ #define RISCV_ISA_EXT_ZIFENCEI 41 #define RISCV_ISA_EXT_ZIHPM 42 #define RISCV_ISA_EXT_XTHEADMATRIX 43 +#define RISCV_ISA_EXT_SSCPUUTIL 44 #define RISCV_ISA_EXT_MAX 64 diff --git a/arch/riscv/include/asm/topology.h b/arch/riscv/include/asm/topology.h index e316ab3b77f3..2f64d3af9823 100644 --- a/arch/riscv/include/asm/topology.h +++ b/arch/riscv/include/asm/topology.h @@ -4,6 +4,49 @@ #include +#ifdef CONFIG_RISCV_ISA_SSCPUUTIL + +#include + +#ifdef CONFIG_64BIT +static inline u64 get_corecyc(void) +{ + return csr_read(CSR_CORECYC); +} + +static inline u64 get_acttime(void) +{ + return csr_read(CSR_ACTTIME); +} +#else +static inline u64 get_corecyc(void) +{ + u32 hi, lo; + + do { + hi = csr_read(CSR_CORECYCH); + lo = csr_read(CSR_CORECYC); + } while (hi != csr_read(CSR_CORECYCH)); + + return ((u64)hi << 32) | lo; +} + +static inline u64 get_acttime(void) +{ + u32 hi, lo; + + do { + hi = csr_read(CSR_ACTTIMEH); + lo = csr_read(CSR_ACTTIME); + } while (hi != csr_read(CSR_ACTTIMEH)); + + return ((u64)hi << 32) | lo; +} +#endif /* CONFIG_64BIT */ + +void update_freq_counters_refs(void); +#endif /* CONFIG_RISCV_ISA_SSCPUUTIL */ + /* Replace task scheduler's default frequency-invariant accounting */ #define arch_scale_freq_tick topology_scale_freq_tick #define arch_set_freq_scale topology_set_freq_scale diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index ba6b228e4c7f..b5f4689e421a 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -104,3 +104,5 @@ obj-$(CONFIG_ARCH_RV64ILP32) += compat_signal.o obj-$(CONFIG_64BIT) += pi/ obj-$(CONFIG_ACPI) += acpi.o + +obj-$(CONFIG_RISCV_ISA_SSCPUUTIL) += topology.o diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 22e45210da70..ec512853ec71 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -177,6 +177,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), + __RISCV_ISA_EXT_DATA(sscpuutil, RISCV_ISA_EXT_SSCPUUTIL), __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index bd972b89149c..a92690bf4dab 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -61,6 +62,9 @@ void __init smp_prepare_cpus(unsigned int max_cpus) if (max_cpus == 0) return; + if (IS_ENABLED(CONFIG_RISCV_ISA_SSCPUUTIL)) + update_freq_counters_refs(); + for_each_possible_cpu(cpuid) { if (cpuid == curr_cpuid) continue; @@ -246,6 +250,9 @@ asmlinkage __visible void smp_callin(void) numa_add_cpu(curr_cpuid); set_cpu_online(curr_cpuid, 1); + + if (IS_ENABLED(CONFIG_RISCV_ISA_SSCPUUTIL)) + update_freq_counters_refs(); check_unaligned_access(curr_cpuid); if (has_vector()) { diff --git a/arch/riscv/kernel/topology.c b/arch/riscv/kernel/topology.c new file mode 100644 index 000000000000..6a37ef9ba80c --- /dev/null +++ b/arch/riscv/kernel/topology.c @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * arch/riscv/kernel/topology.c + * + * Frequency Invariance support for RISC-V using the Sscpuutil extension + * (CPU Utilization Counter Extension). + * + * The Sscpuutil extension provides a pair of CSR counters: + * CSR_SSCPUUTIL_CORECYC - core-cycles counter, increments at the actual CPU + * frequency + * CSR_SSCPUUTIL_ACTTIME - actual-time counter, increments at a fixed + * frequency equal to the nominal CPU frequency + * + * The ratio delta_CORECYC / delta_ACTTIME approximates freq_curr / freq_nominal, + * which is exactly the frequency-invariant utilisation scale factor required + * by the task scheduler. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_ACPI_CPPC_LIB +#include +#endif + +#include +#include +#include +#include +#include + +#ifdef CONFIG_RISCV_ISA_SSCPUUTIL +#define read_corecyc() get_corecyc() +#define read_acttime() get_acttime() +#else +#define read_corecyc() (0UL) +#define read_acttime() (0UL) +#endif + +static DEFINE_PER_CPU(u64, arch_active_time_prev); +static DEFINE_PER_CPU(u64, arch_core_cycles_prev); + +#undef pr_fmt +#define pr_fmt(fmt) "Sscpuutil: " fmt + +static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale); + +/* Bitmask of CPUs for which Sscpuutil-based FIE has been successfully set up */ +static cpumask_var_t sscpuutil_fie_cpus; + +static inline bool freq_counters_valid(int cpu) +{ + if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask)) + return false; + + if (!__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCPUUTIL)) { + pr_debug("CPU%d: Sscpuutil counters are not supported.\n", cpu); + return false; + } + + return true; +} + + +/* + * Probe strategies for obtaining the maximum CPU frequency. + * Tried in order; the first one that returns non-zero wins. + * + * 1. ACPI CPPC - server platforms with ACPI + * 2. DT clock-frequency - embedded / HAPS with DTS + * 3. OPP table - platforms with operating-points-v2 + */ +static u64 get_max_freq_acpi_cppc(int cpu) +{ +#ifdef CONFIG_ACPI_CPPC_LIB + struct cppc_perf_caps caps; + + if (!acpi_disabled && !cppc_get_perf_caps(cpu, &caps)) { + u64 rate; + + /* + * cppc nominal_freq is in MHz when nonzero, + * otherwise fall back to highest_perf (abstract units, + * not directly usable as Hz -> skip). + */ + if (caps.nominal_freq) { + rate = (u64)caps.nominal_freq * 1000000ULL; + pr_info("CPU%d: max_rate from ACPI CPPC: %llu Hz\n", + cpu, rate); + return rate; + } + } +#endif + return 0; +} + +static u64 get_max_freq_dt(int cpu) +{ + struct device_node *cn; + u32 freq; + u64 rate = 0; + + cn = of_cpu_device_node_get(cpu); + if (!cn) + return 0; + + /* + * "clock-frequency" is the standard ePAPR/DT property for + * the CPU's nominal operating frequency in Hz. + */ + if (!of_property_read_u32(cn, "clock-frequency", &freq) && freq) { + rate = (u64)freq; + pr_info("CPU%d: max_rate from DT clock-frequency: %llu Hz\n", + cpu, rate); + goto out; + } + + /* + * Some vendor DTs use "cpu-freq" instead. + */ + if (!of_property_read_u32(cn, "cpu-freq", &freq) && freq) { + rate = (u64)freq; + pr_info("CPU%d: max_rate from DT cpu-freq: %llu Hz\n", + cpu, rate); + goto out; + } + +out: + of_node_put(cn); + return rate; +} + +static u64 get_max_freq_opp(int cpu) +{ + struct device *dev; + unsigned long freq; + int ret; + + dev = get_cpu_device(cpu); + if (!dev) + return 0; + + /* + * Try to add OPP table from DT if not already done. + * Returns -EEXIST if already registered (harmless). + */ + ret = dev_pm_opp_of_add_table(dev); + if (ret && ret != -EEXIST) { + pr_debug("CPU%d: no OPP table in DT (ret=%d)\n", cpu, ret); + return 0; + } + + { + struct dev_pm_opp *opp; + + freq = ULONG_MAX; + opp = dev_pm_opp_find_freq_floor(dev, &freq); + if (IS_ERR(opp)) { + /* clean up if we just added it and it's empty */ + if (ret != -EEXIST) + dev_pm_opp_of_remove_table(dev); + return 0; + } + dev_pm_opp_put(opp); + } + + if (!freq) { + if (ret != -EEXIST) + dev_pm_opp_of_remove_table(dev); + return 0; + } + + pr_info("CPU%d: max_rate from OPP table: %lu Hz\n", cpu, freq); + return (u64)freq; +} + +static u64 sscpuutil_get_max_rate(int cpu) +{ + u64 rate = 0; + + /* 1. ACPI CPPC */ + rate = get_max_freq_acpi_cppc(cpu); + if (rate) + return rate; + + /* 2. DT clock-frequency */ + rate = get_max_freq_dt(cpu); + if (rate) + return rate; + + /* 3. OPP table */ + rate = get_max_freq_opp(cpu); + if (rate) + return rate; + + /* no source found */ + return 0; +} + +/** + * sscpuutil_set_max_ratio - pre-compute the ref/max frequency scale factor + * @cpu: logical CPU number + * @max_rate: maximum CPU frequency in Hz + * @ref_rate: fixed frequency of the Sscpuutil reference counter in Hz + * + * Stores the ratio (ref_rate / max_rate) × SCHED_CAPACITY_SCALE in + * arch_max_freq_scale[cpu]. + * + * Returns 0 on success, -EINVAL if either rate is zero or the ratio + * underflows to zero. + */ +static int sscpuutil_set_max_ratio(int cpu, u64 max_rate, u64 ref_rate) +{ + u64 ratio; + + if (unlikely(!max_rate || !ref_rate)) { + pr_debug("CPU%d: invalid maximum or reference frequency.\n", cpu); + return -EINVAL; + } + + /* + * Pre-compute the fixed ratio between the reference counter frequency + * and the maximum CPU frequency: + * + * ref_rate + * arch_max_freq_scale = ────────── × SCHED_CAPACITY_SCALE² + * max_rate + */ + ratio = ref_rate << (2 * SCHED_CAPACITY_SHIFT); + ratio = div64_u64(ratio, max_rate); + if (!ratio) { + WARN_ONCE(1, "Sscpuutil reference frequency too low.\n"); + return -EINVAL; + } + + per_cpu(arch_max_freq_scale, cpu) = (unsigned long)ratio; + + return 0; +} + + +void update_freq_counters_refs(void) +{ + this_cpu_write(arch_core_cycles_prev, read_corecyc()); + this_cpu_write(arch_active_time_prev, read_acttime()); +} + +/** + * sscpuutil_scale_freq_tick - update arch_freq_scale on every scheduler tick + * + * Registered as the set_freq_scale callback of sscpuutil_sfd and invoked by + * topology_scale_freq_tick() → arch_scale_freq_tick() on every tick. + * + * Computes: + * + * Δcorecyc arch_max_freq_scale + * scale = ──────── × ──────────────────── + * Δacttime SCHED_CAPACITY_SCALE + * + * and writes it to arch_freq_scale for the current CPU. The result is + * clamped to [0, SCHED_CAPACITY_SCALE]. + */ +static void sscpuutil_scale_freq_tick(void) +{ + u64 corecyc_t0, acttime_t0; + u64 corecyc_t1, acttime_t1; + u64 delta_corecyc, delta_acttime; + u64 scale; + + corecyc_t0 = this_cpu_read(arch_core_cycles_prev); + acttime_t0 = this_cpu_read(arch_active_time_prev); + + update_freq_counters_refs(); + + corecyc_t1 = this_cpu_read(arch_core_cycles_prev); + acttime_t1 = this_cpu_read(arch_active_time_prev); + + if (unlikely(corecyc_t1 <= corecyc_t0 || + acttime_t1 <= acttime_t0)) + return; + + delta_acttime = acttime_t1 - acttime_t0; + delta_corecyc = corecyc_t1 - corecyc_t0; + + /* + * Δcorecyc arch_max_freq_scale + * scale = ──────── × ──────────────────── + * Δacttime SCHED_CAPACITY_SCALE + */ + scale = delta_corecyc * this_cpu_read(arch_max_freq_scale); + scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT, + delta_acttime); + + scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE); + this_cpu_write(arch_freq_scale, (unsigned long)scale); + + pr_debug("CPU%d: freq_scale=%lu (%lu%%)\n", + smp_processor_id(), (unsigned long)scale, + (unsigned long)scale * 100 / SCHED_CAPACITY_SCALE); +} + +static struct scale_freq_data sscpuutil_sfd = { + .source = SCALE_FREQ_SOURCE_ARCH, + .set_freq_scale = sscpuutil_scale_freq_tick, +}; + +/** + * sscpuutil_fie_setup - enable Sscpuutil-based FIE for a set of CPUs + * @cpus: the set of CPUs to enable FIE for + * + * For each CPU in @cpus, validates counter availability and computes the + * max-frequency ratio. If all CPUs pass, registers sscpuutil_sfd as the + * SCALE_FREQ_SOURCE_ARCH source for those CPUs. + * + * The function is idempotent: CPUs already in sscpuutil_fie_cpus are skipped. + */ +static void sscpuutil_fie_setup(const struct cpumask *cpus) +{ + int cpu; + + /* Nothing to do if all CPUs in this policy are already set up */ + if (unlikely(cpumask_subset(cpus, sscpuutil_fie_cpus))) + return; + + for_each_cpu(cpu, cpus) { + if (!freq_counters_valid(cpu)) { + pr_info("CPU%d: freq_counters_valid() failed, skip FIE setup\n", cpu); + return; + } + + u64 max_rate = sscpuutil_get_max_rate(cpu); + + if (!max_rate) { + pr_info("CPU%d: failed to determine max frequency, skip FIE setup\n", cpu); + return; + } + + pr_info("CPU%d: max_rate=%llu Hz, riscv_timebase=%lu Hz\n", + cpu, max_rate, riscv_timebase); + + if (sscpuutil_set_max_ratio(cpu, max_rate, riscv_timebase)) { + pr_info("CPU%d: sscpuutil_set_max_ratio() failed, skip FIE setup\n", cpu); + return; + } + } + + cpumask_or(sscpuutil_fie_cpus, sscpuutil_fie_cpus, cpus); + + topology_set_scale_freq_source(&sscpuutil_sfd, sscpuutil_fie_cpus); + + pr_info("CPUs[%*pbl]: Sscpuutil FIE enabled\n", + cpumask_pr_args(cpus)); +} + +/** + * init_sscpuutil_fie - initialise Sscpuutil-based FIE for all present CPUs + * + * Runs as a core_initcall. Directly sets up frequency invariance for + * all present CPUs without depending on cpufreq policy notifications. + */ +static int __init init_sscpuutil_fie(void) +{ + if (!zalloc_cpumask_var(&sscpuutil_fie_cpus, GFP_KERNEL)) + return -ENOMEM; + + sscpuutil_fie_setup(cpu_present_mask); + + return 0; +} +core_initcall(init_sscpuutil_fie);