diff --git a/arch/riscv/include/asm/alternative-macros.h b/arch/riscv/include/asm/alternative-macros.h index 231d777d936c2d..be9835b5e4eba0 100644 --- a/arch/riscv/include/asm/alternative-macros.h +++ b/arch/riscv/include/asm/alternative-macros.h @@ -158,4 +158,77 @@ _ALTERNATIVE_CFG_2(old_content, new_content_1, vendor_id_1, patch_id_1, CONFIG_k_1, \ new_content_2, vendor_id_2, patch_id_2, CONFIG_k_2) +/* + * use_alternative_{likely,unlikely}() returns true if the alternative is + * applied and false otherwise, but in a way where the compiler can optimize + * this check down to a nop instruction that's patched into a jump, or vice + * versa. + * + * Always returns false if the alternatives mechanism is not available. + * + * Usage example: + * if (use_alternative_likely(0, RISCV_ISA_ZBB)) + * + * Similar to static keys, "likely" means use a nop if the alternative is + * selected, and jump if unselected; "unlikely" is the other way around. + */ + +#ifndef __ASSEMBLER__ + +#include + +#ifdef CONFIG_RISCV_ALTERNATIVE + +static __always_inline bool use_alternative_likely(u16 vendor_id, u32 patch_id) +{ + BUILD_BUG_ON(!__builtin_constant_p(vendor_id)); + BUILD_BUG_ON(!__builtin_constant_p(patch_id)); + + asm goto(ALTERNATIVE("j %l[no_alt]", "nop", %[vendor_id], %[patch_id], 1) + : + : [vendor_id] "i"(vendor_id), + [patch_id] "i"(patch_id) + : + : no_alt); + + return true; + +no_alt: + return false; +} + +static __always_inline bool use_alternative_unlikely(u16 vendor_id, u32 patch_id) +{ + BUILD_BUG_ON(!__builtin_constant_p(vendor_id)); + BUILD_BUG_ON(!__builtin_constant_p(patch_id)); + + asm goto(ALTERNATIVE("nop", "j %l[alt]", %[vendor_id], %[patch_id], 1) + : + : [vendor_id] "i"(vendor_id), + [patch_id] "i"(patch_id) + : + : alt); + + return false; + +alt: + return true; +} + +#else + +static inline bool use_alternative_likely(u16 vendor_id, u32 patch_id) +{ + return false; +} + +static inline bool use_alternative_unlikely(u16 vendor_id, u32 patch_id) +{ + return false; +} + +#endif /* CONFIG_RISCV_ALTERNATIVE */ + +#endif /* __ASSEMBLER__ */ + #endif diff --git a/arch/riscv/include/asm/arch_hweight.h b/arch/riscv/include/asm/arch_hweight.h index 0e7cdbbec8efd3..58ed7a3b2d7882 100644 --- a/arch/riscv/include/asm/arch_hweight.h +++ b/arch/riscv/include/asm/arch_hweight.h @@ -20,20 +20,17 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w) { #if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (use_alternative_likely(0, RISCV_ISA_EXT_ZBB)) { + asm (".option push\n" + ".option arch,+zbb\n" + CPOPW "%0, %1\n" + ".option pop\n" + : "=r" (w) : "r" (w) :); - asm (".option push\n" - ".option arch,+zbb\n" - CPOPW "%0, %1\n" - ".option pop\n" - : "=r" (w) : "r" (w) :); - - return w; - -legacy: + return w; + } #endif + return __sw_hweight32(w); } @@ -51,20 +48,17 @@ static inline unsigned int __arch_hweight8(unsigned int w) static __always_inline unsigned long __arch_hweight64(__u64 w) { #if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (use_alternative_likely(0, RISCV_ISA_EXT_ZBB)) { + asm (".option push\n" + ".option arch,+zbb\n" + "cpop %0, %1\n" + ".option pop\n" + : "=r" (w) : "r" (w) :); - asm (".option push\n" - ".option arch,+zbb\n" - "cpop %0, %1\n" - ".option pop\n" - : "=r" (w) : "r" (w) :); - - return w; - -legacy: + return w; + } #endif + return __sw_hweight64(w); } #else /* BITS_PER_LONG == 64 */ diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h index d59310f74c2ba7..0257d547a96293 100644 --- a/arch/riscv/include/asm/bitops.h +++ b/arch/riscv/include/asm/bitops.h @@ -47,20 +47,17 @@ static __always_inline unsigned long variable__ffs(unsigned long word) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); - - asm volatile (".option push\n" - ".option arch,+zbb\n" - "ctz %0, %1\n" - ".option pop\n" - : "=r" (word) : "r" (word) :); - - return word; - -legacy: - return generic___ffs(word); + if (use_alternative_likely(0, RISCV_ISA_EXT_ZBB)) { + asm volatile (".option push\n" + ".option arch,+zbb\n" + "ctz %0, %1\n" + ".option pop\n" + : "=r" (word) : "r" (word) :); + + return word; + } else { + return generic___ffs(word); + } } /** @@ -76,20 +73,17 @@ static __always_inline unsigned long variable__ffs(unsigned long word) static __always_inline unsigned long variable__fls(unsigned long word) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); - - asm volatile (".option push\n" - ".option arch,+zbb\n" - "clz %0, %1\n" - ".option pop\n" - : "=r" (word) : "r" (word) :); - - return BITS_PER_LONG - 1 - word; - -legacy: - return generic___fls(word); + if (use_alternative_likely(0, RISCV_ISA_EXT_ZBB)) { + asm volatile (".option push\n" + ".option arch,+zbb\n" + "clz %0, %1\n" + ".option pop\n" + : "=r" (word) : "r" (word) :); + + return BITS_PER_LONG - 1 - word; + } else { + return generic___fls(word); + } } /** @@ -105,23 +99,20 @@ static __always_inline unsigned long variable__fls(unsigned long word) static __always_inline int variable_ffs(int x) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); - - if (!x) - return 0; - - asm volatile (".option push\n" - ".option arch,+zbb\n" - CTZW "%0, %1\n" - ".option pop\n" - : "=r" (x) : "r" (x) :); - - return x + 1; - -legacy: - return generic_ffs(x); + if (use_alternative_likely(0, RISCV_ISA_EXT_ZBB)) { + if (!x) + return 0; + + asm volatile (".option push\n" + ".option arch,+zbb\n" + CTZW "%0, %1\n" + ".option pop\n" + : "=r" (x) : "r" (x) :); + + return x + 1; + } else { + return generic_ffs(x); + } } /** @@ -137,23 +128,20 @@ static __always_inline int variable_ffs(int x) static __always_inline int variable_fls(unsigned int x) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); - - if (!x) - return 0; - - asm volatile (".option push\n" - ".option arch,+zbb\n" - CLZW "%0, %1\n" - ".option pop\n" - : "=r" (x) : "r" (x) :); - - return 32 - x; - -legacy: - return generic_fls(x); + if (use_alternative_likely(0, RISCV_ISA_EXT_ZBB)) { + if (!x) + return 0; + + asm volatile (".option push\n" + ".option arch,+zbb\n" + CLZW "%0, %1\n" + ".option pop\n" + : "=r" (x) : "r" (x) :); + + return 32 - x; + } else { + return generic_fls(x); + } } /** diff --git a/arch/riscv/include/asm/checksum.h b/arch/riscv/include/asm/checksum.h index da378856f1d590..14f3942007b556 100644 --- a/arch/riscv/include/asm/checksum.h +++ b/arch/riscv/include/asm/checksum.h @@ -49,16 +49,11 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) * ZBB only saves three instructions on 32-bit and five on 64-bit so not * worth checking if supported without Alternatives. */ - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + use_alternative_likely(0, RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); - if (IS_ENABLED(CONFIG_32BIT)) { asm(".option push \n\ .option arch,+zbb \n\ @@ -81,7 +76,7 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) } return (__force __sum16)(csum >> 16); } -no_zbb: + #ifndef CONFIG_32BIT csum += ror64(csum, 32); csum >>= 32; diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 0b749e71021624..1ef6e9de5f6d27 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -370,74 +370,71 @@ static __always_inline void __cmpwait(volatile void *ptr, u32 *__ptr32b; ulong __s, __val, __mask; - asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop", - 0, RISCV_ISA_EXT_ZAWRS, 1) - : : : : no_zawrs); - - switch (size) { - case 1: - __ptr32b = (u32 *)((ulong)(ptr) & ~0x3); - __s = ((ulong)(ptr) & 0x3) * BITS_PER_BYTE; - __val = val << __s; - __mask = 0xff << __s; - - asm volatile( - " lr.w %0, %1\n" - " and %0, %0, %3\n" - " xor %0, %0, %2\n" - " bnez %0, 1f\n" - ZAWRS_WRS_NTO "\n" - "1:" - : "=&r" (tmp), "+A" (*(__ptr32b)) - : "r" (__val), "r" (__mask) - : "memory"); - break; - case 2: - __ptr32b = (u32 *)((ulong)(ptr) & ~0x3); - __s = ((ulong)(ptr) & 0x2) * BITS_PER_BYTE; - __val = val << __s; - __mask = 0xffff << __s; - - asm volatile( - " lr.w %0, %1\n" - " and %0, %0, %3\n" - " xor %0, %0, %2\n" - " bnez %0, 1f\n" - ZAWRS_WRS_NTO "\n" - "1:" - : "=&r" (tmp), "+A" (*(__ptr32b)) - : "r" (__val), "r" (__mask) - : "memory"); - break; - case 4: - asm volatile( - " lr.w %0, %1\n" - " xor %0, %0, %2\n" - " bnez %0, 1f\n" - ZAWRS_WRS_NTO "\n" - "1:" - : "=&r" (tmp), "+A" (*(u32 *)ptr) - : "r" (val)); - break; + if (use_alternative_likely(0, RISCV_ISA_EXT_ZAWRS)) { + switch (size) { + case 1: + __ptr32b = (u32 *)((ulong)(ptr) & ~0x3); + __s = ((ulong)(ptr) & 0x3) * BITS_PER_BYTE; + __val = val << __s; + __mask = 0xff << __s; + + asm volatile( + " lr.w %0, %1\n" + " and %0, %0, %3\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(__ptr32b)) + : "r" (__val), "r" (__mask) + : "memory"); + break; + case 2: + __ptr32b = (u32 *)((ulong)(ptr) & ~0x3); + __s = ((ulong)(ptr) & 0x2) * BITS_PER_BYTE; + __val = val << __s; + __mask = 0xffff << __s; + + asm volatile( + " lr.w %0, %1\n" + " and %0, %0, %3\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(__ptr32b)) + : "r" (__val), "r" (__mask) + : "memory"); + break; + case 4: + asm volatile( + " lr.w %0, %1\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(u32 *)ptr) + : "r" (val)); + break; #if __riscv_xlen == 64 - case 8: - asm volatile( - " lr.d %0, %1\n" - " xor %0, %0, %2\n" - " bnez %0, 1f\n" - ZAWRS_WRS_NTO "\n" - "1:" - : "=&r" (tmp), "+A" (*(u64 *)ptr) - : "r" (val)); - break; + case 8: + asm volatile( + " lr.d %0, %1\n" + " xor %0, %0, %2\n" + " bnez %0, 1f\n" + ZAWRS_WRS_NTO "\n" + "1:" + : "=&r" (tmp), "+A" (*(u64 *)ptr) + : "r" (val)); + break; #endif - default: - BUILD_BUG(); - } + default: + BUILD_BUG(); + } - return; + return; + } -no_zawrs: asm volatile(RISCV_PAUSE : : : "memory"); } diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 91697fbf1f9013..81eb386da837f0 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -495,8 +495,13 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { - asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) - : : : : svvptc); + /* + * Svvptc guarantees that the new valid pte will be visible within + * a bounded timeframe, so when the uarch does not cache invalid + * entries, we don't have to do anything. + */ + if (use_alternative_unlikely(0, RISCV_ISA_EXT_SVVPTC)) + return; /* * The kernel assumes that TLBs don't cache invalid entries, but @@ -508,12 +513,6 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, while (nr--) local_flush_tlb_page(address + nr * PAGE_SIZE); -svvptc:; - /* - * Svvptc guarantees that the new valid pte will be visible within - * a bounded timeframe, so when the uarch does not cache invalid - * entries, we don't have to do anything. - */ } #define update_mmu_cache(vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1) diff --git a/arch/riscv/lib/csum.c b/arch/riscv/lib/csum.c index 9408f50ca59a89..659d3b51d0db10 100644 --- a/arch/riscv/lib/csum.c +++ b/arch/riscv/lib/csum.c @@ -40,20 +40,15 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, uproto = (__force unsigned int)htonl(proto); sum += uproto; - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { + /* + * Zbb is likely available when the kernel is compiled with Zbb + * support. + */ + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + use_alternative_likely(0, RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - /* - * Zbb is likely available when the kernel is compiled with Zbb - * support, so nop when Zbb is available and jump when Zbb is - * not available. - */ - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); asm(".option push \n\ .option arch,+zbb \n\ rori %[fold_temp], %[sum], 32 \n\ @@ -66,7 +61,7 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, : [sum] "+r" (sum), [fold_temp] "=&r" (fold_temp)); return (__force __sum16)(sum >> 16); } -no_zbb: + sum += ror64(sum, 32); sum >>= 32; return csum_fold((__force __wsum)sum); @@ -151,22 +146,16 @@ do_csum_with_alignment(const unsigned char *buff, int len) end = (const unsigned long *)(buff + len); csum = do_csum_common(ptr, end, data); + /* + * Zbb is likely available when the kernel is compiled with Zbb + * support. + */ #ifdef CC_HAS_ASM_GOTO_TIED_OUTPUT - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + use_alternative_likely(0, RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - /* - * Zbb is likely available when the kernel is compiled with Zbb - * support, so nop when Zbb is available and jump when Zbb is - * not available. - */ - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); - #ifdef CONFIG_32BIT asm_goto_output(".option push \n\ .option arch,+zbb \n\ @@ -204,7 +193,7 @@ do_csum_with_alignment(const unsigned char *buff, int len) end: return csum >> 16; } -no_zbb: + #endif /* CC_HAS_ASM_GOTO_TIED_OUTPUT */ #ifndef CONFIG_32BIT csum += ror64(csum, 32); @@ -234,21 +223,15 @@ do_csum_no_alignment(const unsigned char *buff, int len) end = (const unsigned long *)(buff + len); csum = do_csum_common(ptr, end, data); - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { + /* + * Zbb is likely available when the kernel is compiled with Zbb + * support. + */ + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + use_alternative_likely(0, RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - /* - * Zbb is likely available when the kernel is compiled with Zbb - * support, so nop when Zbb is available and jump when Zbb is - * not available. - */ - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); - #ifdef CONFIG_32BIT asm (".option push \n\ .option arch,+zbb \n\ @@ -274,7 +257,7 @@ do_csum_no_alignment(const unsigned char *buff, int len) #endif /* !CONFIG_32BIT */ return csum >> 16; } -no_zbb: + #ifndef CONFIG_32BIT csum += ror64(csum, 32); csum >>= 32; diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c index 8b6c0a112a8db4..e0c414fa0d433f 100644 --- a/arch/riscv/mm/pgtable.c +++ b/arch/riscv/mm/pgtable.c @@ -9,8 +9,16 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty) { - asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) - : : : : svvptc); + if (use_alternative_unlikely(0, RISCV_ISA_EXT_SVVPTC)) { + if (!pte_same(ptep_get(ptep), entry)) { + __set_pte_at(vma->vm_mm, ptep, entry); + /* Here only not svadu is impacted */ + flush_tlb_page(vma, address); + return true; + } + + return false; + } if (!pte_same(ptep_get(ptep), entry)) __set_pte_at(vma->vm_mm, ptep, entry); @@ -19,16 +27,6 @@ int ptep_set_access_flags(struct vm_area_struct *vma, * the case that the PTE changed and the spurious fault case. */ return true; - -svvptc: - if (!pte_same(ptep_get(ptep), entry)) { - __set_pte_at(vma->vm_mm, ptep, entry); - /* Here only not svadu is impacted */ - flush_tlb_page(vma, address); - return true; - } - - return false; } int ptep_test_and_clear_young(struct vm_area_struct *vma,