diff --git a/arch/riscv/include/asm/arch_hweight.h b/arch/riscv/include/asm/arch_hweight.h index 0e7cdbbec8efd3..021bc671de299d 100644 --- a/arch/riscv/include/asm/arch_hweight.h +++ b/arch/riscv/include/asm/arch_hweight.h @@ -19,10 +19,10 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w) { -#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!(IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + __riscv_has_extension_likely(0, RISCV_ISA_EXT_ZBB))) + return __sw_hweight32(w); asm (".option push\n" ".option arch,+zbb\n" @@ -31,10 +31,6 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w) : "=r" (w) : "r" (w) :); return w; - -legacy: -#endif - return __sw_hweight32(w); } static inline unsigned int __arch_hweight16(unsigned int w) @@ -50,10 +46,10 @@ static inline unsigned int __arch_hweight8(unsigned int w) #if BITS_PER_LONG == 64 static __always_inline unsigned long __arch_hweight64(__u64 w) { -#if defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB) - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!(IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + __riscv_has_extension_likely(0, RISCV_ISA_EXT_ZBB))) + return __sw_hweight64(w); asm (".option push\n" ".option arch,+zbb\n" @@ -62,10 +58,6 @@ static __always_inline unsigned long __arch_hweight64(__u64 w) : "=r" (w) : "r" (w) :); return w; - -legacy: -#endif - return __sw_hweight64(w); } #else /* BITS_PER_LONG == 64 */ static inline unsigned long __arch_hweight64(__u64 w) diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h index d59310f74c2ba7..f70ccc0c2ffb86 100644 --- a/arch/riscv/include/asm/bitops.h +++ b/arch/riscv/include/asm/bitops.h @@ -47,9 +47,8 @@ static __always_inline unsigned long variable__ffs(unsigned long word) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!__riscv_has_extension_likely(0, RISCV_ISA_EXT_ZBB)) + return generic___ffs(word); asm volatile (".option push\n" ".option arch,+zbb\n" @@ -58,9 +57,6 @@ static __always_inline unsigned long variable__ffs(unsigned long word) : "=r" (word) : "r" (word) :); return word; - -legacy: - return generic___ffs(word); } /** @@ -76,9 +72,8 @@ static __always_inline unsigned long variable__ffs(unsigned long word) static __always_inline unsigned long variable__fls(unsigned long word) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!__riscv_has_extension_likely(0, RISCV_ISA_EXT_ZBB)) + return generic___fls(word); asm volatile (".option push\n" ".option arch,+zbb\n" @@ -87,9 +82,6 @@ static __always_inline unsigned long variable__fls(unsigned long word) : "=r" (word) : "r" (word) :); return BITS_PER_LONG - 1 - word; - -legacy: - return generic___fls(word); } /** @@ -105,9 +97,8 @@ static __always_inline unsigned long variable__fls(unsigned long word) static __always_inline int variable_ffs(int x) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!__riscv_has_extension_likely(0, RISCV_ISA_EXT_ZBB)) + return generic_ffs(x); if (!x) return 0; @@ -119,9 +110,6 @@ static __always_inline int variable_ffs(int x) : "=r" (x) : "r" (x) :); return x + 1; - -legacy: - return generic_ffs(x); } /** @@ -137,9 +125,8 @@ static __always_inline int variable_ffs(int x) static __always_inline int variable_fls(unsigned int x) { - asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : : : : legacy); + if (!__riscv_has_extension_likely(0, RISCV_ISA_EXT_ZBB)) + return generic_fls(x); if (!x) return 0; @@ -151,9 +138,6 @@ static __always_inline int variable_fls(unsigned int x) : "=r" (x) : "r" (x) :); return 32 - x; - -legacy: - return generic_fls(x); } /** diff --git a/arch/riscv/include/asm/checksum.h b/arch/riscv/include/asm/checksum.h index da378856f1d590..70eb50173fb6ab 100644 --- a/arch/riscv/include/asm/checksum.h +++ b/arch/riscv/include/asm/checksum.h @@ -49,16 +49,11 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) * ZBB only saves three instructions on 32-bit and five on 64-bit so not * worth checking if supported without Alternatives. */ - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + __riscv_has_extension_likely(0, RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); - if (IS_ENABLED(CONFIG_32BIT)) { asm(".option push \n\ .option arch,+zbb \n\ @@ -81,7 +76,7 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) } return (__force __sum16)(csum >> 16); } -no_zbb: + #ifndef CONFIG_32BIT csum += ror64(csum, 32); csum >>= 32; diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h index 0b749e71021624..6a372ab9bcf68b 100644 --- a/arch/riscv/include/asm/cmpxchg.h +++ b/arch/riscv/include/asm/cmpxchg.h @@ -370,9 +370,10 @@ static __always_inline void __cmpwait(volatile void *ptr, u32 *__ptr32b; ulong __s, __val, __mask; - asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop", - 0, RISCV_ISA_EXT_ZAWRS, 1) - : : : : no_zawrs); + if (!__riscv_has_extension_likely(0, RISCV_ISA_EXT_ZAWRS)) { + asm volatile(RISCV_PAUSE : : : "memory"); + return; + } switch (size) { case 1: @@ -434,11 +435,6 @@ static __always_inline void __cmpwait(volatile void *ptr, default: BUILD_BUG(); } - - return; - -no_zawrs: - asm volatile(RISCV_PAUSE : : : "memory"); } #define __cmpwait_relaxed(ptr, val) \ diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 91697fbf1f9013..f37a0c3dab8a8c 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -495,8 +495,13 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr) { - asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) - : : : : svvptc); + /* + * Svvptc guarantees that the new valid pte will be visible within + * a bounded timeframe, so when the uarch does not cache invalid + * entries, we don't have to do anything. + */ + if (__riscv_has_extension_unlikely(0, RISCV_ISA_EXT_SVVPTC)) + return; /* * The kernel assumes that TLBs don't cache invalid entries, but @@ -508,12 +513,6 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf, while (nr--) local_flush_tlb_page(address + nr * PAGE_SIZE); -svvptc:; - /* - * Svvptc guarantees that the new valid pte will be visible within - * a bounded timeframe, so when the uarch does not cache invalid - * entries, we don't have to do anything. - */ } #define update_mmu_cache(vma, addr, ptep) \ update_mmu_cache_range(NULL, vma, addr, ptep, 1) diff --git a/arch/riscv/lib/csum.c b/arch/riscv/lib/csum.c index 9408f50ca59a89..420e9eb93e8531 100644 --- a/arch/riscv/lib/csum.c +++ b/arch/riscv/lib/csum.c @@ -40,20 +40,11 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, uproto = (__force unsigned int)htonl(proto); sum += uproto; - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + __riscv_has_extension_likely(0, RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - /* - * Zbb is likely available when the kernel is compiled with Zbb - * support, so nop when Zbb is available and jump when Zbb is - * not available. - */ - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); asm(".option push \n\ .option arch,+zbb \n\ rori %[fold_temp], %[sum], 32 \n\ @@ -66,7 +57,7 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, : [sum] "+r" (sum), [fold_temp] "=&r" (fold_temp)); return (__force __sum16)(sum >> 16); } -no_zbb: + sum += ror64(sum, 32); sum >>= 32; return csum_fold((__force __wsum)sum); @@ -152,21 +143,11 @@ do_csum_with_alignment(const unsigned char *buff, int len) csum = do_csum_common(ptr, end, data); #ifdef CC_HAS_ASM_GOTO_TIED_OUTPUT - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + __riscv_has_extension_likely(0, RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - /* - * Zbb is likely available when the kernel is compiled with Zbb - * support, so nop when Zbb is available and jump when Zbb is - * not available. - */ - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); - #ifdef CONFIG_32BIT asm_goto_output(".option push \n\ .option arch,+zbb \n\ @@ -204,7 +185,7 @@ do_csum_with_alignment(const unsigned char *buff, int len) end: return csum >> 16; } -no_zbb: + #endif /* CC_HAS_ASM_GOTO_TIED_OUTPUT */ #ifndef CONFIG_32BIT csum += ror64(csum, 32); @@ -234,21 +215,11 @@ do_csum_no_alignment(const unsigned char *buff, int len) end = (const unsigned long *)(buff + len); csum = do_csum_common(ptr, end, data); - if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB)) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && + IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + __riscv_has_extension_likely(0, RISCV_ISA_EXT_ZBB)) { unsigned long fold_temp; - /* - * Zbb is likely available when the kernel is compiled with Zbb - * support, so nop when Zbb is available and jump when Zbb is - * not available. - */ - asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, - RISCV_ISA_EXT_ZBB, 1) - : - : - : - : no_zbb); - #ifdef CONFIG_32BIT asm (".option push \n\ .option arch,+zbb \n\ @@ -274,7 +245,7 @@ do_csum_no_alignment(const unsigned char *buff, int len) #endif /* !CONFIG_32BIT */ return csum >> 16; } -no_zbb: + #ifndef CONFIG_32BIT csum += ror64(csum, 32); csum >>= 32; diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c index 8b6c0a112a8db4..289ca6fa6b4de8 100644 --- a/arch/riscv/mm/pgtable.c +++ b/arch/riscv/mm/pgtable.c @@ -9,8 +9,16 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty) { - asm goto(ALTERNATIVE("nop", "j %l[svvptc]", 0, RISCV_ISA_EXT_SVVPTC, 1) - : : : : svvptc); + if (__riscv_has_extension_unlikely(0, RISCV_ISA_EXT_SVVPTC)) { + if (!pte_same(ptep_get(ptep), entry)) { + __set_pte_at(vma->vm_mm, ptep, entry); + /* Here only not svadu is impacted */ + flush_tlb_page(vma, address); + return true; + } + + return false; + } if (!pte_same(ptep_get(ptep), entry)) __set_pte_at(vma->vm_mm, ptep, entry); @@ -19,16 +27,6 @@ int ptep_set_access_flags(struct vm_area_struct *vma, * the case that the PTE changed and the spurious fault case. */ return true; - -svvptc: - if (!pte_same(ptep_get(ptep), entry)) { - __set_pte_at(vma->vm_mm, ptep, entry); - /* Here only not svadu is impacted */ - flush_tlb_page(vma, address); - return true; - } - - return false; } int ptep_test_and_clear_young(struct vm_area_struct *vma,