diff --git a/Documentation/devicetree/bindings/riscv/extensions.yaml b/Documentation/devicetree/bindings/riscv/extensions.yaml index ede6a58ccf5347..6b8c21807a2da7 100644 --- a/Documentation/devicetree/bindings/riscv/extensions.yaml +++ b/Documentation/devicetree/bindings/riscv/extensions.yaml @@ -248,6 +248,11 @@ properties: ratified at commit e87412e621f1 ("integrate Zaamo and Zalrsc text (#1304)") of the unprivileged ISA specification. + - const: zalasr + description: | + The standard Zalasr extension for load-acquire/store-release as frozen + at commit 194f0094 ("Version 0.9 for freeze") of riscv-zalasr. + - const: zawrs description: | The Zawrs extension for entering a low-power state or for trapping diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h index b8c5726d86acb1..b1d2a9a85256d2 100644 --- a/arch/riscv/include/asm/barrier.h +++ b/arch/riscv/include/asm/barrier.h @@ -51,19 +51,76 @@ */ #define smp_mb__after_spinlock() RISCV_FENCE(iorw, iorw) -#define __smp_store_release(p, v) \ -do { \ - compiletime_assert_atomic_type(*p); \ - RISCV_FENCE(rw, w); \ - WRITE_ONCE(*p, v); \ +extern void __bad_size_call_parameter(void); + +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + switch (sizeof(*p)) { \ + case 1: \ + asm volatile(ALTERNATIVE("fence rw, w;\t\nsb %0, 0(%1)\t\n", \ + SB_RL(%0, %1) "\t\nnop\t\n", \ + 0, RISCV_ISA_EXT_ZALASR, 1) \ + : : "r" (v), "r" (p) : "memory"); \ + break; \ + case 2: \ + asm volatile(ALTERNATIVE("fence rw, w;\t\nsh %0, 0(%1)\t\n", \ + SH_RL(%0, %1) "\t\nnop\t\n", \ + 0, RISCV_ISA_EXT_ZALASR, 1) \ + : : "r" (v), "r" (p) : "memory"); \ + break; \ + case 4: \ + asm volatile(ALTERNATIVE("fence rw, w;\t\nsw %0, 0(%1)\t\n", \ + SW_RL(%0, %1) "\t\nnop\t\n", \ + 0, RISCV_ISA_EXT_ZALASR, 1) \ + : : "r" (v), "r" (p) : "memory"); \ + break; \ + case 8: \ + asm volatile(ALTERNATIVE("fence rw, w;\t\nsd %0, 0(%1)\t\n", \ + SD_RL(%0, %1) "\t\nnop\t\n", \ + 0, RISCV_ISA_EXT_ZALASR, 1) \ + : : "r" (v), "r" (p) : "memory"); \ + break; \ + default: \ + __bad_size_call_parameter(); \ + break; \ + } \ } while (0) -#define __smp_load_acquire(p) \ -({ \ - typeof(*p) ___p1 = READ_ONCE(*p); \ - compiletime_assert_atomic_type(*p); \ - RISCV_FENCE(r, rw); \ - ___p1; \ +#define __smp_load_acquire(p) \ +({ \ + TYPEOF_UNQUAL(*p) val; \ + compiletime_assert_atomic_type(*p); \ + switch (sizeof(*p)) { \ + case 1: \ + asm volatile(ALTERNATIVE("lb %0, 0(%1)\t\nfence r, rw\t\n", \ + LB_AQ(%0, %1) "\t\nnop\t\n", \ + 0, RISCV_ISA_EXT_ZALASR, 1) \ + : "=r" (val) : "r" (p) : "memory"); \ + break; \ + case 2: \ + asm volatile(ALTERNATIVE("lh %0, 0(%1)\t\nfence r, rw\t\n", \ + LH_AQ(%0, %1) "\t\nnop\t\n", \ + 0, RISCV_ISA_EXT_ZALASR, 1) \ + : "=r" (val) : "r" (p) : "memory"); \ + break; \ + case 4: \ + asm volatile(ALTERNATIVE("lw %0, 0(%1)\t\nfence r, rw\t\n", \ + LW_AQ(%0, %1) "\t\nnop\t\n", \ + 0, RISCV_ISA_EXT_ZALASR, 1) \ + : "=r" (val) : "r" (p) : "memory"); \ + break; \ + case 8: \ + asm volatile(ALTERNATIVE("ld %0, 0(%1)\t\nfence r, rw\t\n", \ + LD_AQ(%0, %1) "\t\nnop\t\n", \ + 0, RISCV_ISA_EXT_ZALASR, 1) \ + : "=r" (val) : "r" (p) : "memory"); \ + break; \ + default: \ + __bad_size_call_parameter(); \ + break; \ + } \ + val; \ }) #ifdef CONFIG_RISCV_ISA_ZAWRS diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index affd63e11b0a34..ae3852c4f2ca25 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -106,6 +106,7 @@ #define RISCV_ISA_EXT_ZAAMO 97 #define RISCV_ISA_EXT_ZALRSC 98 #define RISCV_ISA_EXT_ZICBOP 99 +#define RISCV_ISA_EXT_ZALASR 100 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h index d5adbaec1d010d..3fec7e66ce50f2 100644 --- a/arch/riscv/include/asm/insn-def.h +++ b/arch/riscv/include/asm/insn-def.h @@ -179,6 +179,7 @@ #define RV___RS1(v) __RV_REG(v) #define RV___RS2(v) __RV_REG(v) +#define RV_OPCODE_AMO RV_OPCODE(47) #define RV_OPCODE_MISC_MEM RV_OPCODE(15) #define RV_OPCODE_OP_IMM RV_OPCODE(19) #define RV_OPCODE_SYSTEM RV_OPCODE(115) @@ -208,6 +209,84 @@ __ASM_STR(.error "hlv.d requires 64-bit support") #endif +#define LB_AQ(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(0), FUNC7(26), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LB_AQRL(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(0), FUNC7(27), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LH_AQ(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(1), FUNC7(26), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LH_AQRL(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(1), FUNC7(27), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LW_AQ(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(2), FUNC7(26), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LW_AQRL(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(2), FUNC7(27), \ + RD(dest), RS1(addr), __RS2(0)) + +#define SB_RL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(0), FUNC7(29), \ + __RD(0), RS1(addr), RS2(src)) + +#define SB_AQRL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(0), FUNC7(31), \ + __RD(0), RS1(addr), RS2(src)) + +#define SH_RL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(1), FUNC7(29), \ + __RD(0), RS1(addr), RS2(src)) + +#define SH_AQRL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(1), FUNC7(31), \ + __RD(0), RS1(addr), RS2(src)) + +#define SW_RL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(2), FUNC7(29), \ + __RD(0), RS1(addr), RS2(src)) + +#define SW_AQRL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(2), FUNC7(31), \ + __RD(0), RS1(addr), RS2(src)) + +#ifdef CONFIG_64BIT +#define LD_AQ(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(3), FUNC7(26), \ + RD(dest), RS1(addr), __RS2(0)) + +#define LD_AQRL(dest, addr) \ + INSN_R(OPCODE_AMO, FUNC3(3), FUNC7(27), \ + RD(dest), RS1(addr), __RS2(0)) + +#define SD_RL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(3), FUNC7(29), \ + __RD(0), RS1(addr), RS2(src)) + +#define SD_AQRL(src, addr) \ + INSN_R(OPCODE_AMO, FUNC3(3), FUNC7(31), \ + __RD(0), RS1(addr), RS2(src)) +#else +#define LD_AQ(dest, addr) \ + __ASM_STR(.error "ld.aq requires 64-bit support") + +#define LD_AQRL(dest, addr) \ + __ASM_STR(.error "ld.aqrl requires 64-bit support") + +#define SD_RL(dest, addr) \ + __ASM_STR(.error "sd.rl requires 64-bit support") + +#define SD_AQRL(dest, addr) \ + __ASM_STR(.error "sd.aqrl requires 64-bit support") +#endif + #define SINVAL_VMA(vaddr, asid) \ INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(11), \ __RD(0), RS1(vaddr), RS2(asid)) diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 743d53415572e0..bf9d3d92bf3728 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -472,6 +472,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO), __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA), __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), + __RISCV_ISA_EXT_DATA(zalasr, RISCV_ISA_EXT_ZALASR), __RISCV_ISA_EXT_DATA(zalrsc, RISCV_ISA_EXT_ZALRSC), __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS), __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA),