diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 834bff720582..aaf1f52fbf3e 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -13,31 +13,6 @@ #include #ifndef __ASSEMBLY__ -/* - * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a - * Section C3.1 "A64 instruction index by encoding": - * AArch64 main encoding table - * Bit position - * 28 27 26 25 Encoding Group - * 0 0 - - Unallocated - * 1 0 0 - Data processing, immediate - * 1 0 1 - Branch, exception generation and system instructions - * - 1 - 0 Loads and stores - * - 1 0 1 Data processing - register - * 0 1 1 1 Data processing - SIMD and floating point - * 1 1 1 1 Data processing - SIMD and floating point - * "-" means "don't care" - */ -enum aarch64_insn_encoding_class { - AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */ - AARCH64_INSN_CLS_SVE, /* SVE instructions */ - AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */ - AARCH64_INSN_CLS_DP_REG, /* Data processing - register */ - AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */ - AARCH64_INSN_CLS_LDST, /* Loads and stores */ - AARCH64_INSN_CLS_BR_SYS, /* Branch, exception generation and - * system instructions */ -}; enum aarch64_insn_hint_cr_op { AARCH64_INSN_HINT_NOP = 0x0 << 5, @@ -326,6 +301,23 @@ static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \ return (val); \ } +/* + * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a + * Section C3.1 "A64 instruction index by encoding": + * AArch64 main encoding table + * Bit position + * 28 27 26 25 Encoding Group + * 0 0 - - Unallocated + * 1 0 0 - Data processing, immediate + * 1 0 1 - Branch, exception generation and system instructions + * - 1 - 0 Loads and stores + * - 1 0 1 Data processing - register + * 0 1 1 1 Data processing - SIMD and floating point + * 1 1 1 1 Data processing - SIMD and floating point + * "-" means "don't care" + */ +__AARCH64_INSN_FUNCS(class_branch_sys, 0x1c000000, 0x14000000) + __AARCH64_INSN_FUNCS(adr, 0x9F000000, 0x10000000) __AARCH64_INSN_FUNCS(adrp, 0x9F000000, 0x90000000) __AARCH64_INSN_FUNCS(prfm, 0x3FC00000, 0x39800000) @@ -431,58 +423,122 @@ __AARCH64_INSN_FUNCS(pssbb, 0xFFFFFFFF, 0xD503349F) #undef __AARCH64_INSN_FUNCS -bool aarch64_insn_is_steppable_hint(u32 insn); -bool aarch64_insn_is_branch_imm(u32 insn); - -static inline bool aarch64_insn_is_adr_adrp(u32 insn) +static __always_inline bool aarch64_insn_is_steppable_hint(u32 insn) { - return aarch64_insn_is_adr(insn) || aarch64_insn_is_adrp(insn); + if (!aarch64_insn_is_hint(insn)) + return false; + + switch (insn & 0xFE0) { + case AARCH64_INSN_HINT_XPACLRI: + case AARCH64_INSN_HINT_PACIA_1716: + case AARCH64_INSN_HINT_PACIB_1716: + case AARCH64_INSN_HINT_PACIAZ: + case AARCH64_INSN_HINT_PACIASP: + case AARCH64_INSN_HINT_PACIBZ: + case AARCH64_INSN_HINT_PACIBSP: + case AARCH64_INSN_HINT_BTI: + case AARCH64_INSN_HINT_BTIC: + case AARCH64_INSN_HINT_BTIJ: + case AARCH64_INSN_HINT_BTIJC: + case AARCH64_INSN_HINT_NOP: + return true; + default: + return false; + } } -static inline bool aarch64_insn_is_dsb(u32 insn) +static __always_inline bool aarch64_insn_is_branch(u32 insn) { - return aarch64_insn_is_dsb_base(insn) || aarch64_insn_is_dsb_nxs(insn); + /* b, bl, cb*, tb*, ret*, b.cond, br*, blr* */ + + return aarch64_insn_is_b(insn) || + aarch64_insn_is_bl(insn) || + aarch64_insn_is_cbz(insn) || + aarch64_insn_is_cbnz(insn) || + aarch64_insn_is_tbz(insn) || + aarch64_insn_is_tbnz(insn) || + aarch64_insn_is_ret(insn) || + aarch64_insn_is_ret_auth(insn) || + aarch64_insn_is_br(insn) || + aarch64_insn_is_br_auth(insn) || + aarch64_insn_is_blr(insn) || + aarch64_insn_is_blr_auth(insn) || + aarch64_insn_is_bcond(insn); } -static inline bool aarch64_insn_is_barrier(u32 insn) +static __always_inline bool aarch64_insn_is_branch_imm(u32 insn) { - return aarch64_insn_is_dmb(insn) || aarch64_insn_is_dsb(insn) || - aarch64_insn_is_isb(insn) || aarch64_insn_is_sb(insn) || - aarch64_insn_is_clrex(insn) || aarch64_insn_is_ssbb(insn) || + return aarch64_insn_is_b(insn) || + aarch64_insn_is_bl(insn) || + aarch64_insn_is_tbz(insn) || + aarch64_insn_is_tbnz(insn) || + aarch64_insn_is_cbz(insn) || + aarch64_insn_is_cbnz(insn) || + aarch64_insn_is_bcond(insn); +} + +static __always_inline bool aarch64_insn_is_adr_adrp(u32 insn) +{ + return aarch64_insn_is_adr(insn) || + aarch64_insn_is_adrp(insn); +} + +static __always_inline bool aarch64_insn_is_dsb(u32 insn) +{ + return aarch64_insn_is_dsb_base(insn) || + aarch64_insn_is_dsb_nxs(insn); +} + +static __always_inline bool aarch64_insn_is_barrier(u32 insn) +{ + return aarch64_insn_is_dmb(insn) || + aarch64_insn_is_dsb(insn) || + aarch64_insn_is_isb(insn) || + aarch64_insn_is_sb(insn) || + aarch64_insn_is_clrex(insn) || + aarch64_insn_is_ssbb(insn) || aarch64_insn_is_pssbb(insn); } -static inline bool aarch64_insn_is_store_single(u32 insn) +static __always_inline bool aarch64_insn_is_store_single(u32 insn) { return aarch64_insn_is_store_imm(insn) || aarch64_insn_is_store_pre(insn) || aarch64_insn_is_store_post(insn); } -static inline bool aarch64_insn_is_store_pair(u32 insn) +static __always_inline bool aarch64_insn_is_store_pair(u32 insn) { return aarch64_insn_is_stp(insn) || aarch64_insn_is_stp_pre(insn) || aarch64_insn_is_stp_post(insn); } -static inline bool aarch64_insn_is_load_single(u32 insn) +static __always_inline bool aarch64_insn_is_load_single(u32 insn) { return aarch64_insn_is_load_imm(insn) || aarch64_insn_is_load_pre(insn) || aarch64_insn_is_load_post(insn); } -static inline bool aarch64_insn_is_load_pair(u32 insn) +static __always_inline bool aarch64_insn_is_load_pair(u32 insn) { return aarch64_insn_is_ldp(insn) || aarch64_insn_is_ldp_pre(insn) || aarch64_insn_is_ldp_post(insn); } +static __always_inline bool aarch64_insn_uses_literal(u32 insn) +{ + /* ldr/ldrsw (literal), prfm */ + + return aarch64_insn_is_ldr_lit(insn) || + aarch64_insn_is_ldrsw_lit(insn) || + aarch64_insn_is_adr_adrp(insn) || + aarch64_insn_is_prfm_lit(insn); +} + enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn); -bool aarch64_insn_uses_literal(u32 insn); -bool aarch64_insn_is_branch(u32 insn); u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn); u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, u32 insn, u64 imm); @@ -496,8 +552,18 @@ u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr, enum aarch64_insn_branch_type type); u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, enum aarch64_insn_condition cond); -u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op); -u32 aarch64_insn_gen_nop(void); + +static __always_inline u32 +aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op) +{ + return aarch64_insn_get_hint_value() | op; +} + +static __always_inline u32 aarch64_insn_gen_nop(void) +{ + return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP); +} + u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg, enum aarch64_insn_branch_type type); u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg, @@ -580,10 +646,6 @@ u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant, enum aarch64_insn_register Rn, enum aarch64_insn_register Rd, u8 lsb); -u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base, - enum aarch64_insn_prfm_type type, - enum aarch64_insn_prfm_target target, - enum aarch64_insn_prfm_policy policy); #ifdef CONFIG_ARM64_LSE_ATOMICS u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result, enum aarch64_insn_register address, diff --git a/arch/arm64/include/asm/uprobes.h b/arch/arm64/include/asm/uprobes.h index 315eef654e39..ba4bff5ca674 100644 --- a/arch/arm64/include/asm/uprobes.h +++ b/arch/arm64/include/asm/uprobes.h @@ -12,7 +12,7 @@ #define MAX_UINSN_BYTES AARCH64_INSN_SIZE -#define UPROBE_SWBP_INSN BRK64_OPCODE_UPROBES +#define UPROBE_SWBP_INSN cpu_to_le32(BRK64_OPCODE_UPROBES) #define UPROBE_SWBP_INSN_SIZE AARCH64_INSN_SIZE #define UPROBE_XOL_SLOT_BYTES MAX_UINSN_BYTES diff --git a/arch/arm64/kernel/probes/decode-insn.c b/arch/arm64/kernel/probes/decode-insn.c index 104101f633b1..968d5fffe233 100644 --- a/arch/arm64/kernel/probes/decode-insn.c +++ b/arch/arm64/kernel/probes/decode-insn.c @@ -24,7 +24,7 @@ static bool __kprobes aarch64_insn_is_steppable(u32 insn) * currently safe. Lastly, MSR instructions can do any number of nasty * things we can't handle during single-stepping. */ - if (aarch64_get_insn_class(insn) == AARCH64_INSN_CLS_BR_SYS) { + if (aarch64_insn_is_class_branch_sys(insn)) { if (aarch64_insn_is_branch(insn) || aarch64_insn_is_msr_imm(insn) || aarch64_insn_is_msr_reg(insn) || diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c index 49e972beeac7..924934cb85ee 100644 --- a/arch/arm64/lib/insn.c +++ b/arch/arm64/lib/insn.c @@ -20,91 +20,6 @@ #define AARCH64_INSN_N_BIT BIT(22) #define AARCH64_INSN_LSL_12 BIT(22) -static const int aarch64_insn_encoding_class[] = { - AARCH64_INSN_CLS_UNKNOWN, - AARCH64_INSN_CLS_UNKNOWN, - AARCH64_INSN_CLS_SVE, - AARCH64_INSN_CLS_UNKNOWN, - AARCH64_INSN_CLS_LDST, - AARCH64_INSN_CLS_DP_REG, - AARCH64_INSN_CLS_LDST, - AARCH64_INSN_CLS_DP_FPSIMD, - AARCH64_INSN_CLS_DP_IMM, - AARCH64_INSN_CLS_DP_IMM, - AARCH64_INSN_CLS_BR_SYS, - AARCH64_INSN_CLS_BR_SYS, - AARCH64_INSN_CLS_LDST, - AARCH64_INSN_CLS_DP_REG, - AARCH64_INSN_CLS_LDST, - AARCH64_INSN_CLS_DP_FPSIMD, -}; - -enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn) -{ - return aarch64_insn_encoding_class[(insn >> 25) & 0xf]; -} - -bool __kprobes aarch64_insn_is_steppable_hint(u32 insn) -{ - if (!aarch64_insn_is_hint(insn)) - return false; - - switch (insn & 0xFE0) { - case AARCH64_INSN_HINT_XPACLRI: - case AARCH64_INSN_HINT_PACIA_1716: - case AARCH64_INSN_HINT_PACIB_1716: - case AARCH64_INSN_HINT_PACIAZ: - case AARCH64_INSN_HINT_PACIASP: - case AARCH64_INSN_HINT_PACIBZ: - case AARCH64_INSN_HINT_PACIBSP: - case AARCH64_INSN_HINT_BTI: - case AARCH64_INSN_HINT_BTIC: - case AARCH64_INSN_HINT_BTIJ: - case AARCH64_INSN_HINT_BTIJC: - case AARCH64_INSN_HINT_NOP: - return true; - default: - return false; - } -} - -bool aarch64_insn_is_branch_imm(u32 insn) -{ - return (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn) || - aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn) || - aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || - aarch64_insn_is_bcond(insn)); -} - -bool __kprobes aarch64_insn_uses_literal(u32 insn) -{ - /* ldr/ldrsw (literal), prfm */ - - return aarch64_insn_is_ldr_lit(insn) || - aarch64_insn_is_ldrsw_lit(insn) || - aarch64_insn_is_adr_adrp(insn) || - aarch64_insn_is_prfm_lit(insn); -} - -bool __kprobes aarch64_insn_is_branch(u32 insn) -{ - /* b, bl, cb*, tb*, ret*, b.cond, br*, blr* */ - - return aarch64_insn_is_b(insn) || - aarch64_insn_is_bl(insn) || - aarch64_insn_is_cbz(insn) || - aarch64_insn_is_cbnz(insn) || - aarch64_insn_is_tbz(insn) || - aarch64_insn_is_tbnz(insn) || - aarch64_insn_is_ret(insn) || - aarch64_insn_is_ret_auth(insn) || - aarch64_insn_is_br(insn) || - aarch64_insn_is_br_auth(insn) || - aarch64_insn_is_blr(insn) || - aarch64_insn_is_blr_auth(insn) || - aarch64_insn_is_bcond(insn); -} - static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type, u32 *maskp, int *shiftp) { @@ -435,16 +350,6 @@ u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, offset >> 2); } -u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op) -{ - return aarch64_insn_get_hint_value() | op; -} - -u32 __kprobes aarch64_insn_gen_nop(void) -{ - return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP); -} - u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg, enum aarch64_insn_branch_type type) { @@ -816,76 +721,6 @@ u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, } #endif -static u32 aarch64_insn_encode_prfm_imm(enum aarch64_insn_prfm_type type, - enum aarch64_insn_prfm_target target, - enum aarch64_insn_prfm_policy policy, - u32 insn) -{ - u32 imm_type = 0, imm_target = 0, imm_policy = 0; - - switch (type) { - case AARCH64_INSN_PRFM_TYPE_PLD: - break; - case AARCH64_INSN_PRFM_TYPE_PLI: - imm_type = BIT(0); - break; - case AARCH64_INSN_PRFM_TYPE_PST: - imm_type = BIT(1); - break; - default: - pr_err("%s: unknown prfm type encoding %d\n", __func__, type); - return AARCH64_BREAK_FAULT; - } - - switch (target) { - case AARCH64_INSN_PRFM_TARGET_L1: - break; - case AARCH64_INSN_PRFM_TARGET_L2: - imm_target = BIT(0); - break; - case AARCH64_INSN_PRFM_TARGET_L3: - imm_target = BIT(1); - break; - default: - pr_err("%s: unknown prfm target encoding %d\n", __func__, target); - return AARCH64_BREAK_FAULT; - } - - switch (policy) { - case AARCH64_INSN_PRFM_POLICY_KEEP: - break; - case AARCH64_INSN_PRFM_POLICY_STRM: - imm_policy = BIT(0); - break; - default: - pr_err("%s: unknown prfm policy encoding %d\n", __func__, policy); - return AARCH64_BREAK_FAULT; - } - - /* In this case, imm5 is encoded into Rt field. */ - insn &= ~GENMASK(4, 0); - insn |= imm_policy | (imm_target << 1) | (imm_type << 3); - - return insn; -} - -u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base, - enum aarch64_insn_prfm_type type, - enum aarch64_insn_prfm_target target, - enum aarch64_insn_prfm_policy policy) -{ - u32 insn = aarch64_insn_get_prfm_value(); - - insn = aarch64_insn_encode_ldst_size(AARCH64_INSN_SIZE_64, insn); - - insn = aarch64_insn_encode_prfm_imm(type, target, policy, insn); - - insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, - base); - - return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, 0); -} - u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst, enum aarch64_insn_register src, int imm, enum aarch64_insn_variant variant,