mirror of
https://github.com/Motorhead1991/qemu.git
synced 2025-08-07 17:53:56 -06:00
tcg/aarch64: Introduce prepare_host_addr
Merge tcg_out_tlb_load, add_qemu_ldst_label, tcg_out_test_alignment, and some code that lived in both tcg_out_qemu_ld and tcg_out_qemu_st into one function that returns HostAddress and TCGLabelQemuLdst structures. Reviewed-by: Alex Bennée <alex.bennee@linaro.org> Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson
parent
1fac4648fe
commit
1e612dd66a
1 changed files with 133 additions and 180 deletions
|
|
@ -1667,113 +1667,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
|
||||||
tcg_out_goto(s, lb->raddr);
|
tcg_out_goto(s, lb->raddr);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void add_qemu_ldst_label(TCGContext *s, bool is_ld, MemOpIdx oi,
|
|
||||||
TCGType ext, TCGReg data_reg, TCGReg addr_reg,
|
|
||||||
tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
|
|
||||||
{
|
|
||||||
TCGLabelQemuLdst *label = new_ldst_label(s);
|
|
||||||
|
|
||||||
label->is_ld = is_ld;
|
|
||||||
label->oi = oi;
|
|
||||||
label->type = ext;
|
|
||||||
label->datalo_reg = data_reg;
|
|
||||||
label->addrlo_reg = addr_reg;
|
|
||||||
label->raddr = tcg_splitwx_to_rx(raddr);
|
|
||||||
label->label_ptr[0] = label_ptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* We expect to use a 7-bit scaled negative offset from ENV. */
|
|
||||||
QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
|
|
||||||
QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -512);
|
|
||||||
|
|
||||||
/* These offsets are built into the LDP below. */
|
|
||||||
QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, mask) != 0);
|
|
||||||
QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, table) != 8);
|
|
||||||
|
|
||||||
/* Load and compare a TLB entry, emitting the conditional jump to the
|
|
||||||
slow path for the failure case, which will be patched later when finalizing
|
|
||||||
the slow path. Generated code returns the host addend in X1,
|
|
||||||
clobbers X0,X2,X3,TMP. */
|
|
||||||
static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, MemOp opc,
|
|
||||||
tcg_insn_unit **label_ptr, int mem_index,
|
|
||||||
bool is_read)
|
|
||||||
{
|
|
||||||
unsigned a_bits = get_alignment_bits(opc);
|
|
||||||
unsigned s_bits = opc & MO_SIZE;
|
|
||||||
unsigned a_mask = (1u << a_bits) - 1;
|
|
||||||
unsigned s_mask = (1u << s_bits) - 1;
|
|
||||||
TCGReg x3;
|
|
||||||
TCGType mask_type;
|
|
||||||
uint64_t compare_mask;
|
|
||||||
|
|
||||||
mask_type = (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32
|
|
||||||
? TCG_TYPE_I64 : TCG_TYPE_I32);
|
|
||||||
|
|
||||||
/* Load env_tlb(env)->f[mmu_idx].{mask,table} into {x0,x1}. */
|
|
||||||
tcg_out_insn(s, 3314, LDP, TCG_REG_X0, TCG_REG_X1, TCG_AREG0,
|
|
||||||
TLB_MASK_TABLE_OFS(mem_index), 1, 0);
|
|
||||||
|
|
||||||
/* Extract the TLB index from the address into X0. */
|
|
||||||
tcg_out_insn(s, 3502S, AND_LSR, mask_type == TCG_TYPE_I64,
|
|
||||||
TCG_REG_X0, TCG_REG_X0, addr_reg,
|
|
||||||
TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
|
|
||||||
|
|
||||||
/* Add the tlb_table pointer, creating the CPUTLBEntry address into X1. */
|
|
||||||
tcg_out_insn(s, 3502, ADD, 1, TCG_REG_X1, TCG_REG_X1, TCG_REG_X0);
|
|
||||||
|
|
||||||
/* Load the tlb comparator into X0, and the fast path addend into X1. */
|
|
||||||
tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_X0, TCG_REG_X1, is_read
|
|
||||||
? offsetof(CPUTLBEntry, addr_read)
|
|
||||||
: offsetof(CPUTLBEntry, addr_write));
|
|
||||||
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_X1, TCG_REG_X1,
|
|
||||||
offsetof(CPUTLBEntry, addend));
|
|
||||||
|
|
||||||
/* For aligned accesses, we check the first byte and include the alignment
|
|
||||||
bits within the address. For unaligned access, we check that we don't
|
|
||||||
cross pages using the address of the last byte of the access. */
|
|
||||||
if (a_bits >= s_bits) {
|
|
||||||
x3 = addr_reg;
|
|
||||||
} else {
|
|
||||||
tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64,
|
|
||||||
TCG_REG_X3, addr_reg, s_mask - a_mask);
|
|
||||||
x3 = TCG_REG_X3;
|
|
||||||
}
|
|
||||||
compare_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
|
|
||||||
|
|
||||||
/* Store the page mask part of the address into X3. */
|
|
||||||
tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64,
|
|
||||||
TCG_REG_X3, x3, compare_mask);
|
|
||||||
|
|
||||||
/* Perform the address comparison. */
|
|
||||||
tcg_out_cmp(s, TARGET_LONG_BITS == 64, TCG_REG_X0, TCG_REG_X3, 0);
|
|
||||||
|
|
||||||
/* If not equal, we jump to the slow path. */
|
|
||||||
*label_ptr = s->code_ptr;
|
|
||||||
tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
#else
|
#else
|
||||||
static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addr_reg,
|
|
||||||
unsigned a_bits)
|
|
||||||
{
|
|
||||||
unsigned a_mask = (1 << a_bits) - 1;
|
|
||||||
TCGLabelQemuLdst *label = new_ldst_label(s);
|
|
||||||
|
|
||||||
label->is_ld = is_ld;
|
|
||||||
label->addrlo_reg = addr_reg;
|
|
||||||
|
|
||||||
/* tst addr, #mask */
|
|
||||||
tcg_out_logicali(s, I3404_ANDSI, 0, TCG_REG_XZR, addr_reg, a_mask);
|
|
||||||
|
|
||||||
label->label_ptr[0] = s->code_ptr;
|
|
||||||
|
|
||||||
/* b.ne slow_path */
|
|
||||||
tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
|
|
||||||
|
|
||||||
label->raddr = tcg_splitwx_to_rx(s->code_ptr);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
|
static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l)
|
||||||
{
|
{
|
||||||
if (!reloc_pc19(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
|
if (!reloc_pc19(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
|
||||||
|
|
@ -1801,6 +1695,125 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
|
||||||
}
|
}
|
||||||
#endif /* CONFIG_SOFTMMU */
|
#endif /* CONFIG_SOFTMMU */
|
||||||
|
|
||||||
|
/*
|
||||||
|
* For softmmu, perform the TLB load and compare.
|
||||||
|
* For useronly, perform any required alignment tests.
|
||||||
|
* In both cases, return a TCGLabelQemuLdst structure if the slow path
|
||||||
|
* is required and fill in @h with the host address for the fast path.
|
||||||
|
*/
|
||||||
|
static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h,
|
||||||
|
TCGReg addr_reg, MemOpIdx oi,
|
||||||
|
bool is_ld)
|
||||||
|
{
|
||||||
|
TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
|
||||||
|
TCGLabelQemuLdst *ldst = NULL;
|
||||||
|
MemOp opc = get_memop(oi);
|
||||||
|
unsigned a_bits = get_alignment_bits(opc);
|
||||||
|
unsigned a_mask = (1u << a_bits) - 1;
|
||||||
|
|
||||||
|
#ifdef CONFIG_SOFTMMU
|
||||||
|
unsigned s_bits = opc & MO_SIZE;
|
||||||
|
unsigned s_mask = (1u << s_bits) - 1;
|
||||||
|
unsigned mem_index = get_mmuidx(oi);
|
||||||
|
TCGReg x3;
|
||||||
|
TCGType mask_type;
|
||||||
|
uint64_t compare_mask;
|
||||||
|
|
||||||
|
ldst = new_ldst_label(s);
|
||||||
|
ldst->is_ld = is_ld;
|
||||||
|
ldst->oi = oi;
|
||||||
|
ldst->addrlo_reg = addr_reg;
|
||||||
|
|
||||||
|
mask_type = (TARGET_PAGE_BITS + CPU_TLB_DYN_MAX_BITS > 32
|
||||||
|
? TCG_TYPE_I64 : TCG_TYPE_I32);
|
||||||
|
|
||||||
|
/* Load env_tlb(env)->f[mmu_idx].{mask,table} into {x0,x1}. */
|
||||||
|
QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0);
|
||||||
|
QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -512);
|
||||||
|
QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, mask) != 0);
|
||||||
|
QEMU_BUILD_BUG_ON(offsetof(CPUTLBDescFast, table) != 8);
|
||||||
|
tcg_out_insn(s, 3314, LDP, TCG_REG_X0, TCG_REG_X1, TCG_AREG0,
|
||||||
|
TLB_MASK_TABLE_OFS(mem_index), 1, 0);
|
||||||
|
|
||||||
|
/* Extract the TLB index from the address into X0. */
|
||||||
|
tcg_out_insn(s, 3502S, AND_LSR, mask_type == TCG_TYPE_I64,
|
||||||
|
TCG_REG_X0, TCG_REG_X0, addr_reg,
|
||||||
|
TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
|
||||||
|
|
||||||
|
/* Add the tlb_table pointer, creating the CPUTLBEntry address into X1. */
|
||||||
|
tcg_out_insn(s, 3502, ADD, 1, TCG_REG_X1, TCG_REG_X1, TCG_REG_X0);
|
||||||
|
|
||||||
|
/* Load the tlb comparator into X0, and the fast path addend into X1. */
|
||||||
|
tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_X0, TCG_REG_X1,
|
||||||
|
is_ld ? offsetof(CPUTLBEntry, addr_read)
|
||||||
|
: offsetof(CPUTLBEntry, addr_write));
|
||||||
|
tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_X1, TCG_REG_X1,
|
||||||
|
offsetof(CPUTLBEntry, addend));
|
||||||
|
|
||||||
|
/*
|
||||||
|
* For aligned accesses, we check the first byte and include the alignment
|
||||||
|
* bits within the address. For unaligned access, we check that we don't
|
||||||
|
* cross pages using the address of the last byte of the access.
|
||||||
|
*/
|
||||||
|
if (a_bits >= s_bits) {
|
||||||
|
x3 = addr_reg;
|
||||||
|
} else {
|
||||||
|
tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64,
|
||||||
|
TCG_REG_X3, addr_reg, s_mask - a_mask);
|
||||||
|
x3 = TCG_REG_X3;
|
||||||
|
}
|
||||||
|
compare_mask = (uint64_t)TARGET_PAGE_MASK | a_mask;
|
||||||
|
|
||||||
|
/* Store the page mask part of the address into X3. */
|
||||||
|
tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64,
|
||||||
|
TCG_REG_X3, x3, compare_mask);
|
||||||
|
|
||||||
|
/* Perform the address comparison. */
|
||||||
|
tcg_out_cmp(s, TARGET_LONG_BITS == 64, TCG_REG_X0, TCG_REG_X3, 0);
|
||||||
|
|
||||||
|
/* If not equal, we jump to the slow path. */
|
||||||
|
ldst->label_ptr[0] = s->code_ptr;
|
||||||
|
tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
|
||||||
|
|
||||||
|
*h = (HostAddress){
|
||||||
|
.base = TCG_REG_X1,
|
||||||
|
.index = addr_reg,
|
||||||
|
.index_ext = addr_type
|
||||||
|
};
|
||||||
|
#else
|
||||||
|
if (a_mask) {
|
||||||
|
ldst = new_ldst_label(s);
|
||||||
|
|
||||||
|
ldst->is_ld = is_ld;
|
||||||
|
ldst->oi = oi;
|
||||||
|
ldst->addrlo_reg = addr_reg;
|
||||||
|
|
||||||
|
/* tst addr, #mask */
|
||||||
|
tcg_out_logicali(s, I3404_ANDSI, 0, TCG_REG_XZR, addr_reg, a_mask);
|
||||||
|
|
||||||
|
/* b.ne slow_path */
|
||||||
|
ldst->label_ptr[0] = s->code_ptr;
|
||||||
|
tcg_out_insn(s, 3202, B_C, TCG_COND_NE, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (USE_GUEST_BASE) {
|
||||||
|
*h = (HostAddress){
|
||||||
|
.base = TCG_REG_GUEST_BASE,
|
||||||
|
.index = addr_reg,
|
||||||
|
.index_ext = addr_type
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
*h = (HostAddress){
|
||||||
|
.base = addr_reg,
|
||||||
|
.index = TCG_REG_XZR,
|
||||||
|
.index_ext = TCG_TYPE_I64
|
||||||
|
};
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return ldst;
|
||||||
|
}
|
||||||
|
|
||||||
static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext,
|
static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp memop, TCGType ext,
|
||||||
TCGReg data_r, HostAddress h)
|
TCGReg data_r, HostAddress h)
|
||||||
{
|
{
|
||||||
|
|
@ -1857,93 +1870,33 @@ static void tcg_out_qemu_st_direct(TCGContext *s, MemOp memop,
|
||||||
static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
|
static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
|
||||||
MemOpIdx oi, TCGType data_type)
|
MemOpIdx oi, TCGType data_type)
|
||||||
{
|
{
|
||||||
MemOp memop = get_memop(oi);
|
TCGLabelQemuLdst *ldst;
|
||||||
TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
|
|
||||||
HostAddress h;
|
HostAddress h;
|
||||||
|
|
||||||
/* Byte swapping is left to middle-end expansion. */
|
ldst = prepare_host_addr(s, &h, addr_reg, oi, true);
|
||||||
tcg_debug_assert((memop & MO_BSWAP) == 0);
|
tcg_out_qemu_ld_direct(s, get_memop(oi), data_type, data_reg, h);
|
||||||
|
|
||||||
#ifdef CONFIG_SOFTMMU
|
if (ldst) {
|
||||||
tcg_insn_unit *label_ptr;
|
ldst->type = data_type;
|
||||||
|
ldst->datalo_reg = data_reg;
|
||||||
tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 1);
|
ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
|
||||||
|
|
||||||
h = (HostAddress){
|
|
||||||
.base = TCG_REG_X1,
|
|
||||||
.index = addr_reg,
|
|
||||||
.index_ext = addr_type
|
|
||||||
};
|
|
||||||
tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, h);
|
|
||||||
|
|
||||||
add_qemu_ldst_label(s, true, oi, data_type, data_reg, addr_reg,
|
|
||||||
s->code_ptr, label_ptr);
|
|
||||||
#else /* !CONFIG_SOFTMMU */
|
|
||||||
unsigned a_bits = get_alignment_bits(memop);
|
|
||||||
if (a_bits) {
|
|
||||||
tcg_out_test_alignment(s, true, addr_reg, a_bits);
|
|
||||||
}
|
}
|
||||||
if (USE_GUEST_BASE) {
|
|
||||||
h = (HostAddress){
|
|
||||||
.base = TCG_REG_GUEST_BASE,
|
|
||||||
.index = addr_reg,
|
|
||||||
.index_ext = addr_type
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
h = (HostAddress){
|
|
||||||
.base = addr_reg,
|
|
||||||
.index = TCG_REG_XZR,
|
|
||||||
.index_ext = TCG_TYPE_I64
|
|
||||||
};
|
|
||||||
}
|
|
||||||
tcg_out_qemu_ld_direct(s, memop, data_type, data_reg, h);
|
|
||||||
#endif /* CONFIG_SOFTMMU */
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
|
static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
|
||||||
MemOpIdx oi, TCGType data_type)
|
MemOpIdx oi, TCGType data_type)
|
||||||
{
|
{
|
||||||
MemOp memop = get_memop(oi);
|
TCGLabelQemuLdst *ldst;
|
||||||
TCGType addr_type = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
|
|
||||||
HostAddress h;
|
HostAddress h;
|
||||||
|
|
||||||
/* Byte swapping is left to middle-end expansion. */
|
ldst = prepare_host_addr(s, &h, addr_reg, oi, false);
|
||||||
tcg_debug_assert((memop & MO_BSWAP) == 0);
|
tcg_out_qemu_st_direct(s, get_memop(oi), data_reg, h);
|
||||||
|
|
||||||
#ifdef CONFIG_SOFTMMU
|
if (ldst) {
|
||||||
tcg_insn_unit *label_ptr;
|
ldst->type = data_type;
|
||||||
|
ldst->datalo_reg = data_reg;
|
||||||
tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, get_mmuidx(oi), 0);
|
ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
|
||||||
|
|
||||||
h = (HostAddress){
|
|
||||||
.base = TCG_REG_X1,
|
|
||||||
.index = addr_reg,
|
|
||||||
.index_ext = addr_type
|
|
||||||
};
|
|
||||||
tcg_out_qemu_st_direct(s, memop, data_reg, h);
|
|
||||||
|
|
||||||
add_qemu_ldst_label(s, false, oi, data_type, data_reg, addr_reg,
|
|
||||||
s->code_ptr, label_ptr);
|
|
||||||
#else /* !CONFIG_SOFTMMU */
|
|
||||||
unsigned a_bits = get_alignment_bits(memop);
|
|
||||||
if (a_bits) {
|
|
||||||
tcg_out_test_alignment(s, false, addr_reg, a_bits);
|
|
||||||
}
|
}
|
||||||
if (USE_GUEST_BASE) {
|
|
||||||
h = (HostAddress){
|
|
||||||
.base = TCG_REG_GUEST_BASE,
|
|
||||||
.index = addr_reg,
|
|
||||||
.index_ext = addr_type
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
h = (HostAddress){
|
|
||||||
.base = addr_reg,
|
|
||||||
.index = TCG_REG_XZR,
|
|
||||||
.index_ext = TCG_TYPE_I64
|
|
||||||
};
|
|
||||||
}
|
|
||||||
tcg_out_qemu_st_direct(s, memop, data_reg, h);
|
|
||||||
#endif /* CONFIG_SOFTMMU */
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static const tcg_insn_unit *tb_ret_addr;
|
static const tcg_insn_unit *tb_ret_addr;
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue