From 5a55646fc3bb591a636a6cd8ef64bb42672eadf1 Mon Sep 17 00:00:00 2001 From: Benjamin Bouvier Date: Wed, 8 Jul 2020 15:22:00 +0200 Subject: [PATCH] machinst x64: support out-of-bounds memory accesses; --- cranelift/codegen/src/isa/x64/abi.rs | 15 +- cranelift/codegen/src/isa/x64/inst/args.rs | 5 +- cranelift/codegen/src/isa/x64/inst/emit.rs | 130 +++- .../codegen/src/isa/x64/inst/emit_tests.rs | 585 +++++++++++------- cranelift/codegen/src/isa/x64/inst/mod.rs | 110 +++- cranelift/codegen/src/isa/x64/lower.rs | 76 ++- 6 files changed, 598 insertions(+), 323 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 4adbf966ccc8..cc8c41deab48 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -364,6 +364,7 @@ impl ABIBody for X64ABIBody { ext_mode, RegMem::reg(from_reg.to_reg()), dest_reg, + /* infallible load */ None, )); } (ArgumentExtension::Sext, Some(ext_mode)) => { @@ -371,6 +372,7 @@ impl ABIBody for X64ABIBody { ext_mode, RegMem::reg(from_reg.to_reg()), dest_reg, + /* infallible load */ None, )); } _ => ret.push(Inst::gen_move(dest_reg, from_reg.to_reg(), ty)), @@ -394,6 +396,7 @@ impl ABIBody for X64ABIBody { ext_mode, RegMem::reg(from_reg.to_reg()), from_reg, + /* infallible load */ None, )); } (ArgumentExtension::Sext, Some(ext_mode)) => { @@ -401,6 +404,7 @@ impl ABIBody for X64ABIBody { ext_mode, RegMem::reg(from_reg.to_reg()), from_reg, + /* infallible load */ None, )); } _ => {} @@ -965,8 +969,13 @@ fn load_stack(mem: impl Into, into_reg: Writable, ty: Type) let mem = mem.into(); match ext_mode { - Some(ext_mode) => Inst::movsx_rm_r(ext_mode, RegMem::mem(mem), into_reg), - None => Inst::mov64_m_r(mem, into_reg), + Some(ext_mode) => Inst::movsx_rm_r( + ext_mode, + RegMem::mem(mem), + into_reg, + /* infallible load */ None, + ), + None => Inst::mov64_m_r(mem, into_reg, None /* infallible */), } } @@ -982,7 +991,7 @@ fn store_stack(mem: impl Into, from_reg: Reg, ty: Type) -> Inst }; let mem = mem.into(); if is_int { - Inst::mov_r_m(size, from_reg, mem) + Inst::mov_r_m(size, from_reg, mem, /* infallible store */ None) } else { unimplemented!("f32/f64 store_stack"); } diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index cf387e07c82e..3ee23a41066c 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -234,12 +234,11 @@ impl RegMem { pub(crate) fn mem(addr: impl Into) -> Self { Self::Mem { addr: addr.into() } } - /// Add the regs mentioned by `self` to `collector`. pub(crate) fn get_regs_as_uses(&self, collector: &mut RegUsageCollector) { match self { RegMem::Reg { reg } => collector.add_use(*reg), - RegMem::Mem { addr } => addr.get_regs_as_uses(collector), + RegMem::Mem { addr, .. } => addr.get_regs_as_uses(collector), } } } @@ -252,7 +251,7 @@ impl ShowWithRRU for RegMem { fn show_rru_sized(&self, mb_rru: Option<&RealRegUniverse>, size: u8) -> String { match self { RegMem::Reg { reg } => show_ireg_sized(*reg, mb_rru, size), - RegMem::Mem { addr } => addr.show_rru(mb_rru), + RegMem::Mem { addr, .. } => addr.show_rru(mb_rru), } } } diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 3fd314d38231..285319fe987c 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -807,7 +807,12 @@ pub(crate) fn emit( emit_std_reg_reg(sink, LegacyPrefix::None, 0x89, 1, *src, dst.to_reg(), rex); } - Inst::MovZX_RM_R { ext_mode, src, dst } => { + Inst::MovZX_RM_R { + ext_mode, + src, + dst, + srcloc, + } => { let (opcodes, num_opcodes, rex_flags) = match ext_mode { ExtMode::BL => { // MOVZBL is (REX.W==0) 0F B6 /r @@ -849,27 +854,45 @@ pub(crate) fn emit( *src, rex_flags, ), - RegMem::Mem { addr: src } => emit_std_reg_mem( - sink, - LegacyPrefix::None, - opcodes, - num_opcodes, - dst.to_reg(), - &src.finalize(state), - rex_flags, - ), + RegMem::Mem { addr: src } => { + let src = &src.finalize(state); + + if let Some(srcloc) = *srcloc { + // Register the offset at which the actual load instruction starts. + sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); + } + + emit_std_reg_mem( + sink, + LegacyPrefix::None, + opcodes, + num_opcodes, + dst.to_reg(), + src, + rex_flags, + ) + } } } - Inst::Mov64_M_R { src, dst } => emit_std_reg_mem( - sink, - LegacyPrefix::None, - 0x8B, - 1, - dst.to_reg(), - &src.finalize(state), - RexFlags::set_w(), - ), + Inst::Mov64_M_R { src, dst, srcloc } => { + let src = &src.finalize(state); + + if let Some(srcloc) = *srcloc { + // Register the offset at which the actual load instruction starts. + sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); + } + + emit_std_reg_mem( + sink, + LegacyPrefix::None, + 0x8B, + 1, + dst.to_reg(), + src, + RexFlags::set_w(), + ) + } Inst::LoadEffectiveAddress { addr, dst } => emit_std_reg_mem( sink, @@ -881,7 +904,12 @@ pub(crate) fn emit( RexFlags::set_w(), ), - Inst::MovSX_RM_R { ext_mode, src, dst } => { + Inst::MovSX_RM_R { + ext_mode, + src, + dst, + srcloc, + } => { let (opcodes, num_opcodes, rex_flags) = match ext_mode { ExtMode::BL => { // MOVSBL is (REX.W==0) 0F BE /r @@ -915,21 +943,41 @@ pub(crate) fn emit( *src, rex_flags, ), - RegMem::Mem { addr: src } => emit_std_reg_mem( - sink, - LegacyPrefix::None, - opcodes, - num_opcodes, - dst.to_reg(), - &src.finalize(state), - rex_flags, - ), + + RegMem::Mem { addr: src } => { + let src = &src.finalize(state); + + if let Some(srcloc) = *srcloc { + // Register the offset at which the actual load instruction starts. + sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); + } + + emit_std_reg_mem( + sink, + LegacyPrefix::None, + opcodes, + num_opcodes, + dst.to_reg(), + src, + rex_flags, + ) + } } } - Inst::Mov_R_M { size, src, dst } => { + Inst::Mov_R_M { + size, + src, + dst, + srcloc, + } => { let dst = &dst.finalize(state); + if let Some(srcloc) = *srcloc { + // Register the offset at which the actual load instruction starts. + sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); + } + match size { 1 => { // This is one of the few places where the presence of a @@ -1350,7 +1398,7 @@ pub(crate) fn emit( one_way_jmp(sink, CC::NB, *default_label); // idx unsigned >= jmp table size // Copy the index (and make sure to clear the high 32-bits lane of tmp2). - let inst = Inst::movzx_rm_r(ExtMode::LQ, RegMem::reg(*idx), *tmp2); + let inst = Inst::movzx_rm_r(ExtMode::LQ, RegMem::reg(*idx), *tmp2, None); inst.emit(sink, flags, state); // Load base address of jump table. @@ -1366,6 +1414,7 @@ pub(crate) fn emit( ExtMode::LQ, RegMem::mem(Amode::imm_reg_reg_shift(0, tmp1.to_reg(), tmp2.to_reg(), 2)), *tmp2, + None, ); inst.emit(sink, flags, state); @@ -1418,6 +1467,7 @@ pub(crate) fn emit( op, src: src_e, dst: reg_g, + srcloc, } => { let rex = RexFlags::clear_w(); let (prefix, opcode) = match op { @@ -1432,9 +1482,12 @@ pub(crate) fn emit( RegMem::Reg { reg: reg_e } => { emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); } - RegMem::Mem { addr } => { let addr = &addr.finalize(state); + if let Some(srcloc) = *srcloc { + // Register the offset at which the actual load instruction starts. + sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); + } emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex); } } @@ -1462,14 +1515,19 @@ pub(crate) fn emit( RegMem::Reg { reg: reg_e } => { emit_std_reg_reg(sink, prefix, opcode, 2, reg_g.to_reg(), *reg_e, rex); } - RegMem::Mem { addr } => { let addr = &addr.finalize(state); emit_std_reg_mem(sink, prefix, opcode, 2, reg_g.to_reg(), addr, rex); } } } - Inst::XMM_Mov_R_M { op, src, dst } => { + + Inst::XMM_Mov_R_M { + op, + src, + dst, + srcloc, + } => { let rex = RexFlags::clear_w(); let (prefix, opcode) = match op { SseOpcode::Movd => (LegacyPrefix::_66, 0x0F7E), @@ -1478,6 +1536,10 @@ pub(crate) fn emit( }; let dst = &dst.finalize(state); + if let Some(srcloc) = *srcloc { + // Register the offset at which the actual load instruction starts. + sink.add_trap(srcloc, TrapCode::HeapOutOfBounds); + } emit_std_reg_mem(sink, prefix, opcode, 2, *src, dst, rex); } diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index da1aed2442be..33fdeecd9103 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -95,82 +95,82 @@ fn test_x64_emit() { // // Addr_IR, offset zero insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rax), w_rdi, None), "488B38", "movq 0(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rbx), w_rdi, None), "488B3B", "movq 0(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rcx), w_rdi, None), "488B39", "movq 0(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rdx), w_rdi, None), "488B3A", "movq 0(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rbp), w_rdi, None), "488B7D00", "movq 0(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rsp), w_rdi, None), "488B3C24", "movq 0(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rsi), w_rdi, None), "488B3E", "movq 0(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, rdi), w_rdi, None), "488B3F", "movq 0(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r8), w_rdi, None), "498B38", "movq 0(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r9), w_rdi, None), "498B39", "movq 0(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r10), w_rdi, None), "498B3A", "movq 0(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r11), w_rdi, None), "498B3B", "movq 0(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r12), w_rdi, None), "498B3C24", "movq 0(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r13), w_rdi, None), "498B7D00", "movq 0(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r14), w_rdi, None), "498B3E", "movq 0(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0, r15), w_rdi, None), "498B3F", "movq 0(%r15), %rdi", )); @@ -178,82 +178,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset max simm8 insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rax), w_rdi, None), "488B787F", "movq 127(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rbx), w_rdi, None), "488B7B7F", "movq 127(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rcx), w_rdi, None), "488B797F", "movq 127(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rdx), w_rdi, None), "488B7A7F", "movq 127(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rbp), w_rdi, None), "488B7D7F", "movq 127(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rsp), w_rdi, None), "488B7C247F", "movq 127(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rsi), w_rdi, None), "488B7E7F", "movq 127(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, rdi), w_rdi, None), "488B7F7F", "movq 127(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r8), w_rdi, None), "498B787F", "movq 127(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r9), w_rdi, None), "498B797F", "movq 127(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r10), w_rdi, None), "498B7A7F", "movq 127(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r11), w_rdi, None), "498B7B7F", "movq 127(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r12), w_rdi, None), "498B7C247F", "movq 127(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r13), w_rdi, None), "498B7D7F", "movq 127(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r14), w_rdi, None), "498B7E7F", "movq 127(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(127, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(127, r15), w_rdi, None), "498B7F7F", "movq 127(%r15), %rdi", )); @@ -261,82 +261,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset min simm8 insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rax), w_rdi, None), "488B7880", "movq -128(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rbx), w_rdi, None), "488B7B80", "movq -128(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rcx), w_rdi, None), "488B7980", "movq -128(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rdx), w_rdi, None), "488B7A80", "movq -128(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rbp), w_rdi, None), "488B7D80", "movq -128(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rsp), w_rdi, None), "488B7C2480", "movq -128(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rsi), w_rdi, None), "488B7E80", "movq -128(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, rdi), w_rdi, None), "488B7F80", "movq -128(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r8), w_rdi, None), "498B7880", "movq -128(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r9), w_rdi, None), "498B7980", "movq -128(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r10), w_rdi, None), "498B7A80", "movq -128(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r11), w_rdi, None), "498B7B80", "movq -128(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r12), w_rdi, None), "498B7C2480", "movq -128(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r13), w_rdi, None), "498B7D80", "movq -128(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r14), w_rdi, None), "498B7E80", "movq -128(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-128i32 as u32, r15), w_rdi, None), "498B7F80", "movq -128(%r15), %rdi", )); @@ -344,82 +344,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset smallest positive simm32 insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rax), w_rdi, None), "488BB880000000", "movq 128(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rbx), w_rdi, None), "488BBB80000000", "movq 128(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rcx), w_rdi, None), "488BB980000000", "movq 128(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rdx), w_rdi, None), "488BBA80000000", "movq 128(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rbp), w_rdi, None), "488BBD80000000", "movq 128(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rsp), w_rdi, None), "488BBC2480000000", "movq 128(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rsi), w_rdi, None), "488BBE80000000", "movq 128(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, rdi), w_rdi, None), "488BBF80000000", "movq 128(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r8), w_rdi, None), "498BB880000000", "movq 128(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r9), w_rdi, None), "498BB980000000", "movq 128(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r10), w_rdi, None), "498BBA80000000", "movq 128(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r11), w_rdi, None), "498BBB80000000", "movq 128(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r12), w_rdi, None), "498BBC2480000000", "movq 128(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r13), w_rdi, None), "498BBD80000000", "movq 128(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r14), w_rdi, None), "498BBE80000000", "movq 128(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(128, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(128, r15), w_rdi, None), "498BBF80000000", "movq 128(%r15), %rdi", )); @@ -427,82 +427,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset smallest negative simm32 insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rax), w_rdi, None), "488BB87FFFFFFF", "movq -129(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rbx), w_rdi, None), "488BBB7FFFFFFF", "movq -129(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rcx), w_rdi, None), "488BB97FFFFFFF", "movq -129(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rdx), w_rdi, None), "488BBA7FFFFFFF", "movq -129(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rbp), w_rdi, None), "488BBD7FFFFFFF", "movq -129(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rsp), w_rdi, None), "488BBC247FFFFFFF", "movq -129(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rsi), w_rdi, None), "488BBE7FFFFFFF", "movq -129(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, rdi), w_rdi, None), "488BBF7FFFFFFF", "movq -129(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r8), w_rdi, None), "498BB87FFFFFFF", "movq -129(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r9), w_rdi, None), "498BB97FFFFFFF", "movq -129(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r10), w_rdi, None), "498BBA7FFFFFFF", "movq -129(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r11), w_rdi, None), "498BBB7FFFFFFF", "movq -129(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r12), w_rdi, None), "498BBC247FFFFFFF", "movq -129(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r13), w_rdi, None), "498BBD7FFFFFFF", "movq -129(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r14), w_rdi, None), "498BBE7FFFFFFF", "movq -129(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-129i32 as u32, r15), w_rdi, None), "498BBF7FFFFFFF", "movq -129(%r15), %rdi", )); @@ -510,82 +510,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset large positive simm32 insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rax), w_rdi, None), "488BB877207317", "movq 393420919(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rbx), w_rdi, None), "488BBB77207317", "movq 393420919(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rcx), w_rdi, None), "488BB977207317", "movq 393420919(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rdx), w_rdi, None), "488BBA77207317", "movq 393420919(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rbp), w_rdi, None), "488BBD77207317", "movq 393420919(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rsp), w_rdi, None), "488BBC2477207317", "movq 393420919(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rsi), w_rdi, None), "488BBE77207317", "movq 393420919(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, rdi), w_rdi, None), "488BBF77207317", "movq 393420919(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r8), w_rdi, None), "498BB877207317", "movq 393420919(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r9), w_rdi, None), "498BB977207317", "movq 393420919(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r10), w_rdi, None), "498BBA77207317", "movq 393420919(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r11), w_rdi, None), "498BBB77207317", "movq 393420919(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r12), w_rdi, None), "498BBC2477207317", "movq 393420919(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r13), w_rdi, None), "498BBD77207317", "movq 393420919(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r14), w_rdi, None), "498BBE77207317", "movq 393420919(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(0x17732077, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(0x17732077, r15), w_rdi, None), "498BBF77207317", "movq 393420919(%r15), %rdi", )); @@ -593,82 +593,82 @@ fn test_x64_emit() { // ======================================================== // Addr_IR, offset large negative simm32 insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rax), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rax), w_rdi, None), "488BB8D9A6BECE", "movq -826366247(%rax), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rbx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rbx), w_rdi, None), "488BBBD9A6BECE", "movq -826366247(%rbx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rcx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rcx), w_rdi, None), "488BB9D9A6BECE", "movq -826366247(%rcx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rdx), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rdx), w_rdi, None), "488BBAD9A6BECE", "movq -826366247(%rdx), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rbp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rbp), w_rdi, None), "488BBDD9A6BECE", "movq -826366247(%rbp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rsp), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rsp), w_rdi, None), "488BBC24D9A6BECE", "movq -826366247(%rsp), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rsi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rsi), w_rdi, None), "488BBED9A6BECE", "movq -826366247(%rsi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rdi), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, rdi), w_rdi, None), "488BBFD9A6BECE", "movq -826366247(%rdi), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r8), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r8), w_rdi, None), "498BB8D9A6BECE", "movq -826366247(%r8), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r9), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r9), w_rdi, None), "498BB9D9A6BECE", "movq -826366247(%r9), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r10), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r10), w_rdi, None), "498BBAD9A6BECE", "movq -826366247(%r10), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r11), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r11), w_rdi, None), "498BBBD9A6BECE", "movq -826366247(%r11), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r12), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r12), w_rdi, None), "498BBC24D9A6BECE", "movq -826366247(%r12), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r13), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r13), w_rdi, None), "498BBDD9A6BECE", "movq -826366247(%r13), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r14), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r14), w_rdi, None), "498BBED9A6BECE", "movq -826366247(%r14), %rdi", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r15), w_rdi), + Inst::mov64_m_r(Amode::imm_reg(-0x31415927i32 as u32, r15), w_rdi, None), "498BBFD9A6BECE", "movq -826366247(%r15), %rdi", )); @@ -680,42 +680,42 @@ fn test_x64_emit() { // // Addr_IRRS, offset max simm8 insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rax, rax, 0), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rax, rax, 0), w_r11, None), "4C8B5C007F", "movq 127(%rax,%rax,1), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rdi, rax, 1), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rdi, rax, 1), w_r11, None), "4C8B5C477F", "movq 127(%rdi,%rax,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r8, rax, 2), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r8, rax, 2), w_r11, None), "4D8B5C807F", "movq 127(%r8,%rax,4), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r15, rax, 3), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r15, rax, 3), w_r11, None), "4D8B5CC77F", "movq 127(%r15,%rax,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rax, rdi, 3), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rax, rdi, 3), w_r11, None), "4C8B5CF87F", "movq 127(%rax,%rdi,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rdi, rdi, 2), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, rdi, rdi, 2), w_r11, None), "4C8B5CBF7F", "movq 127(%rdi,%rdi,4), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r8, rdi, 1), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r8, rdi, 1), w_r11, None), "4D8B5C787F", "movq 127(%r8,%rdi,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r15, rdi, 0), w_r11), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(127, r15, rdi, 0), w_r11, None), "4D8B5C3F7F", "movq 127(%r15,%rdi,1), %r11", )); @@ -723,42 +723,74 @@ fn test_x64_emit() { // ======================================================== // Addr_IRRS, offset min simm8 insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rax, r8, 2), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(-128i32 as u32, rax, r8, 2), + w_r11, + None, + ), "4E8B5C8080", "movq -128(%rax,%r8,4), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rdi, r8, 3), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(-128i32 as u32, rdi, r8, 3), + w_r11, + None, + ), "4E8B5CC780", "movq -128(%rdi,%r8,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r8, r8, 0), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(-128i32 as u32, r8, r8, 0), + w_r11, + None, + ), "4F8B5C0080", "movq -128(%r8,%r8,1), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r15, r8, 1), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(-128i32 as u32, r15, r8, 1), + w_r11, + None, + ), "4F8B5C4780", "movq -128(%r15,%r8,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rax, r15, 1), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(-128i32 as u32, rax, r15, 1), + w_r11, + None, + ), "4E8B5C7880", "movq -128(%rax,%r15,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, rdi, r15, 0), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(-128i32 as u32, rdi, r15, 0), + w_r11, + None, + ), "4E8B5C3F80", "movq -128(%rdi,%r15,1), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r8, r15, 3), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(-128i32 as u32, r8, r15, 3), + w_r11, + None, + ), "4F8B5CF880", "movq -128(%r8,%r15,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(-128i32 as u32, r15, r15, 2), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(-128i32 as u32, r15, r15, 2), + w_r11, + None, + ), "4F8B5CBF80", "movq -128(%r15,%r15,4), %r11", )); @@ -766,42 +798,74 @@ fn test_x64_emit() { // ======================================================== // Addr_IRRS, offset large positive simm32 insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rax, rax, 0), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(0x4f6625be, rax, rax, 0), + w_r11, + None, + ), "4C8B9C00BE25664F", "movq 1332094398(%rax,%rax,1), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rdi, rax, 1), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(0x4f6625be, rdi, rax, 1), + w_r11, + None, + ), "4C8B9C47BE25664F", "movq 1332094398(%rdi,%rax,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r8, rax, 2), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(0x4f6625be, r8, rax, 2), + w_r11, + None, + ), "4D8B9C80BE25664F", "movq 1332094398(%r8,%rax,4), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r15, rax, 3), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(0x4f6625be, r15, rax, 3), + w_r11, + None, + ), "4D8B9CC7BE25664F", "movq 1332094398(%r15,%rax,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rax, rdi, 3), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(0x4f6625be, rax, rdi, 3), + w_r11, + None, + ), "4C8B9CF8BE25664F", "movq 1332094398(%rax,%rdi,8), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, rdi, rdi, 2), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(0x4f6625be, rdi, rdi, 2), + w_r11, + None, + ), "4C8B9CBFBE25664F", "movq 1332094398(%rdi,%rdi,4), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r8, rdi, 1), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(0x4f6625be, r8, rdi, 1), + w_r11, + None, + ), "4D8B9C78BE25664F", "movq 1332094398(%r8,%rdi,2), %r11", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(0x4f6625be, r15, rdi, 0), w_r11), + Inst::mov64_m_r( + Amode::imm_reg_reg_shift(0x4f6625be, r15, rdi, 0), + w_r11, + None, + ), "4D8B9C3FBE25664F", "movq 1332094398(%r15,%rdi,1), %r11", )); @@ -812,6 +876,7 @@ fn test_x64_emit() { Inst::mov64_m_r( Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rax, r8, 2), w_r11, + None, ), "4E8B9C8070E9B2D9", "movq -642586256(%rax,%r8,4), %r11", @@ -820,6 +885,7 @@ fn test_x64_emit() { Inst::mov64_m_r( Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rdi, r8, 3), w_r11, + None, ), "4E8B9CC770E9B2D9", "movq -642586256(%rdi,%r8,8), %r11", @@ -828,6 +894,7 @@ fn test_x64_emit() { Inst::mov64_m_r( Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r8, r8, 0), w_r11, + None, ), "4F8B9C0070E9B2D9", "movq -642586256(%r8,%r8,1), %r11", @@ -836,6 +903,7 @@ fn test_x64_emit() { Inst::mov64_m_r( Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r15, r8, 1), w_r11, + None, ), "4F8B9C4770E9B2D9", "movq -642586256(%r15,%r8,2), %r11", @@ -844,6 +912,7 @@ fn test_x64_emit() { Inst::mov64_m_r( Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rax, r15, 1), w_r11, + None, ), "4E8B9C7870E9B2D9", "movq -642586256(%rax,%r15,2), %r11", @@ -852,6 +921,7 @@ fn test_x64_emit() { Inst::mov64_m_r( Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, rdi, r15, 0), w_r11, + None, ), "4E8B9C3F70E9B2D9", "movq -642586256(%rdi,%r15,1), %r11", @@ -860,6 +930,7 @@ fn test_x64_emit() { Inst::mov64_m_r( Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r8, r15, 3), w_r11, + None, ), "4F8B9CF870E9B2D9", "movq -642586256(%r8,%r15,8), %r11", @@ -868,6 +939,7 @@ fn test_x64_emit() { Inst::mov64_m_r( Amode::imm_reg_reg_shift(-0x264d1690i32 as u32, r15, r15, 2), w_r11, + None, ), "4F8B9CBF70E9B2D9", "movq -642586256(%r15,%r15,4), %r11", @@ -1330,12 +1402,12 @@ fn test_x64_emit() { // ======================================================== // MovZX_RM_R insns.push(( - Inst::movzx_rm_r(ExtMode::BL, RegMem::reg(rax), w_rsi), + Inst::movzx_rm_r(ExtMode::BL, RegMem::reg(rax), w_rsi, None), "0FB6F0", "movzbl %al, %esi", )); insns.push(( - Inst::movzx_rm_r(ExtMode::BL, RegMem::reg(r15), w_rsi), + Inst::movzx_rm_r(ExtMode::BL, RegMem::reg(r15), w_rsi, None), "410FB6F7", "movzbl %r15b, %esi", )); @@ -1344,6 +1416,7 @@ fn test_x64_emit() { ExtMode::BL, RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), w_rsi, + None, ), "0FB671F9", "movzbl -7(%rcx), %esi", @@ -1353,6 +1426,7 @@ fn test_x64_emit() { ExtMode::BL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), w_rbx, + None, ), "410FB658F9", "movzbl -7(%r8), %ebx", @@ -1362,6 +1436,7 @@ fn test_x64_emit() { ExtMode::BL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), w_r9, + None, ), "450FB64AF9", "movzbl -7(%r10), %r9d", @@ -1371,17 +1446,18 @@ fn test_x64_emit() { ExtMode::BL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), w_rdx, + None, ), "410FB653F9", "movzbl -7(%r11), %edx", )); insns.push(( - Inst::movzx_rm_r(ExtMode::BQ, RegMem::reg(rax), w_rsi), + Inst::movzx_rm_r(ExtMode::BQ, RegMem::reg(rax), w_rsi, None), "480FB6F0", "movzbq %al, %rsi", )); insns.push(( - Inst::movzx_rm_r(ExtMode::BQ, RegMem::reg(r10), w_rsi), + Inst::movzx_rm_r(ExtMode::BQ, RegMem::reg(r10), w_rsi, None), "490FB6F2", "movzbq %r10b, %rsi", )); @@ -1390,6 +1466,7 @@ fn test_x64_emit() { ExtMode::BQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), w_rsi, + None, ), "480FB671F9", "movzbq -7(%rcx), %rsi", @@ -1399,6 +1476,7 @@ fn test_x64_emit() { ExtMode::BQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), w_rbx, + None, ), "490FB658F9", "movzbq -7(%r8), %rbx", @@ -1408,6 +1486,7 @@ fn test_x64_emit() { ExtMode::BQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), w_r9, + None, ), "4D0FB64AF9", "movzbq -7(%r10), %r9", @@ -1417,17 +1496,18 @@ fn test_x64_emit() { ExtMode::BQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), w_rdx, + None, ), "490FB653F9", "movzbq -7(%r11), %rdx", )); insns.push(( - Inst::movzx_rm_r(ExtMode::WL, RegMem::reg(rcx), w_rsi), + Inst::movzx_rm_r(ExtMode::WL, RegMem::reg(rcx), w_rsi, None), "0FB7F1", "movzwl %cx, %esi", )); insns.push(( - Inst::movzx_rm_r(ExtMode::WL, RegMem::reg(r10), w_rsi), + Inst::movzx_rm_r(ExtMode::WL, RegMem::reg(r10), w_rsi, None), "410FB7F2", "movzwl %r10w, %esi", )); @@ -1436,6 +1516,7 @@ fn test_x64_emit() { ExtMode::WL, RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), w_rsi, + None, ), "0FB771F9", "movzwl -7(%rcx), %esi", @@ -1445,6 +1526,7 @@ fn test_x64_emit() { ExtMode::WL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), w_rbx, + None, ), "410FB758F9", "movzwl -7(%r8), %ebx", @@ -1454,6 +1536,7 @@ fn test_x64_emit() { ExtMode::WL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), w_r9, + None, ), "450FB74AF9", "movzwl -7(%r10), %r9d", @@ -1463,17 +1546,18 @@ fn test_x64_emit() { ExtMode::WL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), w_rdx, + None, ), "410FB753F9", "movzwl -7(%r11), %edx", )); insns.push(( - Inst::movzx_rm_r(ExtMode::WQ, RegMem::reg(rcx), w_rsi), + Inst::movzx_rm_r(ExtMode::WQ, RegMem::reg(rcx), w_rsi, None), "480FB7F1", "movzwq %cx, %rsi", )); insns.push(( - Inst::movzx_rm_r(ExtMode::WQ, RegMem::reg(r11), w_rsi), + Inst::movzx_rm_r(ExtMode::WQ, RegMem::reg(r11), w_rsi, None), "490FB7F3", "movzwq %r11w, %rsi", )); @@ -1482,6 +1566,7 @@ fn test_x64_emit() { ExtMode::WQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), w_rsi, + None, ), "480FB771F9", "movzwq -7(%rcx), %rsi", @@ -1491,6 +1576,7 @@ fn test_x64_emit() { ExtMode::WQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), w_rbx, + None, ), "490FB758F9", "movzwq -7(%r8), %rbx", @@ -1500,6 +1586,7 @@ fn test_x64_emit() { ExtMode::WQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), w_r9, + None, ), "4D0FB74AF9", "movzwq -7(%r10), %r9", @@ -1509,12 +1596,13 @@ fn test_x64_emit() { ExtMode::WQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), w_rdx, + None, ), "490FB753F9", "movzwq -7(%r11), %rdx", )); insns.push(( - Inst::movzx_rm_r(ExtMode::LQ, RegMem::reg(rcx), w_rsi), + Inst::movzx_rm_r(ExtMode::LQ, RegMem::reg(rcx), w_rsi, None), "8BF1", "movl %ecx, %esi", )); @@ -1523,6 +1611,7 @@ fn test_x64_emit() { ExtMode::LQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), w_rsi, + None, ), "8B71F9", "movl -7(%rcx), %esi", @@ -1532,6 +1621,7 @@ fn test_x64_emit() { ExtMode::LQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), w_rbx, + None, ), "418B58F9", "movl -7(%r8), %ebx", @@ -1541,6 +1631,7 @@ fn test_x64_emit() { ExtMode::LQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), w_r9, + None, ), "458B4AF9", "movl -7(%r10), %r9d", @@ -1550,6 +1641,7 @@ fn test_x64_emit() { ExtMode::LQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), w_rdx, + None, ), "418B53F9", "movl -7(%r11), %edx", @@ -1558,42 +1650,42 @@ fn test_x64_emit() { // ======================================================== // Mov64_M_R insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, rbx, 0), w_rcx), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, rbx, 0), w_rcx, None), "488B8C18B3000000", "movq 179(%rax,%rbx,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, rbx, 0), w_r8), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, rbx, 0), w_r8, None), "4C8B8418B3000000", "movq 179(%rax,%rbx,1), %r8", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, r9, 0), w_rcx), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, r9, 0), w_rcx, None), "4A8B8C08B3000000", "movq 179(%rax,%r9,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, r9, 0), w_r8), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, rax, r9, 0), w_r8, None), "4E8B8408B3000000", "movq 179(%rax,%r9,1), %r8", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, rbx, 0), w_rcx), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, rbx, 0), w_rcx, None), "498B8C1AB3000000", "movq 179(%r10,%rbx,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, rbx, 0), w_r8), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, rbx, 0), w_r8, None), "4D8B841AB3000000", "movq 179(%r10,%rbx,1), %r8", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, r9, 0), w_rcx), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, r9, 0), w_rcx, None), "4B8B8C0AB3000000", "movq 179(%r10,%r9,1), %rcx", )); insns.push(( - Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, r9, 0), w_r8), + Inst::mov64_m_r(Amode::imm_reg_reg_shift(179, r10, r9, 0), w_r8, None), "4F8B840AB3000000", "movq 179(%r10,%r9,1), %r8", )); @@ -1632,12 +1724,12 @@ fn test_x64_emit() { // ======================================================== // MovSX_RM_R insns.push(( - Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(rcx), w_rsi), + Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(rcx), w_rsi, None), "0FBEF1", "movsbl %cl, %esi", )); insns.push(( - Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(r14), w_rsi), + Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(r14), w_rsi, None), "410FBEF6", "movsbl %r14b, %esi", )); @@ -1646,6 +1738,7 @@ fn test_x64_emit() { ExtMode::BL, RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), w_rsi, + None, ), "0FBE71F9", "movsbl -7(%rcx), %esi", @@ -1655,6 +1748,7 @@ fn test_x64_emit() { ExtMode::BL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), w_rbx, + None, ), "410FBE58F9", "movsbl -7(%r8), %ebx", @@ -1664,6 +1758,7 @@ fn test_x64_emit() { ExtMode::BL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), w_r9, + None, ), "450FBE4AF9", "movsbl -7(%r10), %r9d", @@ -1673,17 +1768,18 @@ fn test_x64_emit() { ExtMode::BL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), w_rdx, + None, ), "410FBE53F9", "movsbl -7(%r11), %edx", )); insns.push(( - Inst::movsx_rm_r(ExtMode::BQ, RegMem::reg(rcx), w_rsi), + Inst::movsx_rm_r(ExtMode::BQ, RegMem::reg(rcx), w_rsi, None), "480FBEF1", "movsbq %cl, %rsi", )); insns.push(( - Inst::movsx_rm_r(ExtMode::BQ, RegMem::reg(r15), w_rsi), + Inst::movsx_rm_r(ExtMode::BQ, RegMem::reg(r15), w_rsi, None), "490FBEF7", "movsbq %r15b, %rsi", )); @@ -1692,6 +1788,7 @@ fn test_x64_emit() { ExtMode::BQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), w_rsi, + None, ), "480FBE71F9", "movsbq -7(%rcx), %rsi", @@ -1701,6 +1798,7 @@ fn test_x64_emit() { ExtMode::BQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), w_rbx, + None, ), "490FBE58F9", "movsbq -7(%r8), %rbx", @@ -1710,6 +1808,7 @@ fn test_x64_emit() { ExtMode::BQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), w_r9, + None, ), "4D0FBE4AF9", "movsbq -7(%r10), %r9", @@ -1719,17 +1818,18 @@ fn test_x64_emit() { ExtMode::BQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), w_rdx, + None, ), "490FBE53F9", "movsbq -7(%r11), %rdx", )); insns.push(( - Inst::movsx_rm_r(ExtMode::WL, RegMem::reg(rcx), w_rsi), + Inst::movsx_rm_r(ExtMode::WL, RegMem::reg(rcx), w_rsi, None), "0FBFF1", "movswl %cx, %esi", )); insns.push(( - Inst::movsx_rm_r(ExtMode::WL, RegMem::reg(r14), w_rsi), + Inst::movsx_rm_r(ExtMode::WL, RegMem::reg(r14), w_rsi, None), "410FBFF6", "movswl %r14w, %esi", )); @@ -1738,6 +1838,7 @@ fn test_x64_emit() { ExtMode::WL, RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), w_rsi, + None, ), "0FBF71F9", "movswl -7(%rcx), %esi", @@ -1747,6 +1848,7 @@ fn test_x64_emit() { ExtMode::WL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), w_rbx, + None, ), "410FBF58F9", "movswl -7(%r8), %ebx", @@ -1756,6 +1858,7 @@ fn test_x64_emit() { ExtMode::WL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), w_r9, + None, ), "450FBF4AF9", "movswl -7(%r10), %r9d", @@ -1765,17 +1868,18 @@ fn test_x64_emit() { ExtMode::WL, RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), w_rdx, + None, ), "410FBF53F9", "movswl -7(%r11), %edx", )); insns.push(( - Inst::movsx_rm_r(ExtMode::WQ, RegMem::reg(rcx), w_rsi), + Inst::movsx_rm_r(ExtMode::WQ, RegMem::reg(rcx), w_rsi, None), "480FBFF1", "movswq %cx, %rsi", )); insns.push(( - Inst::movsx_rm_r(ExtMode::WQ, RegMem::reg(r13), w_rsi), + Inst::movsx_rm_r(ExtMode::WQ, RegMem::reg(r13), w_rsi, None), "490FBFF5", "movswq %r13w, %rsi", )); @@ -1784,6 +1888,7 @@ fn test_x64_emit() { ExtMode::WQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), w_rsi, + None, ), "480FBF71F9", "movswq -7(%rcx), %rsi", @@ -1793,6 +1898,7 @@ fn test_x64_emit() { ExtMode::WQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), w_rbx, + None, ), "490FBF58F9", "movswq -7(%r8), %rbx", @@ -1802,6 +1908,7 @@ fn test_x64_emit() { ExtMode::WQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), w_r9, + None, ), "4D0FBF4AF9", "movswq -7(%r10), %r9", @@ -1811,17 +1918,18 @@ fn test_x64_emit() { ExtMode::WQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), w_rdx, + None, ), "490FBF53F9", "movswq -7(%r11), %rdx", )); insns.push(( - Inst::movsx_rm_r(ExtMode::LQ, RegMem::reg(rcx), w_rsi), + Inst::movsx_rm_r(ExtMode::LQ, RegMem::reg(rcx), w_rsi, None), "4863F1", "movslq %ecx, %rsi", )); insns.push(( - Inst::movsx_rm_r(ExtMode::LQ, RegMem::reg(r15), w_rsi), + Inst::movsx_rm_r(ExtMode::LQ, RegMem::reg(r15), w_rsi, None), "4963F7", "movslq %r15d, %rsi", )); @@ -1830,6 +1938,7 @@ fn test_x64_emit() { ExtMode::LQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, rcx)), w_rsi, + None, ), "486371F9", "movslq -7(%rcx), %rsi", @@ -1839,6 +1948,7 @@ fn test_x64_emit() { ExtMode::LQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r8)), w_rbx, + None, ), "496358F9", "movslq -7(%r8), %rbx", @@ -1848,6 +1958,7 @@ fn test_x64_emit() { ExtMode::LQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r10)), w_r9, + None, ), "4D634AF9", "movslq -7(%r10), %r9", @@ -1857,6 +1968,7 @@ fn test_x64_emit() { ExtMode::LQ, RegMem::mem(Amode::imm_reg(-7i32 as u32, r11)), w_rdx, + None, ), "496353F9", "movslq -7(%r11), %rdx", @@ -1865,325 +1977,325 @@ fn test_x64_emit() { // ======================================================== // Mov_R_M. Byte stores are tricky. Check everything carefully. insns.push(( - Inst::mov_r_m(8, rax, Amode::imm_reg(99, rdi)), + Inst::mov_r_m(8, rax, Amode::imm_reg(99, rdi), None), "48894763", "movq %rax, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(8, rbx, Amode::imm_reg(99, r8)), + Inst::mov_r_m(8, rbx, Amode::imm_reg(99, r8), None), "49895863", "movq %rbx, 99(%r8)", )); insns.push(( - Inst::mov_r_m(8, rcx, Amode::imm_reg(99, rsi)), + Inst::mov_r_m(8, rcx, Amode::imm_reg(99, rsi), None), "48894E63", "movq %rcx, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(8, rdx, Amode::imm_reg(99, r9)), + Inst::mov_r_m(8, rdx, Amode::imm_reg(99, r9), None), "49895163", "movq %rdx, 99(%r9)", )); insns.push(( - Inst::mov_r_m(8, rsi, Amode::imm_reg(99, rax)), + Inst::mov_r_m(8, rsi, Amode::imm_reg(99, rax), None), "48897063", "movq %rsi, 99(%rax)", )); insns.push(( - Inst::mov_r_m(8, rdi, Amode::imm_reg(99, r15)), + Inst::mov_r_m(8, rdi, Amode::imm_reg(99, r15), None), "49897F63", "movq %rdi, 99(%r15)", )); insns.push(( - Inst::mov_r_m(8, rsp, Amode::imm_reg(99, rcx)), + Inst::mov_r_m(8, rsp, Amode::imm_reg(99, rcx), None), "48896163", "movq %rsp, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(8, rbp, Amode::imm_reg(99, r14)), + Inst::mov_r_m(8, rbp, Amode::imm_reg(99, r14), None), "49896E63", "movq %rbp, 99(%r14)", )); insns.push(( - Inst::mov_r_m(8, r8, Amode::imm_reg(99, rdi)), + Inst::mov_r_m(8, r8, Amode::imm_reg(99, rdi), None), "4C894763", "movq %r8, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(8, r9, Amode::imm_reg(99, r8)), + Inst::mov_r_m(8, r9, Amode::imm_reg(99, r8), None), "4D894863", "movq %r9, 99(%r8)", )); insns.push(( - Inst::mov_r_m(8, r10, Amode::imm_reg(99, rsi)), + Inst::mov_r_m(8, r10, Amode::imm_reg(99, rsi), None), "4C895663", "movq %r10, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(8, r11, Amode::imm_reg(99, r9)), + Inst::mov_r_m(8, r11, Amode::imm_reg(99, r9), None), "4D895963", "movq %r11, 99(%r9)", )); insns.push(( - Inst::mov_r_m(8, r12, Amode::imm_reg(99, rax)), + Inst::mov_r_m(8, r12, Amode::imm_reg(99, rax), None), "4C896063", "movq %r12, 99(%rax)", )); insns.push(( - Inst::mov_r_m(8, r13, Amode::imm_reg(99, r15)), + Inst::mov_r_m(8, r13, Amode::imm_reg(99, r15), None), "4D896F63", "movq %r13, 99(%r15)", )); insns.push(( - Inst::mov_r_m(8, r14, Amode::imm_reg(99, rcx)), + Inst::mov_r_m(8, r14, Amode::imm_reg(99, rcx), None), "4C897163", "movq %r14, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(8, r15, Amode::imm_reg(99, r14)), + Inst::mov_r_m(8, r15, Amode::imm_reg(99, r14), None), "4D897E63", "movq %r15, 99(%r14)", )); // insns.push(( - Inst::mov_r_m(4, rax, Amode::imm_reg(99, rdi)), + Inst::mov_r_m(4, rax, Amode::imm_reg(99, rdi), None), "894763", "movl %eax, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(4, rbx, Amode::imm_reg(99, r8)), + Inst::mov_r_m(4, rbx, Amode::imm_reg(99, r8), None), "41895863", "movl %ebx, 99(%r8)", )); insns.push(( - Inst::mov_r_m(4, rcx, Amode::imm_reg(99, rsi)), + Inst::mov_r_m(4, rcx, Amode::imm_reg(99, rsi), None), "894E63", "movl %ecx, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(4, rdx, Amode::imm_reg(99, r9)), + Inst::mov_r_m(4, rdx, Amode::imm_reg(99, r9), None), "41895163", "movl %edx, 99(%r9)", )); insns.push(( - Inst::mov_r_m(4, rsi, Amode::imm_reg(99, rax)), + Inst::mov_r_m(4, rsi, Amode::imm_reg(99, rax), None), "897063", "movl %esi, 99(%rax)", )); insns.push(( - Inst::mov_r_m(4, rdi, Amode::imm_reg(99, r15)), + Inst::mov_r_m(4, rdi, Amode::imm_reg(99, r15), None), "41897F63", "movl %edi, 99(%r15)", )); insns.push(( - Inst::mov_r_m(4, rsp, Amode::imm_reg(99, rcx)), + Inst::mov_r_m(4, rsp, Amode::imm_reg(99, rcx), None), "896163", "movl %esp, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(4, rbp, Amode::imm_reg(99, r14)), + Inst::mov_r_m(4, rbp, Amode::imm_reg(99, r14), None), "41896E63", "movl %ebp, 99(%r14)", )); insns.push(( - Inst::mov_r_m(4, r8, Amode::imm_reg(99, rdi)), + Inst::mov_r_m(4, r8, Amode::imm_reg(99, rdi), None), "44894763", "movl %r8d, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(4, r9, Amode::imm_reg(99, r8)), + Inst::mov_r_m(4, r9, Amode::imm_reg(99, r8), None), "45894863", "movl %r9d, 99(%r8)", )); insns.push(( - Inst::mov_r_m(4, r10, Amode::imm_reg(99, rsi)), + Inst::mov_r_m(4, r10, Amode::imm_reg(99, rsi), None), "44895663", "movl %r10d, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(4, r11, Amode::imm_reg(99, r9)), + Inst::mov_r_m(4, r11, Amode::imm_reg(99, r9), None), "45895963", "movl %r11d, 99(%r9)", )); insns.push(( - Inst::mov_r_m(4, r12, Amode::imm_reg(99, rax)), + Inst::mov_r_m(4, r12, Amode::imm_reg(99, rax), None), "44896063", "movl %r12d, 99(%rax)", )); insns.push(( - Inst::mov_r_m(4, r13, Amode::imm_reg(99, r15)), + Inst::mov_r_m(4, r13, Amode::imm_reg(99, r15), None), "45896F63", "movl %r13d, 99(%r15)", )); insns.push(( - Inst::mov_r_m(4, r14, Amode::imm_reg(99, rcx)), + Inst::mov_r_m(4, r14, Amode::imm_reg(99, rcx), None), "44897163", "movl %r14d, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(4, r15, Amode::imm_reg(99, r14)), + Inst::mov_r_m(4, r15, Amode::imm_reg(99, r14), None), "45897E63", "movl %r15d, 99(%r14)", )); // insns.push(( - Inst::mov_r_m(2, rax, Amode::imm_reg(99, rdi)), + Inst::mov_r_m(2, rax, Amode::imm_reg(99, rdi), None), "66894763", "movw %ax, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(2, rbx, Amode::imm_reg(99, r8)), + Inst::mov_r_m(2, rbx, Amode::imm_reg(99, r8), None), "6641895863", "movw %bx, 99(%r8)", )); insns.push(( - Inst::mov_r_m(2, rcx, Amode::imm_reg(99, rsi)), + Inst::mov_r_m(2, rcx, Amode::imm_reg(99, rsi), None), "66894E63", "movw %cx, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(2, rdx, Amode::imm_reg(99, r9)), + Inst::mov_r_m(2, rdx, Amode::imm_reg(99, r9), None), "6641895163", "movw %dx, 99(%r9)", )); insns.push(( - Inst::mov_r_m(2, rsi, Amode::imm_reg(99, rax)), + Inst::mov_r_m(2, rsi, Amode::imm_reg(99, rax), None), "66897063", "movw %si, 99(%rax)", )); insns.push(( - Inst::mov_r_m(2, rdi, Amode::imm_reg(99, r15)), + Inst::mov_r_m(2, rdi, Amode::imm_reg(99, r15), None), "6641897F63", "movw %di, 99(%r15)", )); insns.push(( - Inst::mov_r_m(2, rsp, Amode::imm_reg(99, rcx)), + Inst::mov_r_m(2, rsp, Amode::imm_reg(99, rcx), None), "66896163", "movw %sp, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(2, rbp, Amode::imm_reg(99, r14)), + Inst::mov_r_m(2, rbp, Amode::imm_reg(99, r14), None), "6641896E63", "movw %bp, 99(%r14)", )); insns.push(( - Inst::mov_r_m(2, r8, Amode::imm_reg(99, rdi)), + Inst::mov_r_m(2, r8, Amode::imm_reg(99, rdi), None), "6644894763", "movw %r8w, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(2, r9, Amode::imm_reg(99, r8)), + Inst::mov_r_m(2, r9, Amode::imm_reg(99, r8), None), "6645894863", "movw %r9w, 99(%r8)", )); insns.push(( - Inst::mov_r_m(2, r10, Amode::imm_reg(99, rsi)), + Inst::mov_r_m(2, r10, Amode::imm_reg(99, rsi), None), "6644895663", "movw %r10w, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(2, r11, Amode::imm_reg(99, r9)), + Inst::mov_r_m(2, r11, Amode::imm_reg(99, r9), None), "6645895963", "movw %r11w, 99(%r9)", )); insns.push(( - Inst::mov_r_m(2, r12, Amode::imm_reg(99, rax)), + Inst::mov_r_m(2, r12, Amode::imm_reg(99, rax), None), "6644896063", "movw %r12w, 99(%rax)", )); insns.push(( - Inst::mov_r_m(2, r13, Amode::imm_reg(99, r15)), + Inst::mov_r_m(2, r13, Amode::imm_reg(99, r15), None), "6645896F63", "movw %r13w, 99(%r15)", )); insns.push(( - Inst::mov_r_m(2, r14, Amode::imm_reg(99, rcx)), + Inst::mov_r_m(2, r14, Amode::imm_reg(99, rcx), None), "6644897163", "movw %r14w, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(2, r15, Amode::imm_reg(99, r14)), + Inst::mov_r_m(2, r15, Amode::imm_reg(99, r14), None), "6645897E63", "movw %r15w, 99(%r14)", )); // insns.push(( - Inst::mov_r_m(1, rax, Amode::imm_reg(99, rdi)), + Inst::mov_r_m(1, rax, Amode::imm_reg(99, rdi), None), "884763", "movb %al, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(1, rbx, Amode::imm_reg(99, r8)), + Inst::mov_r_m(1, rbx, Amode::imm_reg(99, r8), None), "41885863", "movb %bl, 99(%r8)", )); insns.push(( - Inst::mov_r_m(1, rcx, Amode::imm_reg(99, rsi)), + Inst::mov_r_m(1, rcx, Amode::imm_reg(99, rsi), None), "884E63", "movb %cl, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(1, rdx, Amode::imm_reg(99, r9)), + Inst::mov_r_m(1, rdx, Amode::imm_reg(99, r9), None), "41885163", "movb %dl, 99(%r9)", )); insns.push(( - Inst::mov_r_m(1, rsi, Amode::imm_reg(99, rax)), + Inst::mov_r_m(1, rsi, Amode::imm_reg(99, rax), None), "40887063", "movb %sil, 99(%rax)", )); insns.push(( - Inst::mov_r_m(1, rdi, Amode::imm_reg(99, r15)), + Inst::mov_r_m(1, rdi, Amode::imm_reg(99, r15), None), "41887F63", "movb %dil, 99(%r15)", )); insns.push(( - Inst::mov_r_m(1, rsp, Amode::imm_reg(99, rcx)), + Inst::mov_r_m(1, rsp, Amode::imm_reg(99, rcx), None), "40886163", "movb %spl, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(1, rbp, Amode::imm_reg(99, r14)), + Inst::mov_r_m(1, rbp, Amode::imm_reg(99, r14), None), "41886E63", "movb %bpl, 99(%r14)", )); insns.push(( - Inst::mov_r_m(1, r8, Amode::imm_reg(99, rdi)), + Inst::mov_r_m(1, r8, Amode::imm_reg(99, rdi), None), "44884763", "movb %r8b, 99(%rdi)", )); insns.push(( - Inst::mov_r_m(1, r9, Amode::imm_reg(99, r8)), + Inst::mov_r_m(1, r9, Amode::imm_reg(99, r8), None), "45884863", "movb %r9b, 99(%r8)", )); insns.push(( - Inst::mov_r_m(1, r10, Amode::imm_reg(99, rsi)), + Inst::mov_r_m(1, r10, Amode::imm_reg(99, rsi), None), "44885663", "movb %r10b, 99(%rsi)", )); insns.push(( - Inst::mov_r_m(1, r11, Amode::imm_reg(99, r9)), + Inst::mov_r_m(1, r11, Amode::imm_reg(99, r9), None), "45885963", "movb %r11b, 99(%r9)", )); insns.push(( - Inst::mov_r_m(1, r12, Amode::imm_reg(99, rax)), + Inst::mov_r_m(1, r12, Amode::imm_reg(99, rax), None), "44886063", "movb %r12b, 99(%rax)", )); insns.push(( - Inst::mov_r_m(1, r13, Amode::imm_reg(99, r15)), + Inst::mov_r_m(1, r13, Amode::imm_reg(99, r15), None), "45886F63", "movb %r13b, 99(%r15)", )); insns.push(( - Inst::mov_r_m(1, r14, Amode::imm_reg(99, rcx)), + Inst::mov_r_m(1, r14, Amode::imm_reg(99, rcx), None), "44887163", "movb %r14b, 99(%rcx)", )); insns.push(( - Inst::mov_r_m(1, r15, Amode::imm_reg(99, r14)), + Inst::mov_r_m(1, r15, Amode::imm_reg(99, r14), None), "45887E63", "movb %r15b, 99(%r14)", )); @@ -2831,12 +2943,12 @@ fn test_x64_emit() { "andnps %xmm4, %xmm11", )); insns.push(( - Inst::xmm_mov_rm_r(SseOpcode::Movaps, RegMem::reg(xmm5), w_xmm14), + Inst::xmm_mov_rm_r(SseOpcode::Movaps, RegMem::reg(xmm5), w_xmm14, None), "440F28F5", "movaps %xmm5, %xmm14", )); insns.push(( - Inst::xmm_mov_rm_r(SseOpcode::Movd, RegMem::reg(rax), w_xmm15), + Inst::xmm_mov_rm_r(SseOpcode::Movd, RegMem::reg(rax), w_xmm15, None), "66440F6EF8", "movd %eax, %xmm15", )); @@ -2847,19 +2959,24 @@ fn test_x64_emit() { )); insns.push(( - Inst::xmm_mov_r_m(SseOpcode::Movd, xmm0, Amode::imm_reg(321, rbx)), + Inst::xmm_mov_r_m(SseOpcode::Movd, xmm0, Amode::imm_reg(321, rbx), None), "660F7E8341010000", "movd %xmm0, 321(%rbx)", )); insns.push(( - Inst::xmm_mov_r_m(SseOpcode::Movss, xmm15, Amode::imm_reg(128, r12)), + Inst::xmm_mov_r_m(SseOpcode::Movss, xmm15, Amode::imm_reg(128, r12), None), "F3450F11BC2480000000", "movss %xmm15, 128(%r12)", )); insns.push(( - Inst::xmm_mov_rm_r(SseOpcode::Movd, RegMem::mem(Amode::imm_reg(2, r10)), w_xmm9), + Inst::xmm_mov_rm_r( + SseOpcode::Movd, + RegMem::mem(Amode::imm_reg(2, r10)), + w_xmm9, + None, + ), "66450F6E4A02", "movd 2(%r10), %xmm9", )); @@ -2870,12 +2987,12 @@ fn test_x64_emit() { "orps %xmm5, %xmm4", )); insns.push(( - Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(xmm13), w_xmm2), + Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(xmm13), w_xmm2, None), "F3410F10D5", "movss %xmm13, %xmm2", )); insns.push(( - Inst::xmm_mov_rm_r(SseOpcode::Movsd, RegMem::reg(xmm14), w_xmm3), + Inst::xmm_mov_rm_r(SseOpcode::Movsd, RegMem::reg(xmm14), w_xmm3, None), "F2410F10DE", "movsd %xmm14, %xmm3", )); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index e3d4490b5237..a4b9e04e4fa1 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -14,7 +14,7 @@ use regalloc::{RealRegUniverse, Reg, RegClass, RegUsageMapper, SpillSlot, Virtua use smallvec::SmallVec; use crate::binemit::CodeOffset; -use crate::ir::types::{B1, B128, B16, B32, B64, B8, F32, F64, I128, I16, I32, I64, I8}; +use crate::ir::types::*; use crate::ir::{ExternalName, Opcode, SourceLoc, TrapCode, Type}; use crate::machinst::*; use crate::settings::Flags; @@ -114,12 +114,16 @@ pub enum Inst { ext_mode: ExtMode, src: RegMem, dst: Writable, + /// Source location, if the memory access can be out-of-bounds. + srcloc: Option, }, /// A plain 64-bit integer load, since MovZX_RM_R can't represent that. Mov64_M_R { src: SyntheticAmode, dst: Writable, + /// Source location, if the memory access can be out-of-bounds. + srcloc: Option, }, /// Loads the memory address of addr into dst. @@ -133,6 +137,8 @@ pub enum Inst { ext_mode: ExtMode, src: RegMem, dst: Writable, + /// Source location, if the memory access can be out-of-bounds. + srcloc: Option, }, /// Integer stores: mov (b w l q) reg addr. @@ -140,6 +146,8 @@ pub enum Inst { size: u8, // 1, 2, 4 or 8. src: Reg, dst: SyntheticAmode, + /// Source location, if the memory access can be out-of-bounds. + srcloc: Option, }, /// Arithmetic shifts: (shl shr sar) (l q) imm reg. @@ -196,6 +204,8 @@ pub enum Inst { op: SseOpcode, src: RegMem, dst: Writable, + /// Source location, if the memory access can be out-of-bounds. + srcloc: Option, }, /// mov reg addr (good for all memory stores from xmm registers) @@ -203,6 +213,8 @@ pub enum Inst { op: SseOpcode, src: Reg, dst: SyntheticAmode, + /// Source location, if the memory access can be out-of-bounds. + srcloc: Option, }, // ===================================== @@ -367,9 +379,19 @@ impl Inst { Inst::Mov_R_R { is_64, src, dst } } - pub(crate) fn xmm_mov_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Inst { + pub(crate) fn xmm_mov_rm_r( + op: SseOpcode, + src: RegMem, + dst: Writable, + srcloc: Option, + ) -> Inst { debug_assert!(dst.to_reg().get_class() == RegClass::V128); - Inst::XMM_Mov_RM_R { op, src, dst } + Inst::XMM_Mov_RM_R { + op, + src, + dst, + srcloc, + } } pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable) -> Self { @@ -377,38 +399,69 @@ impl Inst { Inst::XMM_RM_R { op, src, dst } } - pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into) -> Inst { + pub(crate) fn xmm_mov_r_m( + op: SseOpcode, + src: Reg, + dst: impl Into, + srcloc: Option, + ) -> Inst { debug_assert!(src.get_class() == RegClass::V128); Inst::XMM_Mov_R_M { op, src, dst: dst.into(), + srcloc, } } - pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable) -> Inst { + pub(crate) fn movzx_rm_r( + ext_mode: ExtMode, + src: RegMem, + dst: Writable, + srcloc: Option, + ) -> Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); - Inst::MovZX_RM_R { ext_mode, src, dst } + Inst::MovZX_RM_R { + ext_mode, + src, + dst, + srcloc, + } } - pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable) -> Inst { + pub(crate) fn movsx_rm_r( + ext_mode: ExtMode, + src: RegMem, + dst: Writable, + srcloc: Option, + ) -> Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); - Inst::MovSX_RM_R { ext_mode, src, dst } + Inst::MovSX_RM_R { + ext_mode, + src, + dst, + srcloc, + } } - pub(crate) fn mov64_m_r(src: impl Into, dst: Writable) -> Inst { + pub(crate) fn mov64_m_r( + src: impl Into, + dst: Writable, + srcloc: Option, + ) -> Inst { debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::Mov64_M_R { src: src.into(), dst, + srcloc, } } /// A convenience function to be able to use a RegMem as the source of a move. - pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable) -> Inst { + pub(crate) fn mov64_rm_r(src: RegMem, dst: Writable, srcloc: Option) -> Inst { match src { RegMem::Reg { reg } => Self::mov_r_r(true, reg, dst), - RegMem::Mem { addr } => Self::mov64_m_r(addr, dst), + RegMem::Mem { addr } => Self::mov64_m_r(addr, dst, srcloc), } } @@ -416,6 +469,7 @@ impl Inst { size: u8, // 1, 2, 4 or 8 src: Reg, dst: impl Into, + srcloc: Option, ) -> Inst { debug_assert!(size == 8 || size == 4 || size == 2 || size == 1); debug_assert!(src.get_class() == RegClass::I64); @@ -423,6 +477,7 @@ impl Inst { size, src, dst: dst.into(), + srcloc, } } @@ -663,13 +718,13 @@ impl ShowWithRRU for Inst { _ => unreachable!(), } .into(), - Inst::XMM_Mov_RM_R { op, src, dst } => format!( + Inst::XMM_Mov_RM_R { op, src, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), src.show_rru_sized(mb_rru, op.src_size()), show_ireg_sized(dst.to_reg(), mb_rru, 8), ), - Inst::XMM_Mov_R_M { op, src, dst } => format!( + Inst::XMM_Mov_R_M { op, src, dst, .. } => format!( "{} {}, {}", ljustify(op.to_string()), show_ireg_sized(*src, mb_rru, 8), @@ -708,7 +763,9 @@ impl ShowWithRRU for Inst { show_ireg_sized(*src, mb_rru, sizeLQ(*is_64)), show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64)) ), - Inst::MovZX_RM_R { ext_mode, src, dst } => { + Inst::MovZX_RM_R { + ext_mode, src, dst, .. + } => { if *ext_mode == ExtMode::LQ { format!( "{} {}, {}", @@ -725,7 +782,7 @@ impl ShowWithRRU for Inst { ) } } - Inst::Mov64_M_R { src, dst } => format!( + Inst::Mov64_M_R { src, dst, .. } => format!( "{} {}, {}", ljustify("movq".to_string()), src.show_rru(mb_rru), @@ -737,13 +794,15 @@ impl ShowWithRRU for Inst { addr.show_rru(mb_rru), dst.show_rru(mb_rru) ), - Inst::MovSX_RM_R { ext_mode, src, dst } => format!( + Inst::MovSX_RM_R { + ext_mode, src, dst, .. + } => format!( "{} {}, {}", ljustify2("movs".to_string(), ext_mode.to_string()), src.show_rru_sized(mb_rru, ext_mode.src_size()), show_ireg_sized(dst.to_reg(), mb_rru, ext_mode.dst_size()) ), - Inst::Mov_R_M { size, src, dst } => format!( + Inst::Mov_R_M { size, src, dst, .. } => format!( "{} {}, {}", ljustify2("mov".to_string(), suffixBWLQ(*size)), show_ireg_sized(*src, mb_rru, *size), @@ -906,7 +965,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { src.get_regs_as_uses(collector); collector.add_def(*dst); } - Inst::Mov64_M_R { src, dst } | Inst::LoadEffectiveAddress { addr: src, dst } => { + Inst::Mov64_M_R { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { src.get_regs_as_uses(collector); collector.add_def(*dst) } @@ -1051,7 +1110,7 @@ impl RegMem { fn map_uses(&mut self, map: &RUM) { match self { RegMem::Reg { ref mut reg } => map_use(map, reg), - RegMem::Mem { ref mut addr } => addr.map_uses(map), + RegMem::Mem { ref mut addr, .. } => addr.map_uses(map), } } } @@ -1123,7 +1182,7 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { src.map_uses(mapper); map_def(mapper, dst); } - Inst::Mov64_M_R { src, dst } | Inst::LoadEffectiveAddress { addr: src, dst } => { + Inst::Mov64_M_R { src, dst, .. } | Inst::LoadEffectiveAddress { addr: src, dst } => { src.map_uses(mapper); map_def(mapper, dst); } @@ -1243,8 +1302,10 @@ impl MachInst for Inst { // conceivably use `movl %reg, %reg` to zero out the top 32 bits of // %reg. match self { - Self::Mov_R_R { is_64, src, dst } if *is_64 => Some((*dst, *src)), - Self::XMM_Mov_RM_R { op, src, dst } + Self::Mov_R_R { + is_64, src, dst, .. + } if *is_64 => Some((*dst, *src)), + Self::XMM_Mov_RM_R { op, src, dst, .. } if *op == SseOpcode::Movss || *op == SseOpcode::Movsd || *op == SseOpcode::Movaps => @@ -1292,8 +1353,8 @@ impl MachInst for Inst { match rc_dst { RegClass::I64 => Inst::mov_r_r(true, src_reg, dst_reg), RegClass::V128 => match ty { - F32 => Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(src_reg), dst_reg), - F64 => Inst::xmm_mov_rm_r(SseOpcode::Movsd, RegMem::reg(src_reg), dst_reg), + F32 => Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(src_reg), dst_reg, None), + F64 => Inst::xmm_mov_rm_r(SseOpcode::Movsd, RegMem::reg(src_reg), dst_reg, None), _ => panic!("unexpected V128 type in gen_move"), }, _ => panic!("gen_move(x64): unhandled regclass"), @@ -1316,6 +1377,7 @@ impl MachInst for Inst { match ty { I8 | I16 | I32 | I64 | B1 | B8 | B16 | B32 | B64 => Ok(RegClass::I64), F32 | F64 | I128 | B128 => Ok(RegClass::V128), + IFLAGS | FFLAGS => Ok(RegClass::I64), _ => Err(CodegenError::Unsupported(format!( "Unexpected SSA-value type: {}", ty diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index bb6eadb7b2bb..0f4f78a5bbdd 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -157,10 +157,14 @@ fn extend_input_to_reg(ctx: Ctx, spec: InsnInput, ext_spec: ExtSpec) -> Reg { let dst = ctx.alloc_tmp(RegClass::I64, requested_ty); match ext_spec { ExtSpec::ZeroExtend32 | ExtSpec::ZeroExtend64 => { - ctx.emit(Inst::movzx_rm_r(ext_mode, src, dst)) + ctx.emit(Inst::movzx_rm_r( + ext_mode, src, dst, /* infallible */ None, + )) } ExtSpec::SignExtend32 | ExtSpec::SignExtend64 => { - ctx.emit(Inst::movsx_rm_r(ext_mode, src, dst)) + ctx.emit(Inst::movsx_rm_r( + ext_mode, src, dst, /* infallible */ None, + )) } } dst.to_reg() @@ -345,11 +349,7 @@ fn lower_insn_to_regs>( tmp, )); - ctx.emit(Inst::imm_r( - ty == I64, - ty.bits() as u64 - 1, - dst, - )); + ctx.emit(Inst::imm_r(ty == I64, ty.bits() as u64 - 1, dst)); ctx.emit(Inst::alu_rmi_r( ty == I64, @@ -415,7 +415,7 @@ fn lower_insn_to_regs>( let cst = ctx.alloc_tmp(RegClass::I64, I64); // mov src, tmp1 - ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1)); + ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1, None)); // shr $1, tmp1 ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); @@ -432,7 +432,7 @@ fn lower_insn_to_regs>( )); // mov src, tmp2 - ctx.emit(Inst::mov64_rm_r(src, tmp2)); + ctx.emit(Inst::mov64_rm_r(src, tmp2, None)); // sub tmp1, tmp2 ctx.emit(Inst::alu_rmi_r( @@ -481,7 +481,7 @@ fn lower_insn_to_regs>( )); // mov tmp2, dst - ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst)); + ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst, None)); // shr $4, dst ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(4), dst)); @@ -526,7 +526,7 @@ fn lower_insn_to_regs>( let tmp2 = ctx.alloc_tmp(RegClass::I64, I64); // mov src, tmp1 - ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1)); + ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1, None)); // shr $1, tmp1 ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); @@ -540,7 +540,7 @@ fn lower_insn_to_regs>( )); // mov src, tmp2 - ctx.emit(Inst::mov64_rm_r(src, tmp2)); + ctx.emit(Inst::mov64_rm_r(src, tmp2, None)); // sub tmp1, tmp2 ctx.emit(Inst::alu_rmi_r( @@ -589,7 +589,7 @@ fn lower_insn_to_regs>( )); // mov tmp2, dst - ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst)); + ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst, None)); // shr $4, dst ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(4), dst)); @@ -662,12 +662,16 @@ fn lower_insn_to_regs>( if let Some(ext_mode) = ext_mode { if op == Opcode::Sextend { - ctx.emit(Inst::movsx_rm_r(ext_mode, src, dst)); + ctx.emit(Inst::movsx_rm_r( + ext_mode, src, dst, /* infallible */ None, + )); } else { - ctx.emit(Inst::movzx_rm_r(ext_mode, src, dst)); + ctx.emit(Inst::movzx_rm_r( + ext_mode, src, dst, /* infallible */ None, + )); } } else { - ctx.emit(Inst::mov64_rm_r(src, dst)); + ctx.emit(Inst::mov64_rm_r(src, dst, /* infallible */ None)); } } @@ -752,7 +756,12 @@ fn lower_insn_to_regs>( // TODO Fmax, Fmin. _ => unimplemented!(), }; - ctx.emit(Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::reg(lhs), dst)); + ctx.emit(Inst::xmm_mov_rm_r( + SseOpcode::Movss, + RegMem::reg(lhs), + dst, + None, + )); ctx.emit(Inst::xmm_rm_r(sse_op, RegMem::reg(rhs), dst)); } else { unimplemented!("unimplemented lowering for opcode {:?}", op); @@ -779,17 +788,20 @@ fn lower_insn_to_regs>( SseOpcode::Movd, RegMem::reg(tmp_gpr1.to_reg()), tmp_xmm1, + None, )); ctx.emit(Inst::xmm_mov_rm_r( SseOpcode::Movaps, RegMem::reg(tmp_xmm1.to_reg()), dst, + None, )); ctx.emit(Inst::xmm_rm_r(SseOpcode::Andnps, RegMem::reg(lhs), dst)); ctx.emit(Inst::xmm_mov_rm_r( SseOpcode::Movss, RegMem::reg(rhs), tmp_xmm2, + None, )); ctx.emit(Inst::xmm_rm_r( SseOpcode::Andps, @@ -890,25 +902,37 @@ fn lower_insn_to_regs>( _ => unreachable!(), }; + let srcloc = Some(ctx.srcloc(insn)); + let dst = output_to_reg(ctx, outputs[0]); match (sign_extend, is_float) { (true, false) => { // The load is sign-extended only when the output size is lower than 64 bits, // so ext-mode is defined in this case. - ctx.emit(Inst::movsx_rm_r(ext_mode.unwrap(), RegMem::mem(addr), dst)); + ctx.emit(Inst::movsx_rm_r( + ext_mode.unwrap(), + RegMem::mem(addr), + dst, + srcloc, + )); } (false, false) => { if elem_ty.bytes() == 8 { // Use a plain load. - ctx.emit(Inst::mov64_m_r(addr, dst)) + ctx.emit(Inst::mov64_m_r(addr, dst, srcloc)) } else { // Use a zero-extended load. - ctx.emit(Inst::movzx_rm_r(ext_mode.unwrap(), RegMem::mem(addr), dst)) + ctx.emit(Inst::movzx_rm_r( + ext_mode.unwrap(), + RegMem::mem(addr), + dst, + srcloc, + )) } } (_, true) => { ctx.emit(match elem_ty { - F32 => Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::mem(addr), dst), + F32 => Inst::xmm_mov_rm_r(SseOpcode::Movss, RegMem::mem(addr), dst, srcloc), _ => unimplemented!("FP load not 32-bit"), }); } @@ -964,13 +988,15 @@ fn lower_insn_to_regs>( let src = input_to_reg(ctx, inputs[0]); + let srcloc = Some(ctx.srcloc(insn)); + if is_float { ctx.emit(match elem_ty { - F32 => Inst::xmm_mov_r_m(SseOpcode::Movss, src, addr), + F32 => Inst::xmm_mov_r_m(SseOpcode::Movss, src, addr, srcloc), _ => unimplemented!("FP store not 32-bit"), }); } else { - ctx.emit(Inst::mov_r_m(elem_ty.bytes() as u8, src, addr)); + ctx.emit(Inst::mov_r_m(elem_ty.bytes() as u8, src, addr, srcloc)); } } @@ -1049,8 +1075,8 @@ fn lower_insn_to_regs>( if size == 1 { // Sign-extend operands to 32, then do a cmove of size 4. let lhs_se = ctx.alloc_tmp(RegClass::I64, I32); - ctx.emit(Inst::movsx_rm_r(ExtMode::BL, lhs, lhs_se)); - ctx.emit(Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(rhs), dst)); + ctx.emit(Inst::movsx_rm_r(ExtMode::BL, lhs, lhs_se, None)); + ctx.emit(Inst::movsx_rm_r(ExtMode::BL, RegMem::reg(rhs), dst, None)); ctx.emit(Inst::cmove(4, cc, RegMem::reg(lhs_se.to_reg()), dst)); } else { ctx.emit(Inst::gen_move(dst, rhs, ty));