diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index cc8c41deab48..dadaa6e35973 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -94,10 +94,10 @@ fn in_vec_reg(ty: types::Type) -> bool { } fn get_intreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option { - assert!(match call_conv { - CallConv::SystemV | CallConv::BaldrdashSystemV => true, - _ => false, - }); + match call_conv { + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::BaldrdashSystemV => {} + _ => panic!("int args only supported for SysV calling convention"), + }; match idx { 0 => Some(regs::rdi()), 1 => Some(regs::rsi()), @@ -110,10 +110,10 @@ fn get_intreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option { } fn get_fltreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option { - assert!(match call_conv { - CallConv::SystemV | CallConv::BaldrdashSystemV => true, - _ => false, - }); + match call_conv { + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::BaldrdashSystemV => {} + _ => panic!("float args only supported for SysV calling convention"), + }; match idx { 0 => Some(regs::xmm0()), 1 => Some(regs::xmm1()), diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 3ee23a41066c..4090f2d33af7 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -282,32 +282,32 @@ impl fmt::Debug for AluRmiROpcode { } } -impl ToString for AluRmiROpcode { - fn to_string(&self) -> String { - format!("{:?}", self) +impl fmt::Display for AluRmiROpcode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self, f) } } #[derive(Clone, PartialEq)] -pub enum ReadOnlyGprRmROpcode { +pub enum UnaryRmROpcode { /// Bit-scan reverse. Bsr, /// Bit-scan forward. Bsf, } -impl fmt::Debug for ReadOnlyGprRmROpcode { +impl fmt::Debug for UnaryRmROpcode { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { match self { - ReadOnlyGprRmROpcode::Bsr => write!(fmt, "bsr"), - ReadOnlyGprRmROpcode::Bsf => write!(fmt, "bsf"), + UnaryRmROpcode::Bsr => write!(fmt, "bsr"), + UnaryRmROpcode::Bsf => write!(fmt, "bsf"), } } } -impl ToString for ReadOnlyGprRmROpcode { - fn to_string(&self) -> String { - format!("{:?}", self) +impl fmt::Display for UnaryRmROpcode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self, f) } } @@ -468,9 +468,9 @@ impl fmt::Debug for SseOpcode { } } -impl ToString for SseOpcode { - fn to_string(&self) -> String { - format!("{:?}", self) +impl fmt::Display for SseOpcode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self, f) } } @@ -519,18 +519,20 @@ impl fmt::Debug for ExtMode { } } -impl ToString for ExtMode { - fn to_string(&self) -> String { - format!("{:?}", self) +impl fmt::Display for ExtMode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self, f) } } -/// These indicate the form of a scalar shift: left, signed right, unsigned right. +/// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right. #[derive(Clone)] pub enum ShiftKind { - Left, - RightZ, - RightS, + ShiftLeft, + /// Inserts zeros in the most significant bits. + ShiftRightLogical, + /// Replicates the sign bit in the most significant bits. + ShiftRightArithmetic, RotateLeft, RotateRight, } @@ -538,9 +540,9 @@ pub enum ShiftKind { impl fmt::Debug for ShiftKind { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { let name = match self { - ShiftKind::Left => "shl", - ShiftKind::RightZ => "shr", - ShiftKind::RightS => "sar", + ShiftKind::ShiftLeft => "shl", + ShiftKind::ShiftRightLogical => "shr", + ShiftKind::ShiftRightArithmetic => "sar", ShiftKind::RotateLeft => "rol", ShiftKind::RotateRight => "ror", }; @@ -548,9 +550,34 @@ impl fmt::Debug for ShiftKind { } } -impl ToString for ShiftKind { - fn to_string(&self) -> String { - format!("{:?}", self) +impl fmt::Display for ShiftKind { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self, f) + } +} + +/// What kind of division or remainer instruction this is? +#[derive(Clone)] +pub enum DivOrRemKind { + SignedDiv, + UnsignedDiv, + SignedRem, + UnsignedRem, +} + +impl DivOrRemKind { + pub(crate) fn is_signed(&self) -> bool { + match self { + DivOrRemKind::SignedDiv | DivOrRemKind::SignedRem => true, + _ => false, + } + } + + pub(crate) fn is_div(&self) -> bool { + match self { + DivOrRemKind::SignedDiv | DivOrRemKind::UnsignedDiv => true, + _ => false, + } } } @@ -665,9 +692,9 @@ impl fmt::Debug for CC { } } -impl ToString for CC { - fn to_string(&self) -> String { - format!("{:?}", self) +impl fmt::Display for CC { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(self, f) } } diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index 285319fe987c..432bbc9a0da5 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -556,7 +556,7 @@ pub(crate) fn emit( } } - Inst::ReadOnly_Gpr_Rm_R { size, op, src, dst } => { + Inst::UnaryRmR { size, op, src, dst } => { let (prefix, rex_flags) = match size { 2 => (LegacyPrefix::_66, RexFlags::clear_w()), 4 => (LegacyPrefix::None, RexFlags::clear_w()), @@ -565,8 +565,8 @@ pub(crate) fn emit( }; let (opcode, num_opcodes) = match op { - ReadOnlyGprRmROpcode::Bsr => (0x0fbd, 2), - ReadOnlyGprRmROpcode::Bsf => (0x0fbc, 2), + UnaryRmROpcode::Bsr => (0x0fbd, 2), + UnaryRmROpcode::Bsf => (0x0fbc, 2), }; match src { @@ -661,8 +661,7 @@ pub(crate) fn emit( } Inst::CheckedDivOrRemSeq { - is_div, - is_signed, + kind, size, divisor, loc, @@ -704,7 +703,7 @@ pub(crate) fn emit( let inst = Inst::trap_if(CC::Z, TrapCode::IntegerDivisionByZero, *loc); inst.emit(sink, flags, state); - let (do_op, done_label) = if *is_signed { + let (do_op, done_label) = if kind.is_signed() { // Now check if the divisor is -1. let inst = Inst::cmp_rmi_r(*size, RegMemImm::imm(0xffffffff), *divisor); inst.emit(sink, flags, state); @@ -715,7 +714,7 @@ pub(crate) fn emit( one_way_jmp(sink, CC::NZ, do_op); // Here, divisor == -1. - if !*is_div { + if !kind.is_div() { // x % -1 = 0; put the result into the destination, $rdx. let done_label = sink.get_label(); @@ -756,7 +755,7 @@ pub(crate) fn emit( } // Fill in the high parts: - if *is_signed { + if kind.is_signed() { // sign-extend the sign-bit of rax into rdx, for signed opcodes. let inst = Inst::sign_extend_rax_to_rdx(*size); inst.emit(sink, flags, state); @@ -766,7 +765,7 @@ pub(crate) fn emit( inst.emit(sink, flags, state); } - let inst = Inst::div(*size, *is_signed, RegMem::reg(*divisor), *loc); + let inst = Inst::div(*size, kind.is_signed(), RegMem::reg(*divisor), *loc); inst.emit(sink, flags, state); // Lowering takes care of moving the result back into the right register, see comment @@ -1047,9 +1046,9 @@ pub(crate) fn emit( let subopcode = match kind { ShiftKind::RotateLeft => 0, ShiftKind::RotateRight => 1, - ShiftKind::Left => 4, - ShiftKind::RightZ => 5, - ShiftKind::RightS => 7, + ShiftKind::ShiftLeft => 4, + ShiftKind::ShiftRightLogical => 5, + ShiftKind::ShiftRightArithmetic => 7, }; let rex = if *is_64 { @@ -1550,6 +1549,7 @@ pub(crate) fn emit( srcloc, } => { // The full address can be encoded in the register, with a relocation. + // Generates: movabsq $name, %dst let enc_dst = int_reg_enc(dst.to_reg()); sink.put1(0x48 | ((enc_dst >> 3) & 1)); sink.put1(0xB8 | (enc_dst & 7)); diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index 33fdeecd9103..ee4674729bb8 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -1227,15 +1227,15 @@ fn test_x64_emit() { )); // ======================================================== - // ReadOnly_Gpr_Rm_R + // UnaryRmR insns.push(( - Inst::read_only_gpr_rm_r(4, ReadOnlyGprRmROpcode::Bsr, RegMem::reg(rsi), w_rdi), + Inst::unary_rm_r(4, UnaryRmROpcode::Bsr, RegMem::reg(rsi), w_rdi), "0FBDFE", "bsrl %esi, %edi", )); insns.push(( - Inst::read_only_gpr_rm_r(8, ReadOnlyGprRmROpcode::Bsr, RegMem::reg(r15), w_rax), + Inst::unary_rm_r(8, UnaryRmROpcode::Bsr, RegMem::reg(r15), w_rax), "490FBDC7", "bsrq %r15, %rax", )); @@ -2303,107 +2303,107 @@ fn test_x64_emit() { // ======================================================== // Shift_R insns.push(( - Inst::shift_r(false, ShiftKind::Left, None, w_rdi), + Inst::shift_r(false, ShiftKind::ShiftLeft, None, w_rdi), "D3E7", "shll %cl, %edi", )); insns.push(( - Inst::shift_r(false, ShiftKind::Left, None, w_r12), + Inst::shift_r(false, ShiftKind::ShiftLeft, None, w_r12), "41D3E4", "shll %cl, %r12d", )); insns.push(( - Inst::shift_r(false, ShiftKind::Left, Some(2), w_r8), + Inst::shift_r(false, ShiftKind::ShiftLeft, Some(2), w_r8), "41C1E002", "shll $2, %r8d", )); insns.push(( - Inst::shift_r(false, ShiftKind::Left, Some(31), w_r13), + Inst::shift_r(false, ShiftKind::ShiftLeft, Some(31), w_r13), "41C1E51F", "shll $31, %r13d", )); insns.push(( - Inst::shift_r(true, ShiftKind::Left, None, w_r13), + Inst::shift_r(true, ShiftKind::ShiftLeft, None, w_r13), "49D3E5", "shlq %cl, %r13", )); insns.push(( - Inst::shift_r(true, ShiftKind::Left, None, w_rdi), + Inst::shift_r(true, ShiftKind::ShiftLeft, None, w_rdi), "48D3E7", "shlq %cl, %rdi", )); insns.push(( - Inst::shift_r(true, ShiftKind::Left, Some(2), w_r8), + Inst::shift_r(true, ShiftKind::ShiftLeft, Some(2), w_r8), "49C1E002", "shlq $2, %r8", )); insns.push(( - Inst::shift_r(true, ShiftKind::Left, Some(3), w_rbx), + Inst::shift_r(true, ShiftKind::ShiftLeft, Some(3), w_rbx), "48C1E303", "shlq $3, %rbx", )); insns.push(( - Inst::shift_r(true, ShiftKind::Left, Some(63), w_r13), + Inst::shift_r(true, ShiftKind::ShiftLeft, Some(63), w_r13), "49C1E53F", "shlq $63, %r13", )); insns.push(( - Inst::shift_r(false, ShiftKind::RightZ, None, w_rdi), + Inst::shift_r(false, ShiftKind::ShiftRightLogical, None, w_rdi), "D3EF", "shrl %cl, %edi", )); insns.push(( - Inst::shift_r(false, ShiftKind::RightZ, Some(2), w_r8), + Inst::shift_r(false, ShiftKind::ShiftRightLogical, Some(2), w_r8), "41C1E802", "shrl $2, %r8d", )); insns.push(( - Inst::shift_r(false, ShiftKind::RightZ, Some(31), w_r13), + Inst::shift_r(false, ShiftKind::ShiftRightLogical, Some(31), w_r13), "41C1ED1F", "shrl $31, %r13d", )); insns.push(( - Inst::shift_r(true, ShiftKind::RightZ, None, w_rdi), + Inst::shift_r(true, ShiftKind::ShiftRightLogical, None, w_rdi), "48D3EF", "shrq %cl, %rdi", )); insns.push(( - Inst::shift_r(true, ShiftKind::RightZ, Some(2), w_r8), + Inst::shift_r(true, ShiftKind::ShiftRightLogical, Some(2), w_r8), "49C1E802", "shrq $2, %r8", )); insns.push(( - Inst::shift_r(true, ShiftKind::RightZ, Some(63), w_r13), + Inst::shift_r(true, ShiftKind::ShiftRightLogical, Some(63), w_r13), "49C1ED3F", "shrq $63, %r13", )); insns.push(( - Inst::shift_r(false, ShiftKind::RightS, None, w_rdi), + Inst::shift_r(false, ShiftKind::ShiftRightArithmetic, None, w_rdi), "D3FF", "sarl %cl, %edi", )); insns.push(( - Inst::shift_r(false, ShiftKind::RightS, Some(2), w_r8), + Inst::shift_r(false, ShiftKind::ShiftRightArithmetic, Some(2), w_r8), "41C1F802", "sarl $2, %r8d", )); insns.push(( - Inst::shift_r(false, ShiftKind::RightS, Some(31), w_r13), + Inst::shift_r(false, ShiftKind::ShiftRightArithmetic, Some(31), w_r13), "41C1FD1F", "sarl $31, %r13d", )); insns.push(( - Inst::shift_r(true, ShiftKind::RightS, None, w_rdi), + Inst::shift_r(true, ShiftKind::ShiftRightArithmetic, None, w_rdi), "48D3FF", "sarq %cl, %rdi", )); insns.push(( - Inst::shift_r(true, ShiftKind::RightS, Some(2), w_r8), + Inst::shift_r(true, ShiftKind::ShiftRightArithmetic, Some(2), w_r8), "49C1F802", "sarq $2, %r8", )); insns.push(( - Inst::shift_r(true, ShiftKind::RightS, Some(63), w_r13), + Inst::shift_r(true, ShiftKind::ShiftRightArithmetic, Some(63), w_r13), "49C1FD3F", "sarq $63, %r13", )); diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index a4b9e04e4fa1..d0c9549892b3 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -51,9 +51,9 @@ pub enum Inst { }, /// Instructions on GPR that only read src and defines dst (dst is not modified): bsr, etc. - ReadOnly_Gpr_Rm_R { + UnaryRmR { size: u8, // 2, 4 or 8 - op: ReadOnlyGprRmROpcode, + op: UnaryRmROpcode, src: RegMem, dst: Writable, }, @@ -66,7 +66,7 @@ pub enum Inst { loc: SourceLoc, }, - /// The high result of a (un)signed multiply: imul/mul using RAX:RDX. + /// The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs. MulHi { size: u8, signed: bool, rhs: RegMem }, /// A synthetic sequence to implement the right inline checks for remainder and division, @@ -77,10 +77,11 @@ pub enum Inst { /// instruction. /// /// Note: %rdx is marked as modified by this instruction, to avoid an early clobber problem - /// with the temporary and divisor. Make sure to zero %rdx right before this instruction! + /// with the temporary and divisor registers. Make sure to zero %rdx right before this + /// instruction, or you might run into regalloc failures where %rdx is live before its first + /// def! CheckedDivOrRemSeq { - is_div: bool, - is_signed: bool, + kind: DivOrRemKind, size: u8, divisor: Reg, tmp: Option>, @@ -283,7 +284,7 @@ pub enum Inst { /// An instruction that will always trigger the illegal instruction exception. Ud2 { trap_info: (SourceLoc, TrapCode) }, - /// Loads an external symbol in a register, with a relocation. + /// Loads an external symbol in a register, with a relocation: movabsq $name, dst LoadExtName { dst: Writable, name: Box, @@ -326,15 +327,15 @@ impl Inst { } } - pub(crate) fn read_only_gpr_rm_r( + pub(crate) fn unary_rm_r( size: u8, - op: ReadOnlyGprRmROpcode, + op: UnaryRmROpcode, src: RegMem, dst: Writable, ) -> Self { debug_assert!(dst.to_reg().get_class() == RegClass::I64); debug_assert!(size == 8 || size == 4 || size == 2); - Self::ReadOnly_Gpr_Rm_R { size, op, src, dst } + Self::UnaryRmR { size, op, src, dst } } pub(crate) fn div(size: u8, signed: bool, divisor: RegMem, loc: SourceLoc) -> Inst { @@ -667,7 +668,7 @@ impl ShowWithRRU for Inst { show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64)), ), - Inst::ReadOnly_Gpr_Rm_R { src, dst, op, size } => format!( + Inst::UnaryRmR { src, dst, op, size } => format!( "{} {}, {}", ljustify2(op.to_string(), suffixBWLQ(*size)), src.show_rru_sized(mb_rru, *size), @@ -700,15 +701,18 @@ impl ShowWithRRU for Inst { rhs.show_rru_sized(mb_rru, *size) ), Inst::CheckedDivOrRemSeq { - is_div, - is_signed, + kind, size, divisor, .. } => format!( - "{}{} $rax:$rdx, {}", - if *is_signed { "s" } else { "u" }, - if *is_div { "div " } else { "rem " }, + "{} $rax:$rdx, {}", + match kind { + DivOrRemKind::SignedDiv => "sdiv", + DivOrRemKind::UnsignedDiv => "udiv", + DivOrRemKind::SignedRem => "srem", + DivOrRemKind::UnsignedRem => "urem", + }, show_ireg_sized(*divisor, mb_rru, *size), ), Inst::SignExtendRaxRdx { size } => match size { @@ -942,7 +946,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) { collector.add_use(regs::rax()); collector.add_mod(Writable::from_reg(regs::rdx())); } - Inst::ReadOnly_Gpr_Rm_R { src, dst, .. } | Inst::XMM_Mov_RM_R { src, dst, .. } => { + Inst::UnaryRmR { src, dst, .. } | Inst::XMM_Mov_RM_R { src, dst, .. } => { src.get_regs_as_uses(collector); collector.add_def(*dst); } @@ -1141,7 +1145,7 @@ fn x64_map_regs(inst: &mut Inst, mapper: &RUM) { ref mut dst, .. } - | Inst::ReadOnly_Gpr_Rm_R { + | Inst::UnaryRmR { ref mut src, ref mut dst, .. diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 0f4f78a5bbdd..4e4074e6f573 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -128,16 +128,16 @@ fn input_to_reg(ctx: Ctx, spec: InsnInput) -> Reg { } enum ExtSpec { - ZeroExtend32, - ZeroExtend64, - SignExtend32, - SignExtend64, + ZeroExtendTo32, + ZeroExtendTo64, + SignExtendTo32, + SignExtendTo64, } fn extend_input_to_reg(ctx: Ctx, spec: InsnInput, ext_spec: ExtSpec) -> Reg { let requested_size = match ext_spec { - ExtSpec::ZeroExtend32 | ExtSpec::SignExtend32 => 32, - ExtSpec::ZeroExtend64 | ExtSpec::SignExtend64 => 64, + ExtSpec::ZeroExtendTo32 | ExtSpec::SignExtendTo32 => 32, + ExtSpec::ZeroExtendTo64 | ExtSpec::SignExtendTo64 => 64, }; let input_size = ctx.input_ty(spec.insn, spec.input).bits(); @@ -156,12 +156,12 @@ fn extend_input_to_reg(ctx: Ctx, spec: InsnInput, ext_spec: ExtSpec) -> Reg { let src = input_to_reg_mem(ctx, spec); let dst = ctx.alloc_tmp(RegClass::I64, requested_ty); match ext_spec { - ExtSpec::ZeroExtend32 | ExtSpec::ZeroExtend64 => { + ExtSpec::ZeroExtendTo32 | ExtSpec::ZeroExtendTo64 => { ctx.emit(Inst::movzx_rm_r( ext_mode, src, dst, /* infallible */ None, )) } - ExtSpec::SignExtend32 | ExtSpec::SignExtend64 => { + ExtSpec::SignExtendTo32 | ExtSpec::SignExtendTo64 => { ctx.emit(Inst::movsx_rm_r( ext_mode, src, dst, /* infallible */ None, )) @@ -292,9 +292,9 @@ fn lower_insn_to_regs>( let dst = output_to_reg(ctx, outputs[0]); let shift_kind = match op { - Opcode::Ishl => ShiftKind::Left, - Opcode::Ushr => ShiftKind::RightZ, - Opcode::Sshr => ShiftKind::RightS, + Opcode::Ishl => ShiftKind::ShiftLeft, + Opcode::Ushr => ShiftKind::ShiftRightLogical, + Opcode::Sshr => ShiftKind::ShiftRightArithmetic, Opcode::Rotl => ShiftKind::RotateLeft, Opcode::Rotr => ShiftKind::RotateRight, _ => unreachable!(), @@ -320,7 +320,7 @@ fn lower_insn_to_regs>( // sub %tmp, %dst let (ext_spec, ty) = match ctx.input_ty(insn, 0) { - I8 | I16 => (Some(ExtSpec::ZeroExtend32), I32), + I8 | I16 => (Some(ExtSpec::ZeroExtendTo32), I32), a if a == I32 || a == I64 => (None, a), _ => unreachable!(), }; @@ -335,9 +335,9 @@ fn lower_insn_to_regs>( let tmp = ctx.alloc_tmp(RegClass::I64, ty); ctx.emit(Inst::imm_r(ty == I64, u64::max_value(), dst)); - ctx.emit(Inst::read_only_gpr_rm_r( + ctx.emit(Inst::unary_rm_r( ty.bytes() as u8, - ReadOnlyGprRmROpcode::Bsr, + UnaryRmROpcode::Bsr, src, tmp, )); @@ -376,9 +376,9 @@ fn lower_insn_to_regs>( let tmp = ctx.alloc_tmp(RegClass::I64, ty); ctx.emit(Inst::imm_r(false /* 64 bits */, ty.bits() as u64, tmp)); - ctx.emit(Inst::read_only_gpr_rm_r( + ctx.emit(Inst::unary_rm_r( ty.bytes() as u8, - ReadOnlyGprRmROpcode::Bsf, + UnaryRmROpcode::Bsf, src, dst, )); @@ -395,7 +395,7 @@ fn lower_insn_to_regs>( // TODO when the x86 flags have use_popcnt, we can use the popcnt instruction. let (ext_spec, ty) = match ctx.input_ty(insn, 0) { - I8 | I16 => (Some(ExtSpec::ZeroExtend32), I32), + I8 | I16 => (Some(ExtSpec::ZeroExtendTo32), I32), a if a == I32 || a == I64 => (None, a), _ => unreachable!(), }; @@ -418,7 +418,12 @@ fn lower_insn_to_regs>( ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1, None)); // shr $1, tmp1 - ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); + ctx.emit(Inst::shift_r( + is_64, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); // mov 0x7777_7777_7777_7777, cst ctx.emit(Inst::imm_r(is_64, 0x7777777777777777, cst)); @@ -443,7 +448,12 @@ fn lower_insn_to_regs>( )); // shr $1, tmp1 - ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); + ctx.emit(Inst::shift_r( + is_64, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); // and cst, tmp1 ctx.emit(Inst::alu_rmi_r( @@ -462,7 +472,12 @@ fn lower_insn_to_regs>( )); // shr $1, tmp1 - ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); + ctx.emit(Inst::shift_r( + is_64, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); // and cst, tmp1 ctx.emit(Inst::alu_rmi_r( @@ -484,7 +499,12 @@ fn lower_insn_to_regs>( ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst, None)); // shr $4, dst - ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(4), dst)); + ctx.emit(Inst::shift_r( + is_64, + ShiftKind::ShiftRightLogical, + Some(4), + dst, + )); // add tmp2, dst ctx.emit(Inst::alu_rmi_r( @@ -517,9 +537,14 @@ fn lower_insn_to_regs>( )); // shr $56, dst - ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(56), dst)); + ctx.emit(Inst::shift_r( + is_64, + ShiftKind::ShiftRightLogical, + Some(56), + dst, + )); } else { - debug_assert_eq!(ty, I32); + assert_eq!(ty, I32); let is_64 = false; let tmp1 = ctx.alloc_tmp(RegClass::I64, I64); @@ -529,7 +554,12 @@ fn lower_insn_to_regs>( ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1, None)); // shr $1, tmp1 - ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); + ctx.emit(Inst::shift_r( + is_64, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); // andq $0x7777_7777, tmp1 ctx.emit(Inst::alu_rmi_r( @@ -551,7 +581,12 @@ fn lower_insn_to_regs>( )); // shr $1, tmp1 - ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); + ctx.emit(Inst::shift_r( + is_64, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); // and 0x7777_7777, tmp1 ctx.emit(Inst::alu_rmi_r( @@ -570,7 +605,12 @@ fn lower_insn_to_regs>( )); // shr $1, tmp1 - ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(1), tmp1)); + ctx.emit(Inst::shift_r( + is_64, + ShiftKind::ShiftRightLogical, + Some(1), + tmp1, + )); // and $0x7777_7777, tmp1 ctx.emit(Inst::alu_rmi_r( @@ -592,7 +632,12 @@ fn lower_insn_to_regs>( ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst, None)); // shr $4, dst - ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(4), dst)); + ctx.emit(Inst::shift_r( + is_64, + ShiftKind::ShiftRightLogical, + Some(4), + dst, + )); // add tmp2, dst ctx.emit(Inst::alu_rmi_r( @@ -619,7 +664,12 @@ fn lower_insn_to_regs>( )); // shr $24, dst - ctx.emit(Inst::shift_r(is_64, ShiftKind::RightZ, Some(24), dst)); + ctx.emit(Inst::shift_r( + is_64, + ShiftKind::ShiftRightLogical, + Some(24), + dst, + )); } } @@ -1085,8 +1135,14 @@ fn lower_insn_to_regs>( } Opcode::Udiv | Opcode::Urem | Opcode::Sdiv | Opcode::Srem => { - let is_div = op == Opcode::Udiv || op == Opcode::Sdiv; - let is_signed = op == Opcode::Sdiv || op == Opcode::Srem; + let kind = match op { + Opcode::Udiv => DivOrRemKind::UnsignedDiv, + Opcode::Sdiv => DivOrRemKind::SignedDiv, + Opcode::Urem => DivOrRemKind::UnsignedRem, + Opcode::Srem => DivOrRemKind::SignedRem, + _ => unreachable!(), + }; + let is_div = kind.is_div(); let input_ty = ctx.input_ty(insn, 0); let size = input_ty.bytes() as u8; @@ -1106,7 +1162,7 @@ fn lower_insn_to_regs>( // pc-relative offsets that must not change, thus requiring regalloc to not // interfere by introducing spills and reloads. // - // Note it keeps the result in $rax (if is_div) or $rdx (if !is_div), so that + // Note it keeps the result in $rax (for divide) or $rdx (for rem), so that // regalloc is aware of the coalescing opportunity between rax/rdx and the // destination register. let divisor = input_to_reg(ctx, inputs[1]); @@ -1117,8 +1173,7 @@ fn lower_insn_to_regs>( }; ctx.emit(Inst::imm_r(true, 0, Writable::from_reg(regs::rdx()))); ctx.emit(Inst::CheckedDivOrRemSeq { - is_div, - is_signed, + kind, size, divisor, tmp, @@ -1128,7 +1183,7 @@ fn lower_insn_to_regs>( let divisor = input_to_reg_mem(ctx, inputs[1]); // Fill in the high parts: - if is_signed { + if kind.is_signed() { // sign-extend the sign-bit of rax into rdx, for signed opcodes. ctx.emit(Inst::sign_extend_rax_to_rdx(size)); } else { @@ -1141,7 +1196,7 @@ fn lower_insn_to_regs>( } // Emit the actual idiv. - ctx.emit(Inst::div(size, is_signed, divisor, ctx.srcloc(insn))); + ctx.emit(Inst::div(size, kind.is_signed(), divisor, ctx.srcloc(insn))); } // Move the result back into the destination reg. @@ -1346,7 +1401,7 @@ impl LowerBackend for X64Backend { insn: branches[0], input: 0, }, - ExtSpec::ZeroExtend32, + ExtSpec::ZeroExtendTo32, ); // Bounds-check (compute flags from idx - jt_size) and branch to default.