Complex layout phase 2

compiler/rustc_abi/src/layout/ty.rs

@@ -7,7 +7,7 @@ use rustc_macros::StableHash;
 
 use crate::layout::{FieldIdx, VariantIdx};
 use crate::{
-    AbiAlign, Align, BackendRepr, FieldsShape, Float, HasDataLayout, LayoutData, Niche,
+    AbiAlign, Align, BackendRepr, FieldsShape, Float, HasDataLayout, LayoutData, Niche, Numeric,
     PointeeInfo, Primitive, Size, Variants,
 };
 
@@ -299,7 +299,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         found
     }
 
-    pub fn complex_float<C>(&self, cx: &C) -> Option<Float>
+    pub fn complex_number<C>(&self, cx: &C) -> Option<Numeric>
     where
         Ty: TyAbiInterface<'a, C> + Copy,
     {
@@ -314,7 +314,18 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         debug_assert_eq!(a, b);
 
         match a.primitive() {
-            Primitive::Float(f) => Some(f),
+            Primitive::Int(i, sign) => Some(Numeric::Int(i, sign)),
+            Primitive::Float(f) => Some(Numeric::Float(f)),
+            Primitive::Pointer(_) => None,
+        }
+    }
+
+    pub fn complex_float<C>(&self, cx: &C) -> Option<Float>
+    where
+        Ty: TyAbiInterface<'a, C> + Copy,
+    {
+        match self.complex_number(cx) {
+            Some(Numeric::Float(f)) => Some(f),
             _ => None,
         }
     }

compiler/rustc_abi/src/lib.rs

@@ -1445,6 +1445,31 @@ impl Float {
     }
 }
 
+/// Numeric primitives.
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "nightly", derive(StableHash))]
+pub enum Numeric {
+    /// The `bool` is the signedness of the `Integer` type.
+    Int(Integer, bool),
+    Float(Float),
+}
+
+impl Numeric {
+    pub fn size(self) -> Size {
+        match self {
+            Numeric::Int(integer, _) => integer.size(),
+            Numeric::Float(float) => float.size(),
+        }
+    }
+
+    pub fn reg_kind(self) -> RegKind {
+        match self {
+            Numeric::Int(_, _) => RegKind::Integer,
+            Numeric::Float(_) => RegKind::Float,
+        }
+    }
+}
+
 /// Fundamental unit of memory access and layout.
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "nightly", derive(StableHash))]

compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs

@@ -123,7 +123,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 _ => unreachable!("{:?}", self.layout.backend_repr),
             },
             PassMode::Cast { ref cast, pad_i32 } => {
-                assert!(!pad_i32, "padding support not yet implemented");
+                assert_eq!(pad_i32, 0, "padding support not yet implemented");
                 cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect()
             }
             PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {

compiler/rustc_codegen_gcc/src/abi.rs

@@ -169,10 +169,10 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     continue;
                 }
                 PassMode::Cast { ref cast, pad_i32 } => {
-                    // add padding
-                    if pad_i32 {
-                        argument_tys.push(Reg::i32().gcc_type(cx));
-                    }
+                    // Add padding.
+                    argument_tys
+                        .extend(std::iter::repeat_n(Reg::i32().gcc_type(cx), usize::from(pad_i32)));
+
                     let ty = cast.gcc_type(cx);
                     apply_attrs(ty, &cast.attrs, argument_tys.len())
                 }

compiler/rustc_codegen_gcc/src/type_of.rs

@@ -346,8 +346,8 @@ impl<'gcc, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         fn_abi.ptr_to_gcc_type(self)
     }
 
-    fn reg_backend_type(&self, _ty: &Reg) -> Type<'gcc> {
-        unimplemented!();
+    fn reg_backend_type(&self, ty: &Reg) -> Type<'gcc> {
+        ty.gcc_type(self)
     }
 
     fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {

compiler/rustc_codegen_llvm/src/abi.rs

@@ -406,10 +406,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 }
                 PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => cx.type_ptr(),
                 PassMode::Cast { cast, pad_i32 } => {
-                    // add padding
-                    if *pad_i32 {
-                        llargument_tys.push(Reg::i32().llvm_type(cx));
-                    }
+                    // Add padding.
+                    llargument_tys.extend(std::iter::repeat_n(
+                        Reg::i32().llvm_type(cx),
+                        usize::from(*pad_i32),
+                    ));
+
                     // Compute the LLVM type we use for this function from the cast type.
                     // We assume here that ABI-compatible Rust types have the same cast type.
                     cast.llvm_type(cx)
@@ -581,7 +583,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     }
                 }
                 PassMode::Cast { cast, pad_i32 } => {
-                    if *pad_i32 {
+                    for _ in 0..*pad_i32 {
                         apply(&ArgAttributes::new());
                     }
                     apply(&cast.attrs);
@@ -667,7 +669,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     apply(bx.cx, b);
                 }
                 PassMode::Cast { cast, pad_i32 } => {
-                    if *pad_i32 {
+                    for _ in 0..*pad_i32 {
                         apply(bx.cx, &ArgAttributes::new());
                     }
                     apply(bx.cx, &cast.attrs);

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1936,9 +1936,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ) {
         match arg.mode {
             PassMode::Ignore => return,
-            PassMode::Cast { pad_i32: true, .. } => {
+            PassMode::Cast { pad_i32, .. } => {
                 // Fill padding with undef value, where applicable.
-                llargs.push(bx.const_undef(bx.reg_backend_type(&Reg::i32())));
+                let undef = bx.const_undef(bx.reg_backend_type(&Reg::i32()));
+                llargs.extend(std::iter::repeat_n(undef, usize::from(pad_i32)));
             }
             PassMode::Pair(..) => match op.val {
                 Pair(a, b) => {

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -501,8 +501,8 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 for i in 0..tupled_arg_tys.len() {
                     let arg = &fx.fn_abi.args[idx];
                     idx += 1;
-                    if let PassMode::Cast { pad_i32: true, .. } = arg.mode {
-                        llarg_idx += 1;
+                    if let PassMode::Cast { pad_i32, .. } = arg.mode {
+                        llarg_idx += usize::from(pad_i32);
                     }
                     let pr_field = place.project_field(bx, i);
                     bx.store_fn_arg(arg, &mut llarg_idx, pr_field);
@@ -529,8 +529,8 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
             let arg = &fx.fn_abi.args[idx];
             idx += 1;
-            if let PassMode::Cast { pad_i32: true, .. } = arg.mode {
-                llarg_idx += 1;
+            if let PassMode::Cast { pad_i32, .. } = arg.mode {
+                llarg_idx += usize::from(pad_i32);
             }
 
             if !memory_locals.contains(local) {

compiler/rustc_codegen_ssa/src/mir/naked_asm.rs

@@ -474,7 +474,7 @@ fn wasm_type<'tcx>(signature: &mut String, arg_abi: &ArgAbi<'_, Ty<'tcx>>, ptr_t
         },
         PassMode::Cast { pad_i32, ref cast } => {
             // For wasm, Cast is used for single-field primitive wrappers like `struct Wrapper(i64);`
-            assert!(!pad_i32, "not currently used by wasm calling convention");
+            assert_eq!(pad_i32, 0, "not currently used by wasm calling convention");
             assert!(cast.prefix.is_empty(), "no prefix");
             assert_eq!(cast.rest.total, arg_abi.layout.size, "single item");
 

compiler/rustc_public/src/abi.rs

@@ -55,7 +55,7 @@ pub enum PassMode {
     /// The argument has a layout abi of `ScalarPair`.
     Pair(Opaque, Opaque),
     /// Pass the argument after casting it.
-    Cast { pad_i32: bool, cast: Opaque },
+    Cast { pad_i32: u8, cast: Opaque },
     /// Pass the argument indirectly via a hidden pointer.
     Indirect { attrs: Opaque, meta_attrs: Opaque, on_stack: bool },
 }

compiler/rustc_target/src/callconv/mips.rs

@@ -1,16 +1,36 @@
-use rustc_abi::{HasDataLayout, Size, TyAbiInterface};
+use rustc_abi::{Float, HasDataLayout, Integer, Numeric, Reg, RegKind, Size, TyAbiInterface};
 
-use crate::callconv::{ArgAbi, FnAbi, Reg, Uniform};
+use crate::callconv::{ArgAbi, CastTarget, FnAbi, Uniform};
 
-fn classify_ret<Ty, C>(cx: &C, ret: &mut ArgAbi<'_, Ty>, offset: &mut Size)
+fn classify_ret<'a, Ty, C>(cx: &C, ret: &mut ArgAbi<'a, Ty>, offset: &mut Size)
 where
+    Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
-    if !ret.layout.is_aggregate() {
-        ret.extend_integer_width_to(32);
-    } else {
+    let dl = cx.data_layout();
+    let size = ret.layout.size;
+
+    if let Some(component) = ret.layout.complex_number(cx) {
+        match component {
+            Numeric::Float(Float::F128) => {
+                // Same as an aggregate.
+                ret.make_indirect();
+                *offset += dl.pointer_size();
+            }
+            Numeric::Int(Integer::I8 | Integer::I16, _) => {
+                // Pack Complex<{integer}> into a single register if that fits.
+                ret.cast_to(Reg { kind: RegKind::Integer, size });
+            }
+            _ => {
+                let reg = Reg { kind: component.reg_kind(), size: ret.layout.field(cx, 0).size };
+                ret.cast_to(CastTarget::pair(reg, reg));
+            }
+        }
+    } else if ret.layout.is_aggregate() {
         ret.make_indirect();
-        *offset += cx.data_layout().pointer_size();
+        *offset += dl.pointer_size();
+    } else {
+        ret.extend_integer_width_to(32);
     }
 }
 
@@ -35,7 +55,7 @@ where
 
     let size = arg.layout.size;
     if arg.layout.is_aggregate() {
-        let pad_i32 = !offset.is_aligned(align);
+        let pad_i32 = u8::from(!offset.is_aligned(align));
         arg.cast_to_and_pad_i32(Uniform::new(Reg::i32(), size), pad_i32);
     } else {
         arg.extend_integer_width_to(32);

compiler/rustc_target/src/callconv/mips64.rs

@@ -1,10 +1,13 @@
 use arrayvec::ArrayVec;
 use rustc_abi::{
-    BackendRepr, FieldsShape, Float, HasDataLayout, Primitive, Reg, Size, TyAbiInterface,
+    BackendRepr, FieldsShape, Float, HasDataLayout, Numeric, Primitive, Reg, RegKind, Size,
+    TyAbiInterface,
 };
 
 use crate::callconv::{ArgAbi, ArgAttribute, ArgExtension, CastTarget, FnAbi, PassMode, Uniform};
 
+const NUM_ARG_SLOTS: u64 = 8;
+
 fn extend_integer_width_mips<Ty>(arg: &mut ArgAbi<'_, Ty>, bits: u64) {
     // Always sign extend u32 values on 64-bit mips
     if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr
@@ -55,6 +58,19 @@ where
     let size = ret.layout.size;
     let bits = size.bits();
     if bits <= 128 {
+        if let Some(component) = ret.layout.complex_number(cx) {
+            if matches!(component, Numeric::Int(..)) && size <= cx.data_layout().pointer_size() {
+                // Return a Complex<{integer}> packed into a single register when that fits.
+                // We match GCC, not Clang, see https://github.com/llvm/llvm-project/issues/212109.
+                ret.cast_to(Reg { kind: RegKind::Integer, size });
+            } else {
+                // Otherwise pass in 2 registers.
+                let reg = Reg { kind: component.reg_kind(), size: component.size() };
+                ret.cast_to(CastTarget::pair(reg, reg));
+            }
+            return;
+        }
+
         // Unlike other architectures which return aggregates in registers, MIPS n64 limits the
         // use of float registers to structures (not unions) containing exactly one or two
         // float fields.
@@ -96,12 +112,40 @@ where
 
     // Detect need for padding
     let align = Ord::clamp(arg.layout.align.abi, dl.i64_align, dl.i128_align);
-    let pad_i32 = !offset.is_aligned(align);
+    let pad_i32 = u8::from(!offset.is_aligned(align));
 
     if !arg.layout.is_aggregate() {
         extend_integer_width_mips(arg, 64);
     } else if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
         arg.make_indirect();
+    } else if arg.layout.complex_float(cx).is_some() && size > dl.pointer_size() * 2 {
+        // Complex<f128> is passed in 4 GPRs, but aligned to 16 so may need padding.
+        let reg = Reg { kind: RegKind::Float, size: arg.layout.field(cx, 0).size };
+        arg.cast_to_and_pad_i32(CastTarget::pair(reg, reg), pad_i32);
+    } else if arg.layout.is_complex_number(cx) && size <= dl.pointer_size() * 2 {
+        let slot = dl.pointer_size();
+        let curr_offset = offset.align_to(align);
+
+        if arg.layout.complex_float(cx).is_some() {
+            // Only pass a Complex<f32>/Complex<f64> in FPRs when two argument slots are free,
+            // otherwise pack it into GPRs (or the stack) like an integer of the same size.
+            if curr_offset.bytes() / slot.bytes() + 2 <= NUM_ARG_SLOTS {
+                // The default `PassMode::Pair` already passes one component per register. Both
+                // components claim a slot, even for a Complex<f32> which could fit into one slot.
+                *offset = curr_offset + slot * 2;
+                return;
+            }
+
+            arg.cast_to_and_pad_i32(Uniform::new(Reg::i64(), size), pad_i32);
+        } else if size > slot {
+            // Complex<i64> is passed as 2 separate arguments, which is what the default
+            // `PassMode::Pair` already does.
+        } else {
+            // Cast Complex<i8> into i16, Complex<i16> to i32, etc. The inreg attribute is required
+            // to make the bits land in the right (upper) bits on big endian targets.
+            let cast_target = CastTarget::from(Reg { kind: RegKind::Integer, size });
+            arg.cast_to(cast_target.with_attrs(ArgAttribute::InReg.into()));
+        }
     } else {
         match arg.layout.fields {
             FieldsShape::Primitive => unreachable!(),

compiler/rustc_target/src/callconv/mod.rs

@@ -55,8 +55,9 @@ pub enum PassMode {
     Pair(ArgAttributes, ArgAttributes),
     /// Pass the argument after casting it. See the `CastTarget` docs for details.
     ///
-    /// `pad_i32` indicates if a `Reg::i32()` dummy argument is emitted before the real argument.
-    Cast { pad_i32: bool, cast: Box<CastTarget> },
+    /// `pad_i32` indicates how many `Reg::i32()` dummy arguments are emitted before the real
+    /// argument.
+    Cast { pad_i32: u8, cast: Box<CastTarget> },
     /// Pass the argument indirectly via a hidden pointer.
     ///
     /// The `meta_attrs` value, if any, is for the metadata (vtable or length) of an unsized
@@ -507,12 +508,11 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
     }
 
     pub fn cast_to<T: Into<CastTarget>>(&mut self, target: T) {
-        self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32: false };
+        self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32: 0 };
     }
 
     pub fn cast_to_with_attrs<T: Into<CastTarget>>(&mut self, target: T, attrs: ArgAttributes) {
-        self.mode =
-            PassMode::Cast { cast: Box::new(target.into().with_attrs(attrs)), pad_i32: false };
+        self.mode = PassMode::Cast { cast: Box::new(target.into().with_attrs(attrs)), pad_i32: 0 };
     }
 
     /// Cast to `target`, forwarding `NoUndef` only when the layout provably has no uninit
@@ -535,7 +535,7 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         self.cast_to_with_attrs(target, attr.into());
     }
 
-    pub fn cast_to_and_pad_i32<T: Into<CastTarget>>(&mut self, target: T, pad_i32: bool) {
+    pub fn cast_to_and_pad_i32<T: Into<CastTarget>>(&mut self, target: T, pad_i32: u8) {
         self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32 };
     }