fix is_homogeneous_aggregate: use unit size, not total size

compiler/rustc_monomorphize/src/mono_checks/abi_check.rs

@@ -65,9 +65,17 @@ fn do_check_simd_vector_abi<'tcx>(
         let size = arg_abi.layout.size;
         match passes_vectors_by_value(&arg_abi.mode, &arg_abi.layout.backend_repr) {
             UsesVectorRegisters::FixedVector => {
+                // Some targets use homogeneous aggregates, where the unit size counts.
+                let unit_size = match &arg_abi.mode {
+                    PassMode::Cast { pad_i32: _, cast } if cast.prefix.is_empty() => {
+                        cast.rest.unit.size
+                    }
+                    _ => size,
+                };
+
                 let feature_def = tcx.sess.target.features_for_correct_fixed_length_vector_abi();
                 // Find the first feature that provides at least this vector size.
-                let feature = match feature_def.iter().find(|(bits, _)| size.bits() <= *bits) {
+                let feature = match feature_def.iter().find(|(bits, _)| unit_size.bits() <= *bits) {
                     Some((_, feature)) => feature,
                     None => {
                         let (span, _hir_id) = loc();

compiler/rustc_target/src/callconv/aarch64.rs

@@ -35,7 +35,7 @@ where
             // The softfloat ABI treats floats like integers, so they
             // do not get homogeneous aggregate treatment.
             RegKind::Float => cx.target_spec().rustc_abi != Some(RustcAbi::Softfloat),
-            RegKind::Vector { .. } => size.bits() == 64 || size.bits() == 128,
+            RegKind::Vector { .. } => unit.size.bits() == 64 || unit.size.bits() == 128,
         };
 
         valid_unit.then_some(Uniform::consecutive(unit, size))

compiler/rustc_target/src/callconv/arm.rs

@@ -26,7 +26,7 @@ where
         let valid_unit = match unit.kind {
             RegKind::Integer => false,
             RegKind::Float => true,
-            RegKind::Vector { .. } => size.bits() == 64 || size.bits() == 128,
+            RegKind::Vector { .. } => unit.size.bits() == 64 || unit.size.bits() == 128,
         };
 
         valid_unit.then_some(Uniform::consecutive(unit, size))

compiler/rustc_target/src/callconv/powerpc64.rs

@@ -36,7 +36,7 @@ where
         let valid_unit = match unit.kind {
             RegKind::Integer => false,
             RegKind::Float => true,
-            RegKind::Vector { .. } => arg.layout.size.bits() == 128,
+            RegKind::Vector { .. } => unit.size.bits() == 128,
         };
 
         valid_unit.then_some(Uniform::consecutive(unit, arg.layout.size))

tests/auxiliary/minicore.rs

@@ -496,4 +496,5 @@ pub mod simd {
     pub type i64x8 = Simd<i64, 8>;
 
     pub type u8x16 = Simd<u8, 16>;
+    pub type u64x2 = Simd<u8, 16>;
 }

tests/codegen-llvm/aarch64-abi/homogeneous-aggregate.rs

@@ -0,0 +1,84 @@
+//@ add-minicore
+//@ compile-flags: -Cno-prepopulate-passes -Copt-level=0
+//
+//@ revisions: linux win
+//@[linux] compile-flags: --target aarch64-unknown-linux-gnu
+//@[win] compile-flags: --target aarch64-pc-windows-msvc
+//
+//@ needs-llvm-components: aarch64
+
+// Test that homogeneous aggregates are passed and returned with the correct ABI.
+
+#![feature(no_core, lang_items)]
+#![crate_type = "lib"]
+#![no_core]
+
+extern crate minicore;
+use minicore::simd::*;
+use minicore::*;
+
+// A homogeneous float aggregate.
+#[repr(C)]
+pub struct Hfa {
+    pub a: f32,
+    pub b: f32,
+}
+impl Copy for Hfa {}
+
+// CHECK: define void @test_hfa([2 x float] %0)
+#[unsafe(no_mangle)]
+pub extern "C" fn test_hfa(a: Hfa) {
+    hint::black_box(a);
+}
+
+// Fields can be vectors too.
+#[repr(C)]
+pub struct Hfa2V2F64 {
+    pub a: f64x2,
+    pub b: f64x2,
+}
+
+// CHECK: define void @test_hfa_2_f64x2([2 x <2 x double>] %0)
+#[unsafe(no_mangle)]
+pub extern "C" fn test_hfa_2_f64x2(a: Hfa2V2F64) {
+    hint::black_box(a);
+}
+
+#[repr(C)]
+pub struct Hfa2V2U64 {
+    pub a: u64x2,
+    pub b: u64x2,
+}
+
+// CHECK: define void @test_hfa_2_u64x2([2 x <16 x i8>] %0)
+#[unsafe(no_mangle)]
+pub extern "C" fn test_hfa_2_u64x2(a: Hfa2V2U64) {
+    hint::black_box(a);
+}
+
+#[repr(C)]
+pub struct Hfa2V2F32 {
+    pub a: f32x2,
+    pub b: f32x2,
+}
+
+// CHECK: define void @test_hfa_2_f32x2([2 x <2 x float>] %0)
+#[unsafe(no_mangle)]
+pub extern "C" fn test_hfa_2_f32x2(a: Hfa2V2F32) {
+    hint::black_box(a);
+}
+
+#[repr(C)]
+pub struct Hfa4V2F64 {
+    pub a: f64x2,
+    pub b: f64x2,
+    pub c: f64x2,
+    pub d: f64x2,
+}
+
+// CHECK: define void @test_hfa_4_f64x2([4 x <2 x double>] %0)
+#[unsafe(no_mangle)]
+#[target_feature(enable = "neon")]
+pub extern "C" fn test_hfa_4_f64x2(a: Hfa4V2F64) {
+    hint::black_box(a);
+}

tests/codegen-llvm/arm-abi/homogeneous-aggregate.rs

@@ -11,10 +11,12 @@
 // Test that homogeneous aggregates are passed and returned with the correct ABI on 32-bit arm.
 
 #![feature(no_core, lang_items)]
+#![feature(arm_target_feature)]
 #![crate_type = "lib"]
 #![no_core]
 
 extern crate minicore;
+use minicore::simd::*;
 use minicore::*;
 
 // A homogeneous float aggregate, which a hard-float ABI passes in VFP registers.
@@ -68,6 +70,69 @@ pub extern "C" fn test_hfa_4_f64(a: Hfa4F64) {
     hint::black_box(a);
 }
 
+// Fields can be vectors too.
+#[repr(C)]
+pub struct Hfa2V2F64 {
+    pub a: f64x2,
+    pub b: f64x2,
+}
+
+// linux: define void @test_hfa_2_f64x2([2 x <2 x double>] %0)
+// eabi: define dso_local void @test_hfa_2_f64x2([4 x i64] %0)
+// watchos: define void @test_hfa_2_f64x2([2 x <2 x double>] %0)
+#[unsafe(no_mangle)]
+#[target_feature(enable = "neon")]
+pub extern "C" fn test_hfa_2_f64x2(a: Hfa2V2F64) {
+    hint::black_box(a);
+}
+
+#[repr(C)]
+pub struct Hfa2V2U64 {
+    pub a: u64x2,
+    pub b: u64x2,
+}
+
+// linux: define void @test_hfa_2_u64x2([2 x <16 x i8>] %0)
+// eabi: define dso_local void @test_hfa_2_u64x2([4 x i64] %0)
+// watchos: define void @test_hfa_2_u64x2([2 x <16 x i8>] %0)
+#[unsafe(no_mangle)]
+#[target_feature(enable = "neon")]
+pub extern "C" fn test_hfa_2_u64x2(a: Hfa2V2U64) {
+    hint::black_box(a);
+}
+
+#[repr(C)]
+pub struct Hfa2V2F32 {
+    pub a: f32x2,
+    pub b: f32x2,
+}
+
+// linux: define void @test_hfa_2_f32x2([2 x <2 x float>] %0)
+// eabi: define dso_local void @test_hfa_2_f32x2([2 x i64] %0)
+// watchos: define void @test_hfa_2_f32x2([2 x <2 x float>] %0)
+#[unsafe(no_mangle)]
+#[target_feature(enable = "neon")]
+pub extern "C" fn test_hfa_2_f32x2(a: Hfa2V2F32) {
+    hint::black_box(a);
+}
+
+#[repr(C)]
+pub struct Hfa4V2F64 {
+    pub a: f64x2,
+    pub b: f64x2,
+    pub c: f64x2,
+    pub d: f64x2,
+}
+
+// linux: define void @test_hfa_4_f64x2([4 x <2 x double>] %0)
+// eabi: define dso_local void @test_hfa_4_f64x2([8 x i64] %0)
+// watchos: define void @test_hfa_4_f64x2([4 x <2 x double>] %0)
+#[unsafe(no_mangle)]
+#[target_feature(enable = "neon")]
+pub extern "C" fn test_hfa_4_f64x2(a: Hfa4V2F64) {
+    hint::black_box(a);
+}
+
 // A homogeneous aggregate can have at most 4 fields, so this does not qualify.
 #[repr(C)]
 pub struct Floats5 {

tests/codegen-llvm/powerpc64-abi/homogeneous-aggregate.rs

@@ -0,0 +1,96 @@
+//@ add-minicore
+//@ compile-flags: -Cno-prepopulate-passes -Copt-level=0
+//
+//@ revisions: ppc64 ppc64_vsx ppc64le
+//@[ppc64] compile-flags: --target powerpc64-unknown-linux-gnu
+//@[ppc64_vsx] compile-flags: --target powerpc64-unknown-linux-gnu -Ctarget-feature=+vsx
+//@[ppc64le] compile-flags: --target powerpc64le-unknown-linux-gnu
+//
+//@ needs-llvm-components: powerpc
+
+// Test that homogeneous aggregates are passed and returned with the correct ABI.
+
+#![feature(no_core, lang_items)]
+#![crate_type = "lib"]
+#![no_core]
+
+extern crate minicore;
+use minicore::simd::*;
+use minicore::*;
+
+// A homogeneous float aggregate.
+#[repr(C)]
+pub struct Hfa {
+    pub a: f32,
+    pub b: f32,
+}
+impl Copy for Hfa {}
+
+// ppc64: define void @test_hfa(i64 %0)
+// ppc64_vsx: define void @test_hfa(i64 %0)
+// ppc64le: define void @test_hfa([2 x float] %0)
+#[unsafe(no_mangle)]
+pub extern "C" fn test_hfa(a: Hfa) {
+    hint::black_box(a);
+}
+
+// Fields can be vectors too.
+#[repr(C)]
+pub struct Hfa2V2F64 {
+    pub a: f64x2,
+    pub b: f64x2,
+}
+
+// ppc64: define void @test_hfa_2_f64x2([2 x i128] %0)
+// ppc64_vsx: define void @test_hfa_2_f64x2([2 x i128] %0)
+// ppc64le: define void @test_hfa_2_f64x2([2 x <2 x double>] %0)
+#[unsafe(no_mangle)]
+pub extern "C" fn test_hfa_2_f64x2(a: Hfa2V2F64) {
+    hint::black_box(a);
+}
+
+#[repr(C)]
+pub struct Hfa2V2U64 {
+    pub a: u64x2,
+    pub b: u64x2,
+}
+
+// ppc64: define void @test_hfa_2_u64x2([2 x i128] %0)
+// ppc64_vsx: define void @test_hfa_2_u64x2([2 x i128] %0)
+// ppc64le: define void @test_hfa_2_u64x2([2 x <16 x i8>] %0)
+#[unsafe(no_mangle)]
+pub extern "C" fn test_hfa_2_u64x2(a: Hfa2V2U64) {
+    hint::black_box(a);
+}
+
+#[repr(C)]
+pub struct Hfa2V2F32 {
+    pub a: f32x2,
+    pub b: f32x2,
+}
+
+// On PowerPC only 128-bit units are eligible for HVA.
+//
+// ppc64: define void @test_hfa_2_f32x2([2 x i64] %0)
+// ppc64_vsx: define void @test_hfa_2_f32x2([2 x i64] %0)
+// ppc64le: define void @test_hfa_2_f32x2([2 x i64] %0)
+#[unsafe(no_mangle)]
+pub extern "C" fn test_hfa_2_f32x2(a: Hfa2V2F32) {
+    hint::black_box(a);
+}
+
+#[repr(C)]
+pub struct Hfa4V2F64 {
+    pub a: f64x2,
+    pub b: f64x2,
+    pub c: f64x2,
+    pub d: f64x2,
+}
+
+// ppc64: define void @test_hfa_4_f64x2([4 x i128] %0)
+// ppc64_vsx: define void @test_hfa_4_f64x2([4 x i128] %0)
+// ppc64le: define void @test_hfa_4_f64x2([4 x <2 x double>] %0)
+#[unsafe(no_mangle)]
+pub extern "C" fn test_hfa_4_f64x2(a: Hfa4V2F64) {
+    hint::black_box(a);
+}

tests/codegen-llvm/preserve-vec-element-types.rs

@@ -52,45 +52,46 @@ mod tests {
     // CHECK: define [2 x <1 x ptr>] @pair_ptrx1_t([2 x <1 x ptr>] {{.*}} %0)
     #[unsafe(no_mangle)] extern "C" fn pair_ptrx1_t(x: Pair<Simd<*const (), 1>>) -> Pair<Simd<*const (), 1>> { x }
 
-    // When it fits in a 128-bit register, it's passed directly.
+    // When the fields are not 64 or 128 bits in size, they do not qualify as a homogeneous
+    // aggregate, and passed as type-erased sequences of integers.
 
-    // CHECK: define [4 x <4 x i8>] @quad_int8x4_t([4 x <4 x i8>] {{.*}} %0)
+    // CHECK: define [2 x i64] @quad_int8x4_t([2 x i64] {{.*}} %0)
     #[unsafe(no_mangle)] extern "C" fn quad_int8x4_t(x: Quad<Simd<i8, 4>>) -> Quad<Simd<i8, 4>> { x }
 
-    // CHECK: define [4 x <2 x i16>] @quad_int16x2_t([4 x <2 x i16>] {{.*}} %0)
+    // CHECK: define [2 x i64] @quad_int16x2_t([2 x i64] {{.*}} %0)
     #[unsafe(no_mangle)] extern "C" fn quad_int16x2_t(x: Quad<Simd<i16, 2>>) -> Quad<Simd<i16, 2>> { x }
 
-    // CHECK: define [4 x <1 x i32>] @quad_int32x1_t([4 x <1 x i32>] {{.*}} %0)
+    // CHECK: define [2 x i64] @quad_int32x1_t([2 x i64] {{.*}} %0)
     #[unsafe(no_mangle)] extern "C" fn quad_int32x1_t(x: Quad<Simd<i32, 1>>) -> Quad<Simd<i32, 1>> { x }
 
-    // CHECK: define [4 x <2 x half>] @quad_float16x2_t([4 x <2 x half>] {{.*}} %0)
+    // CHECK: define [2 x i64] @quad_float16x2_t([2 x i64] {{.*}} %0)
     #[unsafe(no_mangle)] extern "C" fn quad_float16x2_t(x: Quad<Simd<f16, 2>>) -> Quad<Simd<f16, 2>> { x }
 
-    // CHECK: define [4 x <1 x float>] @quad_float32x1_t([4 x <1 x float>] {{.*}} %0)
+    // CHECK: define [2 x i64] @quad_float32x1_t([2 x i64] {{.*}} %0)
     #[unsafe(no_mangle)] extern "C" fn quad_float32x1_t(x: Quad<Simd<f32, 1>>) -> Quad<Simd<f32, 1>> { x }
 
     // When it doesn't quite fit, padding is added which does erase the type.
 
     // CHECK: define [2 x i64] @triple_int8x4_t
     #[unsafe(no_mangle)] extern "C" fn triple_int8x4_t(x: Triple<Simd<i8, 4>>) -> Triple<Simd<i8, 4>> { x }
 
-    // Other configurations are not passed by-value but indirectly.
+    // Other configurations passed directly when they qualify as a homogeneous aggregate.
 
-    // CHECK: define void @pair_int128x1_t
+    // CHECK: define [2 x <1 x i128>] @pair_int128x1_t([2 x <1 x i128>]
     #[unsafe(no_mangle)] extern "C" fn pair_int128x1_t(x: Pair<Simd<i128, 1>>) -> Pair<Simd<i128, 1>> { x }
 
-    // CHECK: define void @pair_float128x1_t
+    // CHECK: define [2 x <1 x fp128>] @pair_float128x1_t([2 x <1 x fp128>]
     #[unsafe(no_mangle)] extern "C" fn pair_float128x1_t(x: Pair<Simd<f128, 1>>) -> Pair<Simd<f128, 1>> { x }
 
-    // CHECK: define void @pair_int8x16_t
+    // CHECK: define [2 x <16 x i8>] @pair_int8x16_t([2 x <16 x i8>]
     #[unsafe(no_mangle)] extern "C" fn pair_int8x16_t(x: Pair<Simd<i8, 16>>) -> Pair<Simd<i8, 16>> { x }
 
-    // CHECK: define void @pair_int16x8_t
+    // CHECK: define [2 x <8 x i16>] @pair_int16x8_t([2 x <8 x i16>]
     #[unsafe(no_mangle)] extern "C" fn pair_int16x8_t(x: Pair<Simd<i16, 8>>) -> Pair<Simd<i16, 8>> { x }
 
-    // CHECK: define void @triple_int16x8_t
+    // CHECK: define [3 x <8 x i16>] @triple_int16x8_t([3 x <8 x i16>]
     #[unsafe(no_mangle)] extern "C" fn triple_int16x8_t(x: Triple<Simd<i16, 8>>) -> Triple<Simd<i16, 8>> { x }
 
-    // CHECK: define void @quad_int16x8_t
+    // CHECK: define [4 x <8 x i16>] @quad_int16x8_t([4 x <8 x i16>]
     #[unsafe(no_mangle)] extern "C" fn quad_int16x8_t(x: Quad<Simd<i16, 8>>) -> Quad<Simd<i16, 8>> { x }
 }