From 01e5d303e07925d2a910773c9790249c9c22a4e3 Mon Sep 17 00:00:00 2001 From: Matt Devereau Date: Tue, 28 Nov 2023 09:34:34 +0000 Subject: [PATCH] Update intrinsics to include f16, bf16, f32 variants as per https://github.com/ARM-software/acle/pull/278 --- .../acle_sme2_luti2_lane_zt.c | 42 +++++++++++++++++++ .../acle_sme2_luti4_lane_zt.c | 42 +++++++++++++++++++ .../aarch64-sme2-intrinsics/acle_sme2_imm.cpp | 28 ++++++++++++- llvm/lib/Target/AArch64/SMEInstrFormats.td | 12 ++++++ .../AArch64/sme2-intrinsics-luti2-lane.ll | 30 +++++++++++++ .../AArch64/sme2-intrinsics-luti4-lane.ll | 30 +++++++++++++ 6 files changed, 182 insertions(+), 2 deletions(-) diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_luti2_lane_zt.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_luti2_lane_zt.c index ebabbfc815c1df..d470d729589e88 100644 --- a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_luti2_lane_zt.c +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_luti2_lane_zt.c @@ -67,6 +67,34 @@ svint16_t test_svluti2_lane_zt_s16(svint16_t zn) __arm_streaming __arm_shared_za return svluti2_lane_zt_s16(0, zn, 2); } +// CHECK-LABEL: @test_svluti2_lane_zt_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti2.lane.zt.nxv8f16(i32 0, [[ZN:%.*]], i32 2) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z24test_svluti2_lane_zt_f16u13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti2.lane.zt.nxv8f16(i32 0, [[ZN:%.*]], i32 2) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svfloat16_t test_svluti2_lane_zt_f16(svfloat16_t zn) __arm_streaming __arm_shared_za __arm_preserves_za { + return svluti2_lane_zt_f16(0, zn, 2); +} + +// CHECK-LABEL: @test_svluti2_lane_zt_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti2.lane.zt.nxv8bf16(i32 0, [[ZN:%.*]], i32 2) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z25test_svluti2_lane_zt_bf16u14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti2.lane.zt.nxv8bf16(i32 0, [[ZN:%.*]], i32 2) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbfloat16_t test_svluti2_lane_zt_bf16(svbfloat16_t zn) __arm_streaming __arm_shared_za __arm_preserves_za { + return svluti2_lane_zt_bf16(0, zn, 2); +} + // CHECK-LABEL: @test_svluti2_lane_zt_u32( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti2.lane.zt.nxv4i32(i32 0, [[ZN:%.*]], i32 2) @@ -94,3 +122,17 @@ svuint32_t test_svluti2_lane_zt_u32(svuint32_t zn) __arm_streaming __arm_shared_ svint32_t test_svluti2_lane_zt_s32(svint32_t zn) __arm_streaming __arm_shared_za __arm_preserves_za { return svluti2_lane_zt_s32(0, zn, 2); } + +// CHECK-LABEL: @test_svluti2_lane_zt_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti2.lane.zt.nxv4f32(i32 0, [[ZN:%.*]], i32 2) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z24test_svluti2_lane_zt_f32u13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti2.lane.zt.nxv4f32(i32 0, [[ZN:%.*]], i32 2) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svfloat32_t test_svluti2_lane_zt_f32(svfloat32_t zn) __arm_streaming __arm_shared_za __arm_preserves_za { + return svluti2_lane_zt_f32(0, zn, 2); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_luti4_lane_zt.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_luti4_lane_zt.c index c6b9b7c8275756..6788cfab447d56 100644 --- a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_luti4_lane_zt.c +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_luti4_lane_zt.c @@ -66,6 +66,34 @@ svint16_t test_svluti4_lane_zt_s16(svint16_t zn) __arm_streaming __arm_shared_za return svluti4_lane_zt_s16(0, zn, 2); } +// CHECK-LABEL: @test_svluti4_lane_zt_f16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti4.lane.zt.nxv8f16(i32 0, [[ZN:%.*]], i32 2) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z24test_svluti4_lane_zt_f16u13__SVFloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti4.lane.zt.nxv8f16(i32 0, [[ZN:%.*]], i32 2) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svfloat16_t test_svluti4_lane_zt_f16(svfloat16_t zn) __arm_streaming __arm_shared_za __arm_preserves_za { + return svluti4_lane_zt_f16(0, zn, 2); +} + +// CHECK-LABEL: @test_svluti4_lane_zt_bf16( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti4.lane.zt.nxv8bf16(i32 0, [[ZN:%.*]], i32 2) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z25test_svluti4_lane_zt_bf16u14__SVBfloat16_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti4.lane.zt.nxv8bf16(i32 0, [[ZN:%.*]], i32 2) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbfloat16_t test_svluti4_lane_zt_bf16(svbfloat16_t zn) __arm_streaming __arm_shared_za __arm_preserves_za { + return svluti4_lane_zt_bf16(0, zn, 2); +} + // CHECK-LABEL: @test_svluti4_lane_zt_u32( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti4.lane.zt.nxv4i32(i32 0, [[ZN:%.*]], i32 2) @@ -93,3 +121,17 @@ svuint32_t test_svluti4_lane_zt_u32(svuint32_t zn) __arm_streaming __arm_shared_ svint32_t test_svluti4_lane_zt_s32(svint32_t zn) __arm_streaming __arm_shared_za __arm_preserves_za { return svluti4_lane_zt_s32(0, zn, 2); } + +// CHECK-LABEL: @test_svluti4_lane_zt_f32( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti4.lane.zt.nxv4f32(i32 0, [[ZN:%.*]], i32 2) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z24test_svluti4_lane_zt_f32u13__SVFloat32_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sme.luti4.lane.zt.nxv4f32(i32 0, [[ZN:%.*]], i32 2) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svfloat32_t test_svluti4_lane_zt_f32(svfloat32_t zn) __arm_streaming __arm_shared_za __arm_preserves_za { + return svluti4_lane_zt_f32(0, zn, 2); +} diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp index 4d6e322122ea35..c36bad10e22760 100644 --- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp +++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_imm.cpp @@ -78,7 +78,7 @@ void test_svluti4_lane_zt_x4(svuint8_t zn) __arm_streaming __arm_shared_za __arm svluti4_lane_zt_f32_x4(0, zn, 2); // expected-error {{argument value 2 is outside the valid range [0, 1]}} } -void test_svluti2_lane_zt(svuint8_t zn_u8, svuint16_t zn_u16, svuint32_t zn_u32) __arm_streaming __arm_shared_za __arm_preserves_za { +void test_svluti2_lane_zt(svuint8_t zn_u8, svuint16_t zn_u16, svuint32_t zn_u32, svfloat16_t zn_f16, svbfloat16_t zn_bf16, svfloat32_t zn_f32) __arm_streaming __arm_shared_za __arm_preserves_za { // Test Reg Offset svluti2_lane_zt_u8(1, zn_u8, 2); // expected-error {{argument value 1 is outside the valid range [0, 0]}} // Test index value range @@ -88,12 +88,24 @@ void test_svluti2_lane_zt(svuint8_t zn_u8, svuint16_t zn_u16, svuint32_t zn_u32) // Test index value range svluti2_lane_zt_u16(0, zn_u16, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} // Test Reg Offset + svluti2_lane_zt_f16(1, zn_f16, 2); // expected-error {{argument value 1 is outside the valid range [0, 0]}} + // Test index value range + svluti2_lane_zt_f16(0, zn_f16, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + // Test Reg Offset + svluti2_lane_zt_bf16(1, zn_bf16, 2); // expected-error {{argument value 1 is outside the valid range [0, 0]}} + // Test index value range + svluti2_lane_zt_bf16(0, zn_bf16, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + // Test Reg Offset svluti2_lane_zt_u32(1, zn_u32, 2); // expected-error {{argument value 1 is outside the valid range [0, 0]}} // Test index value range svluti2_lane_zt_u32(0, zn_u32, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} + // Test Reg Offset + svluti2_lane_zt_f32(1, zn_f32, 2); // expected-error {{argument value 1 is outside the valid range [0, 0]}} + // Test index value range + svluti2_lane_zt_f32(0, zn_f32, 16); // expected-error {{argument value 16 is outside the valid range [0, 15]}} } -void test_svluti4_lane_zt(svuint8_t zn_u8, svuint16_t zn_u16, svuint32_t zn_u32) __arm_streaming __arm_shared_za __arm_preserves_za { +void test_svluti4_lane_zt(svuint8_t zn_u8, svuint16_t zn_u16, svuint32_t zn_u32, svfloat16_t zn_f16, svbfloat16_t zn_bf16, svfloat32_t zn_f32) __arm_streaming __arm_shared_za __arm_preserves_za { // Test Reg Offset svluti4_lane_zt_u8(1, zn_u8, 2); // expected-error {{argument val]ue 1 is outside the valid range [0, 0]}} // Test index value range @@ -103,7 +115,19 @@ void test_svluti4_lane_zt(svuint8_t zn_u8, svuint16_t zn_u16, svuint32_t zn_u32) // Test index value range svluti4_lane_zt_u16(0, zn_u16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} // Test Reg Offset + svluti4_lane_zt_f16(1, zn_f16, 2); // expected-error {{argument value 1 is outside the valid range [0, 0]}} + // Test index value range + svluti4_lane_zt_f16(0, zn_f16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + // Test Reg Offset + svluti4_lane_zt_bf16(1, zn_bf16, 2); // expected-error {{argument value 1 is outside the valid range [0, 0]}} + // Test index value range + svluti4_lane_zt_bf16(0, zn_bf16, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + // Test Reg Offset svluti4_lane_zt_u32(1, zn_u32, 2); // expected-error {{argument value 1 is outside the valid range [0, 0]}} // Test index value range svluti4_lane_zt_u32(0, zn_u32, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} + // Test Reg Offset + svluti4_lane_zt_f32(1, zn_f32, 2); // expected-error {{argument value 1 is outside the valid range [0, 0]}} + // Test index value range + svluti4_lane_zt_f32(0, zn_f32, 8); // expected-error {{argument value 8 is outside the valid range [0, 7]}} } diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td index 64e096dd10b28d..f86ae8e82f7b2f 100644 --- a/llvm/lib/Target/AArch64/SMEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td @@ -3243,6 +3243,12 @@ multiclass sme2_luti2_vector_index (!cast(NAME # _H) $zt, nxv8i16:$zn, (i32 VectorIndexB32b_timm:$imm))>; def : Pat<(nxv4i32 (intrinsic (imm_to_zt untyped:$zt), nxv4i32:$zn, (i32 VectorIndexB32b_timm:$imm))), (!cast(NAME # _S) $zt, nxv4i32:$zn, (i32 VectorIndexB32b_timm:$imm))>; + def : Pat<(nxv8f16 (intrinsic (imm_to_zt untyped:$zt), nxv8f16:$zn, (i32 VectorIndexB32b_timm:$imm))), + (!cast(NAME # _H) $zt, nxv8f16:$zn, (i32 VectorIndexB32b_timm:$imm))>; + def : Pat<(nxv8bf16 (intrinsic (imm_to_zt untyped:$zt), nxv8bf16:$zn, (i32 VectorIndexB32b_timm:$imm))), + (!cast(NAME # _H) $zt, nxv8bf16:$zn, (i32 VectorIndexB32b_timm:$imm))>; + def : Pat<(nxv4f32 (intrinsic (imm_to_zt untyped:$zt), nxv4f32:$zn, (i32 VectorIndexB32b_timm:$imm))), + (!cast(NAME # _S) $zt, nxv4f32:$zn, (i32 VectorIndexB32b_timm:$imm))>; } class sme2_luti4_vector_index sz, RegisterOperand vector_ty, @@ -3263,6 +3269,12 @@ multiclass sme2_luti4_vector_index (!cast(NAME # _H) $zt, nxv8i16:$zn, (i32 VectorIndexH32b_timm:$imm))>; def : Pat<(nxv4i32 (intrinsic (imm_to_zt untyped:$zt), nxv4i32:$zn, (i32 VectorIndexH32b_timm:$imm))), (!cast(NAME # _S) $zt, nxv4i32:$zn, (i32 VectorIndexH32b_timm:$imm))>; + def : Pat<(nxv8f16 (intrinsic (imm_to_zt untyped:$zt), nxv8f16:$zn, (i32 VectorIndexH32b_timm:$imm))), + (!cast(NAME # _H) $zt, nxv8f16:$zn, (i32 VectorIndexH32b_timm:$imm))>; + def : Pat<(nxv8bf16 (intrinsic (imm_to_zt untyped:$zt), nxv8bf16:$zn, (i32 VectorIndexH32b_timm:$imm))), + (!cast(NAME # _H) $zt, nxv8bf16:$zn, (i32 VectorIndexH32b_timm:$imm))>; + def : Pat<(nxv4f32 (intrinsic (imm_to_zt untyped:$zt), nxv4f32:$zn, (i32 VectorIndexH32b_timm:$imm))), + (!cast(NAME # _S) $zt, nxv4f32:$zn, (i32 VectorIndexH32b_timm:$imm))>; } // SME2 lookup table expand two contiguous registers diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-luti2-lane.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-luti2-lane.ll index 0a743b43a43ecf..cc6076e5707512 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-luti2-lane.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-luti2-lane.ll @@ -30,6 +30,36 @@ define @luti2_i32( %x) { ret %res } +define @luti2_f16( %x) { +; CHECK-LABEL: luti2_f16: +; CHECK: // %bb.0: +; CHECK-NEXT: luti2 z0.h, zt0, z0[15] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sme.luti2.lane.zt.nxv8f16(i32 0, %x, i32 15) + ret %res +} + +define @luti2_bf16( %x) { +; CHECK-LABEL: luti2_bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: luti2 z0.h, zt0, z0[15] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sme.luti2.lane.zt.nxv8bf16(i32 0, %x, i32 15) + ret %res +} + +define @luti2_f32( %x) { +; CHECK-LABEL: luti2_f32: +; CHECK: // %bb.0: +; CHECK-NEXT: luti2 z0.s, zt0, z0[15] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sme.luti2.lane.zt.nxv4f32(i32 0, %x, i32 15) + ret %res +} + declare @llvm.aarch64.sme.luti2.lane.zt.nxv16i8(i32, , i32) declare @llvm.aarch64.sme.luti2.lane.zt.nxv8i16(i32, , i32) declare @llvm.aarch64.sme.luti2.lane.zt.nxv4i32(i32, , i32) +declare @llvm.aarch64.sme.luti2.lane.zt.nxv8f16(i32, , i32) +declare @llvm.aarch64.sme.luti2.lane.zt.nxv8bf16(i32, , i32) +declare @llvm.aarch64.sme.luti2.lane.zt.nxv4f32(i32, , i32) diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-luti4-lane.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-luti4-lane.ll index eebd9ae59bb384..5ec862eeddbbd1 100644 --- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-luti4-lane.ll +++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-luti4-lane.ll @@ -30,6 +30,36 @@ define @luti4_i32( %x) { ret %res } +define @luti4_f16( %x) { +; CHECK-LABEL: luti4_f16: +; CHECK: // %bb.0: +; CHECK-NEXT: luti4 z0.h, zt0, z0[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sme.luti4.lane.zt.nxv8f16(i32 0, %x, i32 7) + ret %res +} + +define @luti4_bf16( %x) { +; CHECK-LABEL: luti4_bf16: +; CHECK: // %bb.0: +; CHECK-NEXT: luti4 z0.h, zt0, z0[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sme.luti4.lane.zt.nxv8bf16(i32 0, %x, i32 7) + ret %res +} + +define @luti4_f32( %x) { +; CHECK-LABEL: luti4_f32: +; CHECK: // %bb.0: +; CHECK-NEXT: luti4 z0.s, zt0, z0[7] +; CHECK-NEXT: ret + %res = call @llvm.aarch64.sme.luti4.lane.zt.nxv4f32(i32 0, %x, i32 7) + ret %res +} + declare @llvm.aarch64.sme.luti4.lane.zt.nxv16i8(i32, , i32) declare @llvm.aarch64.sme.luti4.lane.zt.nxv8i16(i32, , i32) declare @llvm.aarch64.sme.luti4.lane.zt.nxv4i32(i32, , i32) +declare @llvm.aarch64.sme.luti4.lane.zt.nxv8f16(i32, , i32) +declare @llvm.aarch64.sme.luti4.lane.zt.nxv8bf16(i32, , i32) +declare @llvm.aarch64.sme.luti4.lane.zt.nxv4f32(i32, , i32)