diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 69112d868bff8..af7a39b2580a3 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1081,7 +1081,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_DEINTERLEAVE, ISD::VECTOR_INTERLEAVE, - ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE}, + ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE, + ISD::VECTOR_COMPRESS}, VT, Custom); MVT EltVT = VT.getVectorElementType(); if (isTypeLegal(EltVT)) @@ -1333,7 +1334,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::UNDEF, VT, Custom); setOperationAction({ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE, - ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, + ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR, + ISD::VECTOR_COMPRESS}, VT, Custom); // FIXME: mload, mstore, mgather, mscatter, vp_load/store, @@ -1440,8 +1442,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT}, VT, Custom); - - setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom); } // Custom-legalize bitcasts from fixed-length vectors to scalar types. diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compress-fp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compress-fp.ll index 8f1ff7ed4a11e..3069d60110757 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compress-fp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-compress-fp.ll @@ -1,6 +1,92 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin < %s | FileCheck %s + +define <1 x bfloat> @vector_compress_v1bf16(<1 x bfloat> %v, <1 x i1> %mask) { +; CHECK-LABEL: vector_compress_v1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 1, e16, mf4, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %ret = call <1 x bfloat> @llvm.experimental.vector.compress.v1bf16(<1 x bfloat> %v, <1 x i1> %mask, <1 x bfloat> undef) + ret <1 x bfloat> %ret +} + +define <1 x bfloat> @vector_compress_v1bf16_passthru(<1 x bfloat> %passthru, <1 x bfloat> %v, <1 x i1> %mask) { +; CHECK-LABEL: vector_compress_v1bf16_passthru: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 1, e16, mf4, tu, ma +; CHECK-NEXT: vcompress.vm v8, v9, v0 +; CHECK-NEXT: ret + %ret = call <1 x bfloat> @llvm.experimental.vector.compress.v1bf16(<1 x bfloat> %v, <1 x i1> %mask, <1 x bfloat> %passthru) + ret <1 x bfloat> %ret +} + +define <2 x bfloat> @vector_compress_v2bf16(<2 x bfloat> %v, <2 x i1> %mask) { +; CHECK-LABEL: vector_compress_v2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 2, e16, mf4, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %ret = call <2 x bfloat> @llvm.experimental.vector.compress.v2bf16(<2 x bfloat> %v, <2 x i1> %mask, <2 x bfloat> undef) + ret <2 x bfloat> %ret +} + +define <2 x bfloat> @vector_compress_v2bf16_passthru(<2 x bfloat> %passthru, <2 x bfloat> %v, <2 x i1> %mask) { +; CHECK-LABEL: vector_compress_v2bf16_passthru: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 2, e16, mf4, tu, ma +; CHECK-NEXT: vcompress.vm v8, v9, v0 +; CHECK-NEXT: ret + %ret = call <2 x bfloat> @llvm.experimental.vector.compress.v2bf16(<2 x bfloat> %v, <2 x i1> %mask, <2 x bfloat> %passthru) + ret <2 x bfloat> %ret +} + +define <4 x bfloat> @vector_compress_v4bf16(<4 x bfloat> %v, <4 x i1> %mask) { +; CHECK-LABEL: vector_compress_v4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %ret = call <4 x bfloat> @llvm.experimental.vector.compress.v4bf16(<4 x bfloat> %v, <4 x i1> %mask, <4 x bfloat> undef) + ret <4 x bfloat> %ret +} + +define <4 x bfloat> @vector_compress_v4bf16_passthru(<4 x bfloat> %passthru, <4 x bfloat> %v, <4 x i1> %mask) { +; CHECK-LABEL: vector_compress_v4bf16_passthru: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, tu, ma +; CHECK-NEXT: vcompress.vm v8, v9, v0 +; CHECK-NEXT: ret + %ret = call <4 x bfloat> @llvm.experimental.vector.compress.v4bf16(<4 x bfloat> %v, <4 x i1> %mask, <4 x bfloat> %passthru) + ret <4 x bfloat> %ret +} + +define <8 x bfloat> @vector_compress_v8bf16(<8 x bfloat> %v, <8 x i1> %mask) { +; CHECK-LABEL: vector_compress_v8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e16, m1, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vmv.v.v v8, v9 +; CHECK-NEXT: ret + %ret = call <8 x bfloat> @llvm.experimental.vector.compress.v8bf16(<8 x bfloat> %v, <8 x i1> %mask, <8 x bfloat> undef) + ret <8 x bfloat> %ret +} + +define <8 x bfloat> @vector_compress_v8bf16_passthru(<8 x bfloat> %passthru, <8 x bfloat> %v, <8 x i1> %mask) { +; CHECK-LABEL: vector_compress_v8bf16_passthru: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e16, m1, tu, ma +; CHECK-NEXT: vcompress.vm v8, v9, v0 +; CHECK-NEXT: ret + %ret = call <8 x bfloat> @llvm.experimental.vector.compress.v8bf16(<8 x bfloat> %v, <8 x i1> %mask, <8 x bfloat> %passthru) + ret <8 x bfloat> %ret +} define <1 x half> @vector_compress_v1f16(<1 x half> %v, <1 x i1> %mask) { ; CHECK-LABEL: vector_compress_v1f16: diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-compress.ll b/llvm/test/CodeGen/RISCV/rvv/vector-compress.ll index 85d72ad2fe9cb..7516a72a92bc8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vector-compress.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vector-compress.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 -; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh %s -o - | FileCheck %s -; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh %s -o - | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfh,+zvfbfmin < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin < %s | FileCheck %s +; RUN: llc -verify-machineinstrs -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zvfhmin,+zvfbfmin < %s | FileCheck %s ; Vector compress for i8 type @@ -472,6 +474,134 @@ define @vector_compress_nxv8i64_passthru( % ret %ret } +; Vector compress for bf16 type + +define @vector_compress_nxv1bf16( %data, %mask) { +; CHECK-LABEL: vector_compress_nxv1bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, mf4, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv1bf16( %data, %mask, undef) + ret %ret +} + +define @vector_compress_nxv1bf16_passthru( %passthru, %data, %mask) { +; CHECK-LABEL: vector_compress_nxv1bf16_passthru: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, mf4, tu, ma +; CHECK-NEXT: vcompress.vm v8, v9, v0 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv1bf16( %data, %mask, %passthru) + ret %ret +} + +define @vector_compress_nxv2bf16( %data, %mask) { +; CHECK-LABEL: vector_compress_nxv2bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv2bf16( %data, %mask, undef) + ret %ret +} + +define @vector_compress_nxv2bf16_passthru( %passthru, %data, %mask) { +; CHECK-LABEL: vector_compress_nxv2bf16_passthru: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, mf2, tu, ma +; CHECK-NEXT: vcompress.vm v8, v9, v0 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv2bf16( %data, %mask, %passthru) + ret %ret +} + +define @vector_compress_nxv4bf16( %data, %mask) { +; CHECK-LABEL: vector_compress_nxv4bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m1, ta, ma +; CHECK-NEXT: vcompress.vm v9, v8, v0 +; CHECK-NEXT: vmv.v.v v8, v9 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv4bf16( %data, %mask, undef) + ret %ret +} + +define @vector_compress_nxv4bf16_passthru( %passthru, %data, %mask) { +; CHECK-LABEL: vector_compress_nxv4bf16_passthru: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m1, tu, ma +; CHECK-NEXT: vcompress.vm v8, v9, v0 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv4bf16( %data, %mask, %passthru) + ret %ret +} + +define @vector_compress_nxv8bf16( %data, %mask) { +; CHECK-LABEL: vector_compress_nxv8bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m2, ta, ma +; CHECK-NEXT: vcompress.vm v10, v8, v0 +; CHECK-NEXT: vmv.v.v v8, v10 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv8bf16( %data, %mask, undef) + ret %ret +} + +define @vector_compress_nxv8bf16_passthru( %passthru, %data, %mask) { +; CHECK-LABEL: vector_compress_nxv8bf16_passthru: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m2, tu, ma +; CHECK-NEXT: vcompress.vm v8, v10, v0 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv8bf16( %data, %mask, %passthru) + ret %ret +} + +define @vector_compress_nxv16bf16( %data, %mask) { +; CHECK-LABEL: vector_compress_nxv16bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma +; CHECK-NEXT: vcompress.vm v12, v8, v0 +; CHECK-NEXT: vmv.v.v v8, v12 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv16bf16( %data, %mask, undef) + ret %ret +} + +define @vector_compress_nxv16bf16_passthru( %passthru, %data, %mask) { +; CHECK-LABEL: vector_compress_nxv16bf16_passthru: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m4, tu, ma +; CHECK-NEXT: vcompress.vm v8, v12, v0 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv16bf16( %data, %mask, %passthru) + ret %ret +} + +define @vector_compress_nxv32bf16( %data, %mask) { +; CHECK-LABEL: vector_compress_nxv32bf16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m8, ta, ma +; CHECK-NEXT: vcompress.vm v16, v8, v0 +; CHECK-NEXT: vmv.v.v v8, v16 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv32bf16( %data, %mask, undef) + ret %ret +} + +define @vector_compress_nxv32bf16_passthru( %passthru, %data, %mask) { +; CHECK-LABEL: vector_compress_nxv32bf16_passthru: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a0, zero, e16, m8, tu, ma +; CHECK-NEXT: vcompress.vm v8, v16, v0 +; CHECK-NEXT: ret + %ret = call @llvm.experimental.vector.compress.nxv32bf16( %data, %mask, %passthru) + ret %ret +} + ; Vector compress for f16 type define @vector_compress_nxv1f16( %data, %mask) {