Skip to content

[AArch64][SVE] Use INS when moving elements from bottom 128b of SVE type #114034

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 31, 2024
Merged

[AArch64][SVE] Use INS when moving elements from bottom 128b of SVE type #114034

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5fce5b4
[AArch64][SVE] Use INS when moving elements from bottom 128b of SVE type
SpencerAbson Oct 29, 2024
2d6e6df
Merge new patterns into existing Neon_INS_elt_patterns work
SpencerAbson Oct 29, 2024
c357f65
simplify implementation of new patterns
SpencerAbson Oct 30, 2024
4f236b0
Restore ordering of pattern defs
SpencerAbson Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 84 additions & 27 deletions llvm/lib/Target/AArch64/AArch64InstrInfo.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7222,44 +7222,101 @@ def : Pat<(v2i64 (int_aarch64_neon_vcopy_lane
V128:$Vd, VectorIndexD:$idx, V128:$Vs, VectorIndexD:$idx2)
)>;

multiclass Neon_INS_elt_pattern<ValueType VT128, ValueType VT64,
ValueType VTScal, Instruction INS> {
def : Pat<(VT128 (vector_insert V128:$src,
(VTScal (vector_extract (VT128 V128:$Rn), (i64 imm:$Immn))),
// Insert an extracted vector element into a 128-bit Neon vector
multiclass Neon_INS_elt_pattern_v128<ValueType VT128, ValueType VT64, ValueType VTSVE,
ValueType VTScal, Operand ExIdxTy, Instruction INS> {
// Extracting from the lower 128-bits of an SVE vector
def : Pat<(VT128 (vector_insert VT128:$Rn,
(VTScal (vector_extract VTSVE:$Rm, (i64 ExIdxTy:$Immn))),
(i64 imm:$Immd))),
(INS VT128:$Rn, imm:$Immd, (VT128 (EXTRACT_SUBREG VTSVE:$Rm, zsub)), ExIdxTy:$Immn)>;

// Extracting from another Neon vector
def : Pat<(VT128 (vector_insert V128:$Rn,
(VTScal (vector_extract (VT128 V128:$Rm), (i64 imm:$Immn))),
(i64 imm:$Immd))),
(INS V128:$src, imm:$Immd, V128:$Rn, imm:$Immn)>;

def : Pat<(VT128 (vector_insert V128:$src,
(VTScal (vector_extract (VT64 V64:$Rn), (i64 imm:$Immn))),
(i64 imm:$Immd))),
(INS V128:$src, imm:$Immd,
(SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immn)>;
(INS V128:$Rn, imm:$Immd, V128:$Rm, imm:$Immn)>;

def : Pat<(VT128 (vector_insert V128:$Rn,
(VTScal (vector_extract (VT64 V64:$Rm), (i64 imm:$Immn))),
(i64 imm:$Immd))),
(INS V128:$Rn, imm:$Immd,
(SUBREG_TO_REG (i64 0), V64:$Rm, dsub), imm:$Immn)>;
}

// Insert an extracted vector element into a 64-bit Neon vector
multiclass Neon_INS_elt_pattern_v64<ValueType VT128, ValueType VT64, ValueType VTSVE,
ValueType VTScal, Operand ExIdxTy, Instruction INS> {
// Extracting from the lower 128-bits of an SVE vector
def : Pat<(VT64 (vector_insert VT64:$Rn,
(VTScal (vector_extract VTSVE:$Rm, (i64 ExIdxTy:$Immn))),
(i64 imm:$Immd))),
(EXTRACT_SUBREG (INS (SUBREG_TO_REG (i64 0), VT64:$Rn, dsub), imm:$Immd,
(VT128 (EXTRACT_SUBREG VTSVE:$Rm, zsub)), ExIdxTy:$Immn),
dsub)>;

def : Pat<(VT64 (vector_insert V64:$src,
(VTScal (vector_extract (VT128 V128:$Rn), (i64 imm:$Immn))),
// Extracting from another Neon vector
def : Pat<(VT64 (vector_insert V64:$Rn,
(VTScal (vector_extract (VT128 V128:$Rm), (i64 imm:$Immn))),
(i64 imm:$Immd))),
(EXTRACT_SUBREG (INS (SUBREG_TO_REG (i64 0), V64:$src, dsub),
imm:$Immd, V128:$Rn, imm:$Immn),
(EXTRACT_SUBREG (INS (SUBREG_TO_REG (i64 0), V64:$Rn, dsub),
imm:$Immd, V128:$Rm, imm:$Immn),
dsub)>;

def : Pat<(VT64 (vector_insert V64:$src,
(VTScal (vector_extract (VT64 V64:$Rn), (i64 imm:$Immn))),
def : Pat<(VT64 (vector_insert V64:$Rn,
(VTScal (vector_extract (VT64 V64:$Rm), (i64 imm:$Immn))),
(i64 imm:$Immd))),
(EXTRACT_SUBREG
(INS (SUBREG_TO_REG (i64 0), V64:$src, dsub), imm:$Immd,
(SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immn),
(INS (SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immd,
(SUBREG_TO_REG (i64 0), V64:$Rm, dsub), imm:$Immn),
dsub)>;
}

// Special case for <1 x double/i64> - insertion may be vector_from_scalar or
// (vector_insert (vec) 0).
multiclass Neon_INS_elt_pattern_v64d<ValueType VT128, ValueType VT64, ValueType VTSVE,
ValueType VTScal> {
// Extracting from the lower 128-bits of an SVE vector
def : Pat<(VT64 (vec_ins_or_scal_vec
(VTScal (vector_extract VTSVE:$Rm, VectorIndexD:$Immn)))),
(EXTRACT_SUBREG
(INSvi64lane (IMPLICIT_DEF), 0, (VT128 (EXTRACT_SUBREG VTSVE:$Rm, zsub)),
VectorIndexD:$Immn),
dsub)>;

def : Pat<(VT64 (vec_ins_or_scal_vec
(VTScal (vector_extract (VT128 V128:$Rm), (i64 imm:$Immn))))),
(EXTRACT_SUBREG
(INSvi64lane (IMPLICIT_DEF), (i64 0), V128:$Rm, imm:$Immn),
dsub)>;

// Extracting from another NEON vector
def : Pat<(VT64 (vec_ins_or_scal_vec
(VTScal (vector_extract (VT64 V64:$Rm), (i64 imm:$Immn))))),
(EXTRACT_SUBREG
(INSvi64lane (IMPLICIT_DEF), (i64 0),
(SUBREG_TO_REG (i64 0), V64:$Rm, dsub), imm:$Immn),
dsub)>;
}

multiclass Neon_INS_elt_pattern<ValueType VT128, ValueType VT64, ValueType SVESrcVT,
ValueType VTScal, Operand ExIdxTy, Instruction INS> {
defm : Neon_INS_elt_pattern_v64<VT128, VT64, SVESrcVT, VTScal, ExIdxTy, INS>;
defm : Neon_INS_elt_pattern_v128<VT128, VT64, SVESrcVT, VTScal, ExIdxTy, INS>;
}

defm : Neon_INS_elt_pattern<v8f16, v4f16, f16, INSvi16lane>;
defm : Neon_INS_elt_pattern<v8bf16, v4bf16, bf16, INSvi16lane>;
defm : Neon_INS_elt_pattern<v4f32, v2f32, f32, INSvi32lane>;
defm : Neon_INS_elt_pattern<v2f64, v1f64, f64, INSvi64lane>;
defm : Neon_INS_elt_pattern<v4f32, v2f32, nxv4f32, f32, VectorIndexS, INSvi32lane>;
defm : Neon_INS_elt_pattern<v8f16, v4f16, nxv8f16, f16, VectorIndexH, INSvi16lane>;
defm : Neon_INS_elt_pattern<v8bf16, v4bf16, nxv8bf16, bf16, VectorIndexH, INSvi16lane>;
defm : Neon_INS_elt_pattern<v4f32, v2f32, nxv4f32, f32, VectorIndexS, INSvi32lane>;
defm : Neon_INS_elt_pattern<v16i8, v8i8, nxv16i8, i32, VectorIndexB, INSvi8lane>;
defm : Neon_INS_elt_pattern<v8i16, v4i16, nxv8i16, i32, VectorIndexH, INSvi16lane>;
defm : Neon_INS_elt_pattern<v4i32, v2i32, nxv4i32, i32, VectorIndexS, INSvi32lane>;

defm : Neon_INS_elt_pattern<v16i8, v8i8, i32, INSvi8lane>;
defm : Neon_INS_elt_pattern<v8i16, v4i16, i32, INSvi16lane>;
defm : Neon_INS_elt_pattern<v4i32, v2i32, i32, INSvi32lane>;
defm : Neon_INS_elt_pattern<v2i64, v1i64, i64, INSvi64lane>;
defm : Neon_INS_elt_pattern_v128<v2f64, v1f64, nxv2f64, f64, VectorIndexD, INSvi64lane>;
defm : Neon_INS_elt_pattern_v64d<v2f64, v1f64, nxv2f64, f64>;
defm : Neon_INS_elt_pattern_v128<v2i64, v1i64, nxv2i64, i64, VectorIndexD, INSvi64lane>;
defm : Neon_INS_elt_pattern_v64d<v2i64, v1i64, nxv2i64, i64>;

// Insert from bitcast
// vector_insert(bitcast(f32 src), n, lane) -> INSvi32lane(src, lane, INSERT_SUBREG(-, n), 0)
Expand Down
50 changes: 0 additions & 50 deletions llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3367,47 +3367,6 @@ let Predicates = [HasSVEorSME] in {
(UMOVvi64 (v2i64 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexD:$index)>;
} // End HasNEON

// Use INS (element) when moving an element from the bottom 128-bits of an SVE type to a NEON vector.
multiclass Neon_ins_sve_elt_pattern<ValueType NeonTy, ValueType NeonQTy, ValueType SVETy, ValueType ScalTy,
Operand IdxTy, Operand NarrowIdxTy, Instruction INS> {
// Insert into 128-bit NEON type from lowest 128-bits of SVE type
def : Pat<(NeonQTy (vector_insert V128:$src,
(ScalTy (vector_extract SVETy:$Rn, IdxTy:$idx_extract)),
(IdxTy:$idx_insert))),
(INS V128:$src, IdxTy:$idx_insert,
(NeonQTy (EXTRACT_SUBREG SVETy:$Rn, zsub)), IdxTy:$idx_extract)>;

// Insert into 64-bit NEON type from lowest 128-bits of SVE type
def : Pat<(NeonTy (vector_insert V64:$src,
(ScalTy (vector_extract SVETy:$Rn, IdxTy:$idx_extract)),
(NarrowIdxTy:$idx_insert))),
(EXTRACT_SUBREG
(INS
(INSERT_SUBREG (NeonQTy (IMPLICIT_DEF)), V64:$src, dsub), NarrowIdxTy:$idx_insert,
(NeonQTy (EXTRACT_SUBREG SVETy:$Rn, zsub)), IdxTy:$idx_extract),
dsub)>;
}

// Inserting into <1 x double/i64> will just create a new vector from the scalar value.
multiclass Neon_ins_64b_sve_elt_pattern<ValueType NeonTy, ValueType NeonQTy, ValueType SVETy,
ValueType ScalTy> {
// Insert into 128-bit NEON type from lowest 128-bits of SVE type
def : Pat<(NeonQTy (vector_insert V128:$src,
(ScalTy (vector_extract SVETy:$Rn, VectorIndexD:$idx_extract)),
(VectorIndexD:$idx_insert))),
(INSvi64lane
V128:$src, VectorIndexD:$idx_insert, (NeonQTy (EXTRACT_SUBREG SVETy:$Rn, zsub)),
VectorIndexD:$idx_extract)>;

// Insert into 64-bit NEON type from lowest 128-bits of SVE type
def : Pat<(NeonTy (scalar_to_vector
(ScalTy (vector_extract SVETy:$Rn, VectorIndexD:$idx_extract)))),
(EXTRACT_SUBREG
(INSvi64lane (IMPLICIT_DEF), 0, (NeonQTy (EXTRACT_SUBREG SVETy:$Rn, zsub)),
VectorIndexD:$idx_extract),
dsub)>;
}

let Predicates = [HasNEON] in {
def : Pat<(sext_inreg (vector_extract nxv16i8:$vec, VectorIndexB:$index), i8),
(SMOVvi8to32 (v16i8 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexB:$index)>;
Expand All @@ -3421,15 +3380,6 @@ let Predicates = [HasSVEorSME] in {

def : Pat<(sext (i32 (vector_extract nxv4i32:$vec, VectorIndexS:$index))),
(SMOVvi32to64 (v4i32 (EXTRACT_SUBREG ZPR:$vec, zsub)), VectorIndexS:$index)>;

defm : Neon_ins_sve_elt_pattern<v8i8, v16i8, nxv16i8, i32, VectorIndexB, VectorIndexH, INSvi8lane>;
defm : Neon_ins_sve_elt_pattern<v4f16, v8f16, nxv8f16, f16, VectorIndexH, VectorIndexS, INSvi16lane>;
defm : Neon_ins_sve_elt_pattern<v4bf16, v8bf16, nxv8bf16, bf16, VectorIndexH, VectorIndexS, INSvi16lane>;
defm : Neon_ins_sve_elt_pattern<v4i16, v8i16, nxv8i16, i32, VectorIndexH, VectorIndexS, INSvi16lane>;
defm : Neon_ins_sve_elt_pattern<v2f32, v4f32, nxv4f32, f32, VectorIndexS, VectorIndexD, INSvi32lane>;
defm : Neon_ins_sve_elt_pattern<v2i32, v4i32, nxv4i32, i32, VectorIndexS, VectorIndexD, INSvi32lane>;
defm : Neon_ins_64b_sve_elt_pattern<v1f64, v2f64, nxv2f64, f64>;
defm : Neon_ins_64b_sve_elt_pattern<v1i64, v2i64, nxv2i64, i64>;
} // End HasNEON

// Extract first element from vector.
Expand Down
Loading