-
Notifications
You must be signed in to change notification settings - Fork 15.2k
[X86][AMX] Support AMX-FP8 #113850
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[X86][AMX] Support AMX-FP8 #113850
Changes from 5 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| /*===------------- amxfp8intrin.h - AMX intrinsics -*- C++ -*----------------=== | ||
| * | ||
| * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
| * See https://llvm.org/LICENSE.txt for license information. | ||
| * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
| * | ||
| *===------------------------------------------------------------------------=== | ||
| */ | ||
|
|
||
| #ifndef __IMMINTRIN_H | ||
| #error "Never use <amxfp8intrin.h> directly; include <immintrin.h> instead." | ||
| #endif /* __IMMINTRIN_H */ | ||
|
|
||
| #ifndef __AMXFP8INTRIN_H | ||
| #define __AMXFP8INTRIN_H | ||
| #ifdef __x86_64__ | ||
|
|
||
| /// These instructions compute dot product of brain-float8 (BF8) or | ||
| /// hybrid-float8 (HF8) accumulating into a single precision (FP32). The input | ||
| /// elements can be BF8 or HF8. These instructions have three tile operands, one | ||
| /// source/dest accumulator operand, and two source operands, \a a and \a b. The | ||
| /// \a a and \a b operands can be BF8 or HF8 independently, and the source/dest | ||
| /// operand, \a dst is always FP32. | ||
| /// | ||
| /// \headerfile <immintrin.h> | ||
| /// | ||
| /// \code | ||
| /// void _tile_dpbf8ps (__tile dst, __tile a, __tile b) | ||
| /// \endcode | ||
| /// | ||
| /// This intrinsic corresponds to the \c TDPBF8PS instruction, which is the dot | ||
| /// product of a BF8 value (\a a) by a BF8 value (\a b) accumulating into a | ||
|
||
| /// Single Precision (FP32) source/dest (\a dst). | ||
| /// | ||
| /// \param dst | ||
| /// The destination tile. Max size is 1024 Bytes. | ||
| /// \param a | ||
| /// The 1st source tile. Max size is 1024 Bytes. | ||
| /// \param b | ||
| /// The 2nd source tile. Max size is 1024 Bytes. | ||
| #define _tile_dpbf8ps(dst, a, b) __builtin_ia32_tdpbf8ps((dst), (a), (b)) | ||
|
|
||
| /// \code | ||
| /// void _tile_dpbhf8ps (__tile dst, __tile a, __tile b) | ||
| /// \endcode | ||
| /// | ||
| /// This intrinsic corresponds to the \c TDPBHF8PS instruction, which is the dot | ||
| /// product of a BF8 value (\a a) by an HF8 value (\a b) accumulating into a | ||
| /// Single Precision (FP32) source/dest (\a dst). | ||
| /// | ||
| /// \param dst | ||
| /// The destination tile. Max size is 1024 Bytes. | ||
| /// \param a | ||
| /// The 1st source tile. Max size is 1024 Bytes. | ||
| /// \param b | ||
| /// The 2nd source tile. Max size is 1024 Bytes. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The current rule of rule of thumb is See examples in other files.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks. Updated. |
||
| #define _tile_dpbhf8ps(dst, a, b) __builtin_ia32_tdpbhf8ps((dst), (a), (b)) | ||
|
|
||
| /// \code | ||
| /// void _tile_dphbf8ps (__tile dst, __tile a, __tile b) | ||
| /// \endcode | ||
| /// | ||
| /// This intrinsic corresponds to the \c TDPHBF8PS instruction, which is the dot | ||
| /// product of an HF8 value (\a a) by a BF8 value (\a b) accumulating into a | ||
| /// Single Precision (FP32) source/dest (\a dst). | ||
| /// | ||
| /// \param dst | ||
| /// The destination tile. Max size is 1024 Bytes. | ||
| /// \param a | ||
| /// The 1st source tile. Max size is 1024 Bytes. | ||
| /// \param b | ||
| /// The 2nd source tile. Max size is 1024 Bytes. | ||
| #define _tile_dphbf8ps(dst, a, b) __builtin_ia32_tdphbf8ps((dst), (a), (b)) | ||
|
|
||
| /// \code | ||
| /// void _tile_dphf8ps (__tile dst, __tile a, __tile b) | ||
| /// \endcode | ||
| /// | ||
| /// This intrinsic corresponds to the \c TDPHF8PS instruction, which is the dot | ||
| /// product of an HF8 value (\a a) by an HF8 value (\a b) accumulating into a | ||
| /// Single Precision (FP32) source/dest (\a dst). | ||
| /// | ||
| /// \param dst | ||
| /// The destination tile. Max size is 1024 Bytes. | ||
| /// \param a | ||
| /// The 1st source tile. Max size is 1024 Bytes. | ||
| /// \param b | ||
| /// The 2nd source tile. Max size is 1024 Bytes. | ||
| #define _tile_dphf8ps(dst, a, b) __builtin_ia32_tdphf8ps((dst), (a), (b)) | ||
|
|
||
| #endif /* __x86_64__ */ | ||
| #endif /* __AMXFP8INTRIN_H */ | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| // RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +amx-fp8 \ | ||
| // RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s | ||
| #include <immintrin.h> | ||
|
|
||
| void test_amx(void *data) { | ||
| //CHECK-LABEL: @test_amx | ||
| //CHECK: call void @llvm.x86.tdpbf8ps(i8 1, i8 2, i8 3) | ||
| _tile_dpbf8ps(1, 2, 3); | ||
| } | ||
|
|
||
| void test_amx2(void *data) { | ||
| //CHECK-LABEL: @test_amx2 | ||
| //CHECK: call void @llvm.x86.tdpbhf8ps(i8 1, i8 2, i8 3) | ||
| _tile_dpbhf8ps(1, 2, 3); | ||
| } | ||
|
|
||
| void test_amx3(void *data) { | ||
| //CHECK-LABEL: @test_amx3 | ||
| //CHECK: call void @llvm.x86.tdphbf8ps(i8 1, i8 2, i8 3) | ||
| _tile_dphbf8ps(1, 2, 3); | ||
| } | ||
|
|
||
| void test_amx4(void *data) { | ||
| //CHECK-LABEL: @test_amx4 | ||
| //CHECK: call void @llvm.x86.tdphf8ps(i8 1, i8 2, i8 3) | ||
| _tile_dphf8ps(1, 2, 3); | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| // RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +amx-tile -target-feature +amx-fp8 -verify | ||
|
|
||
| #include <immintrin.h> | ||
|
|
||
| void test_amx(void *data) { | ||
| _tile_dpbf8ps(4, 3, 3); // expected-error {{tile arguments must refer to different tiles}} | ||
| _tile_dpbhf8ps(4, 3, 3); // expected-error {{tile arguments must refer to different tiles}} | ||
| _tile_dphbf8ps(4, 3, 3); // expected-error {{tile arguments must refer to different tiles}} | ||
| _tile_dphf8ps(4, 3, 3); // expected-error {{tile arguments must refer to different tiles}} | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| // RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +amx-fp8 -emit-llvm -o - -Wall -Werror -pedantic | FileCheck %s | ||
|
|
||
| void f_tilemul(short a) | ||
| { | ||
| //CHECK: call void asm sideeffect "tileloadd 0(%rsi,%r13,4), %tmm0 \0A\09tileloadd 0(%rdx,%r14,4), %tmm6 \0A\09tdpbf8ps %tmm6, %tmm0, %tmm7 \0A\09tilestored %tmm7, 0(%r12,%r15,4) \0A\09", "~{memory},~{tmm0},~{tmm6},~{tmm7},~{dirflag},~{fpsr},~{flags}"() | ||
| __asm__ volatile ("tileloadd 0(%%rsi,%%r13,4), %%tmm0 \n\t" | ||
| "tileloadd 0(%%rdx,%%r14,4), %%tmm6 \n\t" | ||
| "tdpbf8ps %%tmm6, %%tmm0, %%tmm7 \n\t" | ||
| "tilestored %%tmm7, 0(%%r12,%%r15,4) \n\t" | ||
| ::: "memory", "tmm0", "tmm6", "tmm7"); | ||
|
|
||
| //CHECK: call void asm sideeffect "tileloadd 0(%rsi,%r13,4), %tmm0 \0A\09tileloadd 0(%rdx,%r14,4), %tmm6 \0A\09tdpbhf8ps %tmm6, %tmm0, %tmm7 \0A\09tilestored %tmm7, 0(%r12,%r15,4) \0A\09", "~{memory},~{tmm0},~{tmm6},~{tmm7},~{dirflag},~{fpsr},~{flags}"() | ||
| __asm__ volatile ("tileloadd 0(%%rsi,%%r13,4), %%tmm0 \n\t" | ||
| "tileloadd 0(%%rdx,%%r14,4), %%tmm6 \n\t" | ||
| "tdpbhf8ps %%tmm6, %%tmm0, %%tmm7 \n\t" | ||
| "tilestored %%tmm7, 0(%%r12,%%r15,4) \n\t" | ||
| ::: "memory", "tmm0", "tmm6", "tmm7"); | ||
|
|
||
| //CHECK: call void asm sideeffect "tileloadd 0(%rsi,%r13,4), %tmm0 \0A\09tileloadd 0(%rdx,%r14,4), %tmm6 \0A\09tdphbf8ps %tmm6, %tmm0, %tmm7 \0A\09tilestored %tmm7, 0(%r12,%r15,4) \0A\09", "~{memory},~{tmm0},~{tmm6},~{tmm7},~{dirflag},~{fpsr},~{flags}"() | ||
| __asm__ volatile ("tileloadd 0(%%rsi,%%r13,4), %%tmm0 \n\t" | ||
| "tileloadd 0(%%rdx,%%r14,4), %%tmm6 \n\t" | ||
| "tdphbf8ps %%tmm6, %%tmm0, %%tmm7 \n\t" | ||
| "tilestored %%tmm7, 0(%%r12,%%r15,4) \n\t" | ||
| ::: "memory", "tmm0", "tmm6", "tmm7"); | ||
|
|
||
| //CHECK: call void asm sideeffect "tileloadd 0(%rsi,%r13,4), %tmm0 \0A\09tileloadd 0(%rdx,%r14,4), %tmm6 \0A\09tdphf8ps %tmm6, %tmm0, %tmm7 \0A\09tilestored %tmm7, 0(%r12,%r15,4) \0A\09", "~{memory},~{tmm0},~{tmm6},~{tmm7},~{dirflag},~{fpsr},~{flags}"() | ||
| __asm__ volatile ("tileloadd 0(%%rsi,%%r13,4), %%tmm0 \n\t" | ||
| "tileloadd 0(%%rdx,%%r14,4), %%tmm6 \n\t" | ||
| "tdphf8ps %%tmm6, %%tmm0, %%tmm7 \n\t" | ||
| "tilestored %%tmm7, 0(%%r12,%%r15,4) \n\t" | ||
| ::: "memory", "tmm0", "tmm6", "tmm7"); | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5994,6 +5994,23 @@ let TargetPrefix = "x86" in { | |
| [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, | ||
| llvm_x86amx_ty, llvm_x86amx_ty, | ||
| llvm_x86amx_ty], []>; | ||
|
|
||
| def int_x86_tdpbf8ps : ClangBuiltin<"__builtin_ia32_tdpbf8ps">, | ||
| Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], | ||
| [ImmArg<ArgIndex<0>>, | ||
| ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; | ||
| def int_x86_tdpbhf8ps : ClangBuiltin<"__builtin_ia32_tdpbhf8ps">, | ||
| Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], | ||
| [ImmArg<ArgIndex<0>>, | ||
| ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; | ||
| def int_x86_tdphbf8ps : ClangBuiltin<"__builtin_ia32_tdphbf8ps">, | ||
| Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], | ||
| [ImmArg<ArgIndex<0>>, | ||
| ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; | ||
| def int_x86_tdphf8ps : ClangBuiltin<"__builtin_ia32_tdphf8ps">, | ||
| Intrinsic<[], [llvm_i8_ty, llvm_i8_ty, llvm_i8_ty], | ||
| [ImmArg<ArgIndex<0>>, | ||
| ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; | ||
|
Comment on lines
+5997
to
+6013
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Move them to line 5919. This is for internal intrinsics. |
||
| } | ||
|
|
||
| //===----------------------------------------------------------------------===// | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We usually don't add description for the whole file. I think we can simply remove it, ot If you want, you can follow the way like https://github.com/llvm/llvm-project/blob/main/clang/lib/Headers/stddef.h#L10-L21
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Okay. I'll remove it.