From f9cdd2208e15e9c349b5de5173426731d3abf071 Mon Sep 17 00:00:00 2001 From: Jean-Didier Pailleux Date: Fri, 26 Sep 2025 10:05:53 +0200 Subject: [PATCH 1/2] [mlir][LLVMIR] Adding `ivdepEnable` parameter in LoopVectorizeAttr `loop.vectorize.ivdep.enable` metadata has been added to LLVM to ignoring vector dependencies. Adding this parameter to LoopVectorizeAttr will allow Flang to use this metadata with the `!DIR$ IVDEP` directive. --- .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td | 17 ++++++++--------- .../Target/LLVMIR/LoopAnnotationImporter.cpp | 8 +++++--- .../LLVMIR/LoopAnnotationTranslation.cpp | 1 + .../Target/LLVMIR/Import/metadata-loop.ll | 19 ++++++++++--------- mlir/test/Target/LLVMIR/loop-metadata.mlir | 5 +++-- 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td index 75bce6b0a0e54..965f273ab53d4 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td @@ -106,15 +106,14 @@ def LoopVectorizeAttr : LLVM_Attr<"LoopVectorize", "loop_vectorize"> { the "!llvm.loop.vectorize" metadata. }]; - let parameters = (ins - OptionalParameter<"BoolAttr">:$disable, - OptionalParameter<"BoolAttr">:$predicateEnable, - OptionalParameter<"BoolAttr">:$scalableEnable, - OptionalParameter<"IntegerAttr">:$width, - OptionalParameter<"LoopAnnotationAttr">:$followupVectorized, - OptionalParameter<"LoopAnnotationAttr">:$followupEpilogue, - OptionalParameter<"LoopAnnotationAttr">:$followupAll - ); + let parameters = (ins OptionalParameter<"BoolAttr">:$disable, + OptionalParameter<"BoolAttr">:$predicateEnable, + OptionalParameter<"BoolAttr">:$scalableEnable, + OptionalParameter<"BoolAttr">:$ivdepEnable, + OptionalParameter<"IntegerAttr">:$width, + OptionalParameter<"LoopAnnotationAttr">:$followupVectorized, + OptionalParameter<"LoopAnnotationAttr">:$followupEpilogue, + OptionalParameter<"LoopAnnotationAttr">:$followupAll); let assemblyFormat = "`<` struct(params) `>`"; diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp index e4905423347a2..9354d2112db72 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationImporter.cpp @@ -291,6 +291,8 @@ FailureOr LoopMetadataConversion::convertVectorizeAttr() { lookupBoolNode("llvm.loop.vectorize.predicate.enable"); FailureOr scalableEnable = lookupBoolNode("llvm.loop.vectorize.scalable.enable"); + FailureOr ivdepEnable = + lookupBoolNode("llvm.loop.vectorize.ivdep.enable"); FailureOr width = lookupIntNode("llvm.loop.vectorize.width"); FailureOr followupVec = lookupFollowupNode("llvm.loop.vectorize.followup_vectorized"); @@ -299,9 +301,9 @@ FailureOr LoopMetadataConversion::convertVectorizeAttr() { FailureOr followupAll = lookupFollowupNode("llvm.loop.vectorize.followup_all"); - return createIfNonNull(ctx, enable, predicateEnable, - scalableEnable, width, followupVec, - followupEpi, followupAll); + return createIfNonNull( + ctx, enable, predicateEnable, scalableEnable, ivdepEnable, width, + followupVec, followupEpi, followupAll); } FailureOr LoopMetadataConversion::convertInterleaveAttr() { diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp index f3ec18c9074e4..cd892404f113e 100644 --- a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp @@ -108,6 +108,7 @@ void LoopAnnotationConversion::convertLoopOptions(LoopVectorizeAttr options) { options.getPredicateEnable()); convertBoolNode("llvm.loop.vectorize.scalable.enable", options.getScalableEnable()); + convertBoolNode("llvm.loop.vectorize.ivdep.enable", options.getIvdepEnable()); convertI32Node("llvm.loop.vectorize.width", options.getWidth()); convertFollowupNode("llvm.loop.vectorize.followup_vectorized", options.getFollowupVectorized()); diff --git a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll index 20431a7412bd1..b4fc36727bc84 100644 --- a/mlir/test/Target/LLVMIR/Import/metadata-loop.ll +++ b/mlir/test/Target/LLVMIR/Import/metadata-loop.ll @@ -73,7 +73,7 @@ end: ; // ----- ; CHECK-DAG: #[[FOLLOWUP:.*]] = #llvm.loop_annotation -; CHECK-DAG: #[[VECTORIZE_ATTR:.*]] = #llvm.loop_vectorize +; CHECK-DAG: #[[VECTORIZE_ATTR:.*]] = #llvm.loop_vectorize ; CHECK-DAG: #[[$ANNOT_ATTR:.*]] = #llvm.loop_annotation ; CHECK-LABEL: @vectorize @@ -85,17 +85,18 @@ end: ret void } -!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8} +!1 = distinct !{!1, !2, !3, !4, !5, !6, !7, !8, !9} !2 = !{!"llvm.loop.vectorize.enable", i1 1} !3 = !{!"llvm.loop.vectorize.predicate.enable", i1 1} !4 = !{!"llvm.loop.vectorize.scalable.enable", i1 0} -!5 = !{!"llvm.loop.vectorize.width", i32 16} -!6 = !{!"llvm.loop.vectorize.followup_vectorized", !9} -!7 = !{!"llvm.loop.vectorize.followup_epilogue", !9} -!8 = !{!"llvm.loop.vectorize.followup_all", !9} - -!9 = distinct !{!9, !10} -!10 = !{!"llvm.loop.disable_nonforced"} +!5 = !{!"llvm.loop.vectorize.ivdep.enable", i1 0} +!6 = !{!"llvm.loop.vectorize.width", i32 16} +!7 = !{!"llvm.loop.vectorize.followup_vectorized", !10} +!8 = !{!"llvm.loop.vectorize.followup_epilogue", !10} +!9 = !{!"llvm.loop.vectorize.followup_all", !10} + +!10 = distinct !{!10, !11} +!11 = !{!"llvm.loop.disable_nonforced"} ; // ----- diff --git a/mlir/test/Target/LLVMIR/loop-metadata.mlir b/mlir/test/Target/LLVMIR/loop-metadata.mlir index 2fe4a994aeb66..fa8d522b742dc 100644 --- a/mlir/test/Target/LLVMIR/loop-metadata.mlir +++ b/mlir/test/Target/LLVMIR/loop-metadata.mlir @@ -45,7 +45,7 @@ llvm.func @isvectorized() { llvm.func @vectorizeOptions() { // CHECK: br {{.*}} !llvm.loop ![[LOOP_NODE:[0-9]+]] llvm.br ^bb1 {loop_annotation = #llvm.loop_annotation >} ^bb1: @@ -54,10 +54,11 @@ llvm.func @vectorizeOptions() { // CHECK-DAG: ![[NON_FORCED:[0-9]+]] = !{!"llvm.loop.disable_nonforced"} // CHECK-DAG: ![[FOLLOWUP:[0-9]+]] = distinct !{![[FOLLOWUP]], ![[NON_FORCED]]} -// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} +// CHECK-DAG: ![[LOOP_NODE]] = distinct !{![[LOOP_NODE]], !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}, !{{[0-9]+}}} // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.enable", i1 true} // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.predicate.enable", i1 true} // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.scalable.enable", i1 false} +// CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.ivdep.enable", i1 false} // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.width", i32 16} // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_vectorized", ![[FOLLOWUP]]} // CHECK-DAG: !{{[0-9]+}} = !{!"llvm.loop.vectorize.followup_epilogue", ![[FOLLOWUP]]} From 002922906284fa624f35a9f080ab54f01539c46c Mon Sep 17 00:00:00 2001 From: Jean-Didier Pailleux Date: Fri, 26 Sep 2025 10:37:43 +0200 Subject: [PATCH 2/2] [flang] Implement !DIR$ IVDEP directive --- flang/include/flang/Parser/dump-parse-tree.h | 1 + flang/include/flang/Parser/parse-tree.h | 4 +- flang/lib/Lower/Bridge.cpp | 37 ++++++++++++++----- flang/lib/Optimizer/Builder/HLFIRTools.cpp | 2 +- flang/lib/Parser/Fortran-parsers.cpp | 3 ++ flang/lib/Parser/unparse.cpp | 1 + .../lib/Semantics/canonicalize-directives.cpp | 7 +++- flang/lib/Semantics/resolve-names.cpp | 3 +- flang/test/Integration/ivdep.f90 | 16 ++++++++ flang/test/Lower/ivdep.f90 | 26 +++++++++++++ flang/test/Parser/compiler-directives.f90 | 7 ++++ 11 files changed, 93 insertions(+), 14 deletions(-) create mode 100644 flang/test/Integration/ivdep.f90 create mode 100644 flang/test/Lower/ivdep.f90 diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h index b2341226c7688..47f62c6129662 100644 --- a/flang/include/flang/Parser/dump-parse-tree.h +++ b/flang/include/flang/Parser/dump-parse-tree.h @@ -206,6 +206,7 @@ class ParseTreeDumper { NODE(parser, CompilerDirective) NODE(CompilerDirective, AssumeAligned) NODE(CompilerDirective, IgnoreTKR) + NODE(CompilerDirective, IVDep) NODE(CompilerDirective, LoopCount) NODE(CompilerDirective, NameValue) NODE(CompilerDirective, Unrecognized) diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h index 39dbeb5e7cfbe..4e136a29efee9 100644 --- a/flang/include/flang/Parser/parse-tree.h +++ b/flang/include/flang/Parser/parse-tree.h @@ -3354,6 +3354,7 @@ struct StmtFunctionStmt { // !DIR$ NOVECTOR // !DIR$ NOUNROLL // !DIR$ NOUNROLL_AND_JAM +// !DIR$ IVDEP // !DIR$ struct CompilerDirective { UNION_CLASS_BOILERPLATE(CompilerDirective); @@ -3382,11 +3383,12 @@ struct CompilerDirective { EMPTY_CLASS(NoVector); EMPTY_CLASS(NoUnroll); EMPTY_CLASS(NoUnrollAndJam); + EMPTY_CLASS(IVDep); EMPTY_CLASS(Unrecognized); CharBlock source; std::variant, LoopCount, std::list, VectorAlways, std::list, Unroll, UnrollAndJam, Unrecognized, - NoVector, NoUnroll, NoUnrollAndJam> + NoVector, NoUnroll, NoUnrollAndJam, IVDep> u; }; diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp index 4a5b9885bb7c4..70c56758e025d 100644 --- a/flang/lib/Lower/Bridge.cpp +++ b/flang/lib/Lower/Bridge.cpp @@ -2428,22 +2428,33 @@ class FirConverter : public Fortran::lower::AbstractConverter { {}, {}, {}, {}); } + // Enabling loop vectorization attribute. + mlir::LLVM::LoopVectorizeAttr + genLoopVectorizeAttr(mlir::BoolAttr disableAttr, + mlir::BoolAttr ivdepEnableAttr) { + mlir::LLVM::LoopVectorizeAttr va; + if (disableAttr || ivdepEnableAttr) + va = mlir::LLVM::LoopVectorizeAttr::get(builder->getContext(), + /*disable=*/disableAttr, {}, {}, + /*ivdepEnable*/ ivdepEnableAttr, + {}, {}, {}, {}); + return va; + } + void addLoopAnnotationAttr( IncrementLoopInfo &info, llvm::SmallVectorImpl &dirs) { - mlir::LLVM::LoopVectorizeAttr va; + mlir::BoolAttr disableVecAttr, ivdepEnableAttr; mlir::LLVM::LoopUnrollAttr ua; mlir::LLVM::LoopUnrollAndJamAttr uja; + llvm::SmallVector aga; bool has_attrs = false; for (const auto *dir : dirs) { Fortran::common::visit( Fortran::common::visitors{ [&](const Fortran::parser::CompilerDirective::VectorAlways &) { - mlir::BoolAttr falseAttr = + disableVecAttr = mlir::BoolAttr::get(builder->getContext(), false); - va = mlir::LLVM::LoopVectorizeAttr::get(builder->getContext(), - /*disable=*/falseAttr, - {}, {}, {}, {}, {}, {}); has_attrs = true; }, [&](const Fortran::parser::CompilerDirective::Unroll &u) { @@ -2455,11 +2466,8 @@ class FirConverter : public Fortran::lower::AbstractConverter { has_attrs = true; }, [&](const Fortran::parser::CompilerDirective::NoVector &u) { - mlir::BoolAttr trueAttr = + disableVecAttr = mlir::BoolAttr::get(builder->getContext(), true); - va = mlir::LLVM::LoopVectorizeAttr::get(builder->getContext(), - /*disable=*/trueAttr, - {}, {}, {}, {}, {}, {}); has_attrs = true; }, [&](const Fortran::parser::CompilerDirective::NoUnroll &u) { @@ -2470,10 +2478,16 @@ class FirConverter : public Fortran::lower::AbstractConverter { uja = genLoopUnrollAndJamAttr(/*unrollingFactor=*/0); has_attrs = true; }, - + [&](const Fortran::parser::CompilerDirective::IVDep &iv) { + ivdepEnableAttr = + mlir::BoolAttr::get(builder->getContext(), true); + has_attrs = true; + }, [&](const auto &) {}}, dir->u); } + mlir::LLVM::LoopVectorizeAttr va = + genLoopVectorizeAttr(disableVecAttr, ivdepEnableAttr); mlir::LLVM::LoopAnnotationAttr la = mlir::LLVM::LoopAnnotationAttr::get( builder->getContext(), {}, /*vectorize=*/va, {}, /*unroll*/ ua, /*unroll_and_jam*/ uja, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}); @@ -3178,6 +3192,9 @@ class FirConverter : public Fortran::lower::AbstractConverter { [&](const Fortran::parser::CompilerDirective::NoUnrollAndJam &) { attachDirectiveToLoop(dir, &eval); }, + [&](const Fortran::parser::CompilerDirective::IVDep &) { + attachDirectiveToLoop(dir, &eval); + }, [&](const auto &) {}}, dir.u); } diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp index f93eaf7ba90b4..4c6aa37f11ade 100644 --- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp +++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp @@ -983,7 +983,7 @@ hlfir::LoopNest hlfir::genLoopNest(mlir::Location loc, if (!couldVectorize) { mlir::LLVM::LoopVectorizeAttr va{mlir::LLVM::LoopVectorizeAttr::get( builder.getContext(), - /*disable=*/builder.getBoolAttr(true), {}, {}, {}, {}, {}, {})}; + /*disable=*/builder.getBoolAttr(true), {}, {}, {}, {}, {}, {}, {})}; mlir::LLVM::LoopAnnotationAttr la = mlir::LLVM::LoopAnnotationAttr::get( builder.getContext(), {}, /*vectorize=*/va, {}, /*unroll*/ {}, /*unroll_and_jam*/ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}); diff --git a/flang/lib/Parser/Fortran-parsers.cpp b/flang/lib/Parser/Fortran-parsers.cpp index fbe629ab52935..bd90d2db6f50f 100644 --- a/flang/lib/Parser/Fortran-parsers.cpp +++ b/flang/lib/Parser/Fortran-parsers.cpp @@ -1294,6 +1294,7 @@ TYPE_PARSER(construct("STAT =" >> statVariable) || // !DIR$ LOOP COUNT (n1[, n2]...) // !DIR$ name[=value] [, name[=value]]... // !DIR$ UNROLL [n] +// !DIR$ IVDEP // !DIR$ constexpr auto ignore_tkr{ "IGNORE_TKR" >> optionalList(construct( @@ -1314,6 +1315,7 @@ constexpr auto novector{"NOVECTOR" >> construct()}; constexpr auto nounroll{"NOUNROLL" >> construct()}; constexpr auto nounrollAndJam{ "NOUNROLL_AND_JAM" >> construct()}; +constexpr auto ivdep{"IVDEP" >> construct()}; TYPE_PARSER(beginDirective >> "DIR$ "_tok >> sourced((construct(ignore_tkr) || construct(loopCount) || @@ -1324,6 +1326,7 @@ TYPE_PARSER(beginDirective >> "DIR$ "_tok >> construct(novector) || construct(nounrollAndJam) || construct(nounroll) || + construct(ivdep) || construct( many(construct( name, maybe(("="_tok || ":"_tok) >> digitString64))))) / diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp index 3455b535ccb51..55bb40c2bdea4 100644 --- a/flang/lib/Parser/unparse.cpp +++ b/flang/lib/Parser/unparse.cpp @@ -1867,6 +1867,7 @@ class UnparseVisitor { [&](const CompilerDirective::NoUnrollAndJam &) { Word("!DIR$ NOUNROLL_AND_JAM"); }, + [&](const CompilerDirective::IVDep &) { Word("!DIR$ IVDEP"); }, [&](const CompilerDirective::Unrecognized &) { Word("!DIR$ "); Word(x.source.ToString()); diff --git a/flang/lib/Semantics/canonicalize-directives.cpp b/flang/lib/Semantics/canonicalize-directives.cpp index 104df253ab642..7fb6f0424db49 100644 --- a/flang/lib/Semantics/canonicalize-directives.cpp +++ b/flang/lib/Semantics/canonicalize-directives.cpp @@ -60,7 +60,9 @@ static bool IsExecutionDirective(const parser::CompilerDirective &dir) { std::holds_alternative(dir.u) || std::holds_alternative(dir.u) || std::holds_alternative(dir.u) || - std::holds_alternative(dir.u); + std::holds_alternative( + dir.u) || + std::holds_alternative(dir.u); } void CanonicalizationOfDirectives::Post(parser::SpecificationPart &spec) { @@ -131,6 +133,9 @@ void CanonicalizationOfDirectives::Post(parser::Block &block) { [&](parser::CompilerDirective::NoUnrollAndJam &) { CheckLoopDirective(*dir, block, it); }, + [&](parser::CompilerDirective::IVDep &) { + CheckLoopDirective(*dir, block, it); + }, [&](auto &) {}}, dir->u); } diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp index d4e3b9cd4e00f..a1dd1fffb533e 100644 --- a/flang/lib/Semantics/resolve-names.cpp +++ b/flang/lib/Semantics/resolve-names.cpp @@ -9957,7 +9957,8 @@ void ResolveNamesVisitor::Post(const parser::CompilerDirective &x) { std::holds_alternative(x.u) || std::holds_alternative(x.u) || std::holds_alternative(x.u) || - std::holds_alternative(x.u)) { + std::holds_alternative(x.u) || + std::holds_alternative(x.u)) { return; } if (const auto *tkr{ diff --git a/flang/test/Integration/ivdep.f90 b/flang/test/Integration/ivdep.f90 new file mode 100644 index 0000000000000..c1ba963f4583e --- /dev/null +++ b/flang/test/Integration/ivdep.f90 @@ -0,0 +1,16 @@ +! RUN: %flang_fc1 -emit-llvm -o - %s | FileCheck %s + +! CHECK-LABEL: ivdep +subroutine ivdep + integer :: a(10) + !dir$ ivdep + ! CHECK: br i1 {{.*}}, label {{.*}}, label {{.*}} + ! CHECK-NOT: !llvm.loop + ! CHECK: br label {{.*}}, !llvm.loop ![[ANNOTATION:.*]] + do i=1,10 + a(i)=i + end do +end subroutine ivdep + +! CHECK: ![[ANNOTATION]] = distinct !{![[ANNOTATION]], ![[IVDEP:.*]]} +! CHECK: ![[IVDEP]] = !{!"llvm.loop.vectorize.ivdep.enable", i1 true} diff --git a/flang/test/Lower/ivdep.f90 b/flang/test/Lower/ivdep.f90 new file mode 100644 index 0000000000000..7070ed7703e74 --- /dev/null +++ b/flang/test/Lower/ivdep.f90 @@ -0,0 +1,26 @@ +! RUN: %flang_fc1 -emit-hlfir -o - %s | FileCheck %s + +! CHECK: #loop_vectorize = #llvm.loop_vectorize +! CHECK: #loop_annotation = #llvm.loop_annotation + +! CHECK-LABEL: @_QPivdep +subroutine ivdep + integer :: a(10) + !dir$ ivdep + !CHECK: fir.do_loop {{.*}} attributes {loopAnnotation = #loop_annotation} + do i=1,10 + a(i)=i + end do +end subroutine ivdep + + +! CHECK-LABEL: @_QPintermediate_directive +subroutine intermediate_directive + integer :: a(10) + !dir$ ivdep + !dir$ unknown + !CHECK: fir.do_loop {{.*}} attributes {loopAnnotation = #loop_annotation} + do i=1,10 + a(i)=i + end do +end subroutine intermediate_directive diff --git a/flang/test/Parser/compiler-directives.f90 b/flang/test/Parser/compiler-directives.f90 index 04d22ff0fd8ee..e904319c472a2 100644 --- a/flang/test/Parser/compiler-directives.f90 +++ b/flang/test/Parser/compiler-directives.f90 @@ -72,3 +72,10 @@ subroutine no_vector do i=1,10 enddo end subroutine + +subroutine ivdep + !dir$ ivdep + ! CHECK: !DIR$ IVDEP + do i=1,10 + enddo +end subroutine