[flang-rt] Fix recently added memcpy that need runtime wrappers (llvm#3219)

Author: ronlieb

flang-rt/include/flang-rt/runtime/descriptor.h

@@ -35,9 +35,9 @@
 RT_OFFLOAD_VAR_GROUP_BEGIN
 /// Value used for asyncId when no specific stream is specified.
 static constexpr std::int64_t kNoAsyncId = -1;
-RT_OFFLOAD_VAR_GROUP_END
 /// Value used for asyncObject when no specific stream is specified.
 static constexpr std::int64_t *kNoAsyncObject = nullptr;
+RT_OFFLOAD_VAR_GROUP_END
 
 namespace Fortran::runtime {
 

flang-rt/include/flang-rt/runtime/work-queue.h

@@ -127,8 +127,8 @@ class Elementwise {
   const Descriptor &instance_, *from_{nullptr};
   std::size_t elements_{instance_.InlineElements()};
   std::size_t elementAt_{0};
-  SubscriptValue subscripts_[common::maxRank];
-  SubscriptValue fromSubscripts_[common::maxRank];
+  SubscriptValue subscripts_[maxRank];
+  SubscriptValue fromSubscripts_[maxRank];
 };
 
 // Base class for ticket workers that operate over derived type components.
@@ -162,7 +162,7 @@ class Componentwise {
   const typeInfo::DerivedType &derived_;
   std::size_t components_{0}, componentAt_{0};
   const typeInfo::Component *component_{nullptr};
-  StaticDescriptor<common::maxRank, true, 0> componentDescriptor_;
+  StaticDescriptor<maxRank, true, 0> componentDescriptor_;
 
 private:
   RT_API_ATTRS void GetFirstComponent() {
@@ -275,7 +275,7 @@ class InitializeCloneTicket
   const Descriptor &clone_;
   bool hasStat_{false};
   const Descriptor *errMsg_{nullptr};
-  StaticDescriptor<common::maxRank, true, 0> cloneComponentDescriptor_;
+  StaticDescriptor<maxRank, true, 0> cloneComponentDescriptor_;
 };
 
 // Implements derived type instance finalization
@@ -331,7 +331,7 @@ class AssignTicket : public ImmediateTicketRunner<AssignTicket> {
   const Descriptor *from_{nullptr};
   int flags_{0}; // enum AssignFlags
   MemmoveFct memmoveFct_{nullptr};
-  StaticDescriptor<common::maxRank, true, 0> tempDescriptor_;
+  StaticDescriptor<maxRank, true, 0> tempDescriptor_;
   const typeInfo::DerivedType *declaredType_{nullptr};
   const typeInfo::DerivedType *toDerived_{nullptr};
   Descriptor *toDeallocate_{nullptr};
@@ -364,7 +364,7 @@ class DerivedAssignTicket
   int flags_{0};
   MemmoveFct memmoveFct_{nullptr};
   Descriptor *deallocateAfter_{nullptr};
-  StaticDescriptor<common::maxRank, true, 0> fromComponentDescriptor_;
+  StaticDescriptor<maxRank, true, 0> fromComponentDescriptor_;
 };
 
 namespace io::descr {
@@ -392,7 +392,7 @@ class DescriptorIoTicket
   common::optional<typeInfo::SpecialBinding> nonTbpSpecial_;
   const typeInfo::DerivedType *derived_{nullptr};
   const typeInfo::SpecialBinding *special_{nullptr};
-  StaticDescriptor<common::maxRank, true, 0> elementDescriptor_;
+  StaticDescriptor<maxRank, true, 0> elementDescriptor_;
 };
 
 template <io::Direction DIR>

flang-rt/lib/runtime/assign.cpp

@@ -279,7 +279,7 @@ RT_API_ATTRS int AssignTicket::Begin(WorkQueue &workQueue) {
     if (mustDeallocateLHS) {
       // Convert the LHS into a temporary, then make it look deallocated.
       toDeallocate_ = &tempDescriptor_.descriptor();
-      std::memcpy(
+      Fortran::runtime::memcpy(
           reinterpret_cast<void *>(toDeallocate_), &to_, to_.SizeInBytes());
       to_.set_base_addr(nullptr);
       if (toDerived_ && (flags_ & NeedFinalization)) {
@@ -298,7 +298,7 @@ RT_API_ATTRS int AssignTicket::Begin(WorkQueue &workQueue) {
       auto descBytes{from_->SizeInBytes()};
       Descriptor &newFrom{tempDescriptor_.descriptor()};
       persist_ = true; // tempDescriptor_ state must outlive child tickets
-      std::memcpy(reinterpret_cast<void *>(&newFrom), from_, descBytes);
+      Fortran::runtime::memcpy(reinterpret_cast<void *>(&newFrom), from_, descBytes);
       // Pretend the temporary descriptor is for an ALLOCATABLE
       // entity, otherwise, the Deallocate() below will not
       // free the descriptor memory.

flang-rt/lib/runtime/derived.cpp

@@ -71,7 +71,7 @@ RT_API_ATTRS int InitializeTicket::Continue(WorkQueue &workQueue) {
       // Explicit initialization of data pointers and
       // non-allocatable non-automatic components
       std::size_t bytes{component_->SizeInBytes(instance_)};
-      std::memcpy(rawComponent, init, bytes);
+      Fortran::runtime::memcpy(rawComponent, init, bytes);
     } else if (component_->genre() == typeInfo::Component::Genre::Pointer) {
       // Data pointers without explicit initialization are established
       // so that they are valid right-hand side targets of pointer
@@ -108,20 +108,20 @@ RT_API_ATTRS int InitializeTicket::Continue(WorkQueue &workQueue) {
             chunk = done;
           }
           char *uninitialized{rawInstance + done * *stride};
-          std::memcpy(uninitialized, rawInstance, chunk * *stride);
+          Fortran::runtime::memcpy(uninitialized, rawInstance, chunk * *stride);
           done += chunk;
         }
       } else {
         for (std::size_t done{1}; done < elements_; ++done) {
           char *uninitialized{rawInstance + done * *stride};
-          std::memcpy(uninitialized, rawInstance, elementBytes);
+          Fortran::runtime::memcpy(uninitialized, rawInstance, elementBytes);
         }
       }
     } else { // one at a time with subscription
       for (Elementwise::Advance(); !Elementwise::IsComplete();
           Elementwise::Advance()) {
         char *element{instance_.Element<char>(subscripts_)};
-        std::memcpy(element, rawInstance, elementBytes);
+        Fortran::runtime::memcpy(element, rawInstance, elementBytes);
       }
     }
   }

flang-rt/lib/runtime/unit.h

@@ -21,7 +21,9 @@
 #include "flang-rt/runtime/lock.h"
 #include "flang-rt/runtime/memory.h"
 #include "flang-rt/runtime/terminator.h"
+RT_OFFLOAD_VAR_GROUP_BEGIN
 #include "flang/Common/constexpr-bitset.h"
+RT_OFFLOAD_VAR_GROUP_END
 #include "flang/Common/optional.h"
 #include <cstdlib>
 #include <cstring>

flang-rt/lib/runtime/work-queue.cpp

@@ -14,7 +14,7 @@
 
 namespace Fortran::runtime {
 
-#if !defined(RT_DEVICE_COMPILATION)
+#if !defined(RT_DEVICE_COMPILATION) && !defined(OMP_OFFLOAD_BUILD)
 // FLANG_RT_DEBUG code is disabled when false.
 static constexpr bool enableDebugOutput{false};
 #endif
@@ -79,7 +79,7 @@ RT_API_ATTRS Ticket &WorkQueue::StartTicket() {
     last_ = newTicket;
   }
   newTicket->ticket.begun = false;
-#if !defined(RT_DEVICE_COMPILATION)
+#if !defined(RT_DEVICE_COMPILATION) && !defined(OMP_OFFLOAD_BUILD)
   if (enableDebugOutput &&
       (executionEnvironment.internalDebugging &
           ExecutionEnvironment::WorkQueue)) {
@@ -93,7 +93,7 @@ RT_API_ATTRS int WorkQueue::Run() {
   while (last_) {
     TicketList *at{last_};
     insertAfter_ = last_;
-#if !defined(RT_DEVICE_COMPILATION)
+#if !defined(RT_DEVICE_COMPILATION) && !defined(OMP_OFFLOAD_BUILD)
     if (enableDebugOutput &&
         (executionEnvironment.internalDebugging &
             ExecutionEnvironment::WorkQueue)) {
@@ -102,7 +102,7 @@ RT_API_ATTRS int WorkQueue::Run() {
     }
 #endif
     int stat{at->ticket.Continue(*this)};
-#if !defined(RT_DEVICE_COMPILATION)
+#if !defined(RT_DEVICE_COMPILATION) && !defined(OMP_OFFLOAD_BUILD)
     if (enableDebugOutput &&
         (executionEnvironment.internalDebugging &
             ExecutionEnvironment::WorkQueue)) {