move unsafe code out of destructors

Author: adurang

offload/plugins-nextgen/level_zero/include/L0Context.h

@@ -79,13 +79,16 @@ class L0ContextTy {
   L0ContextTy &operator=(const L0ContextTy &&) = delete;
 
   /// Release resources
-  ~L0ContextTy() {
+  ~L0ContextTy() {}
+
+  Error deinit() {
     EventPool.deinit();
     auto Err = HostMemAllocator.deinit();
     if (Err)
-      consumeError(std::move(Err));
+      return Err;
     if (zeContext)
-      CALL_ZE_RET_VOID(zeContextDestroy, zeContext);
+      CALL_ZE_RET_ERROR(zeContextDestroy, zeContext);
+    return Error::success();
   }
 
   auto &getPlugin() const { return Plugin; }

offload/plugins-nextgen/level_zero/include/L0Device.h

@@ -306,11 +306,7 @@ class L0DeviceTy final : public GenericDeviceTy {
 
   Error setContext() override { return Plugin::success(); }
   Error initImpl(GenericPluginTy &Plugin) override;
-  Error deinitImpl() override {
-    Programs.clear();
-    return MemAllocator.deinit();
-  }
-
+  Error deinitImpl() override;
   auto getZeDevice() const { return zeDevice; }
 
   const L0ContextTy &getL0Context() const { return l0Context; }

offload/plugins-nextgen/level_zero/include/L0Kernel.h

@@ -132,10 +132,7 @@ class L0KernelTy : public GenericKernelTy {
 public:
   /// Create a L0 kernel with a name and an execution mode.
   L0KernelTy(const char *Name) : GenericKernelTy(Name), zeKernel(nullptr) {}
-  ~L0KernelTy() {
-    if (zeKernel)
-      CALL_ZE_RET_VOID(zeKernelDestroy, zeKernel);
-  }
+  ~L0KernelTy() {}
   L0KernelTy(const L0KernelTy &) = delete;
   L0KernelTy(L0KernelTy &&) = delete;
   L0KernelTy &operator=(const L0KernelTy &) = delete;
@@ -150,6 +147,10 @@ class L0KernelTy : public GenericKernelTy {
                    uint32_t NumBlocks[3], KernelArgsTy &KernelArgs,
                    KernelLaunchParamsTy LaunchParams,
                    AsyncInfoWrapperTy &AsyncInfoWrapper) const override;
+  Error deinit() {
+    CALL_ZE_RET_ERROR(zeKernelDestroy, zeKernel);
+    return Plugin::success();
+  }
 
   Expected<uint64_t> maxGroupSize(GenericDeviceTy &GenericDevice,
                                   uint64_t DynamicMemSize) const override {

offload/plugins-nextgen/level_zero/include/L0Memory.h

@@ -218,10 +218,11 @@ class MemAllocatorTy {
     MemPoolTy(MemPoolTy &&) = delete;
     MemPoolTy &operator=(const MemPoolTy &) = delete;
     MemPoolTy &operator=(const MemPoolTy &&) = delete;
+    ~MemPoolTy() {}
 
     void printUsage();
     /// Release resources used in the pool.
-    ~MemPoolTy();
+    Error deinit();
 
     /// Allocate the requested size of memory from this pool.
     /// AllocSize is the chunk size internally used for the returned memory.
@@ -313,7 +314,6 @@ class MemAllocatorTy {
   MemAllocatorTy(MemAllocatorTy &&) = delete;
   MemAllocatorTy &operator=(const MemAllocatorTy &) = delete;
   MemAllocatorTy &operator=(const MemAllocatorTy &&) = delete;
-
   ~MemAllocatorTy() {}
 
   /// Release resources and report statistics if requested
@@ -517,10 +517,7 @@ class StagingBufferTy {
   StagingBufferTy &operator=(const StagingBufferTy &) = delete;
   StagingBufferTy &operator=(const StagingBufferTy &&) = delete;
 
-  ~StagingBufferTy() {
-    if (initialized())
-      clear();
-  }
+  ~StagingBufferTy() { }
 
   void clear() {
     ze_result_t Rc;

offload/plugins-nextgen/level_zero/include/L0Program.h

@@ -75,14 +75,15 @@ class L0ProgramTy : public DeviceImageTy {
   L0ProgramTy(int32_t ImageId, GenericDeviceTy &Device,
               std::unique_ptr<MemoryBuffer> Image)
       : DeviceImageTy(ImageId, Device, std::move(Image)) {}
-
-  ~L0ProgramTy();
+  ~L0ProgramTy() {}
 
   L0ProgramTy(const L0ProgramTy &other) = delete;
   L0ProgramTy(L0ProgramTy &&) = delete;
   L0ProgramTy &operator=(const L0ProgramTy &) = delete;
   L0ProgramTy &operator=(const L0ProgramTy &&) = delete;
 
+  Error deinit();
+
   static L0ProgramTy &makeL0Program(DeviceImageTy &Device) {
     return static_cast<L0ProgramTy &>(Device);
   }

offload/plugins-nextgen/level_zero/src/L0Device.cpp

@@ -223,6 +223,13 @@ Error L0DeviceTy::initImpl(GenericPluginTy &Plugin) {
   return Plugin::success();
 }
 
+Error L0DeviceTy::deinitImpl() {
+  for (auto &PGM : Programs)
+    if (auto Err = PGM.deinit())
+      return Err;
+  return MemAllocator.deinit();
+}
+
 int32_t L0DeviceTy::synchronize(__tgt_async_info *AsyncInfo,
                                 bool ReleaseQueue) {
   bool IsAsync = AsyncInfo && asyncEnabled();

offload/plugins-nextgen/level_zero/src/L0Memory.cpp

@@ -233,19 +233,20 @@ void MemAllocatorTy::MemPoolTy::printUsage() {
 }
 
 /// Release resources used in the pool.
-MemAllocatorTy::MemPoolTy::~MemPoolTy() {
+Error MemAllocatorTy::MemPoolTy::deinit() {
   const int DebugLevel = getDebugLevel();
   if (DebugLevel > 0)
     printUsage();
   for (auto &Bucket : Buckets) {
     for (auto *Block : Bucket) {
       if (DebugLevel > 0)
         Allocator->log(0, Block->Size, AllocKind);
-      CALL_ZE_RET_VOID(zeMemFree, Allocator->L0Context->getZeContext(),
-                       reinterpret_cast<void *>(Block->Base));
+      CALL_ZE_RET_ERROR(zeMemFree, Allocator->L0Context->getZeContext(),
+                        reinterpret_cast<void *>(Block->Base));
       delete Block;
     }
   }
+  return Plugin::success();
 }
 
 /// Allocate the requested size of memory from this pool.
@@ -419,8 +420,23 @@ Error MemAllocatorTy::deinit() {
     if (Err)
       return Err;
   }
-  ReductionPool.reset(nullptr);
-  CounterPool.reset(nullptr);
+  for (auto &Pool : Pools) {
+    if (Pool) {
+      if (auto Err = Pool->deinit())
+        return Err;
+      Pool.reset(nullptr);
+    }
+  }
+  if (ReductionPool) {
+    if (auto Err = ReductionPool->deinit())
+      return Err;
+    ReductionPool.reset(nullptr);
+  }
+  if (CounterPool) {
+    if (auto Err = CounterPool->deinit())
+      return Err;
+    CounterPool.reset(nullptr);
+  }
   // Report memory usage if requested
   if (getDebugLevel() > 0) {
     for (auto &Stat : Stats) {

offload/plugins-nextgen/level_zero/src/L0Plugin.cpp

@@ -149,6 +149,9 @@ Error LevelZeroPluginTy::deinitImpl() {
   ContextTLSTable.clear();
   DeviceTLSTable.clear();
   ThreadTLSTable.clear();
+  for (auto &Context : ContextList)
+    if (auto Err = Context.deinit())
+      return Err;
   ContextList.clear();
   DP("Level0 plugin deinitialized successfully\n");
   return Plugin::success();

offload/plugins-nextgen/level_zero/src/L0Program.cpp

@@ -47,16 +47,16 @@ inline L0DeviceTy &L0ProgramTy::getL0Device() const {
   return L0DeviceTy::makeL0Device(getDevice());
 }
 
-L0ProgramTy::~L0ProgramTy() {
+Error L0ProgramTy::deinit() {
   for (auto *Kernel : Kernels) {
-    // We need explicit destructor and deallocate calls to release the kernels
-    // created by `GenericDeviceTy::constructKernel()`.
-    Kernel->~L0KernelTy();
+    if (auto Err = Kernel->deinit())
+      return Err;
     getL0Device().getPlugin().free(Kernel);
   }
   for (auto Module : Modules) {
-    CALL_ZE_RET_VOID(zeModuleDestroy, Module);
+    CALL_ZE_RET_ERROR(zeModuleDestroy, Module);
   }
+  return Plugin::success();
 }
 
 void L0ProgramTy::setLibModule() {