Move release of level-1 scratch pad and fix broken copy semantics

Get rid of ParallelX<TeamPolicy,{Cuda,HIP}>::~ParallelX and free memory
in epilogue of ParallelX::execute().

Signed-off-by: Damien L-G <dalg24@gmail.com>

Author: dalg24

core/src/Cuda/Kokkos_Cuda_Parallel_Team.hpp

@@ -533,6 +533,12 @@ class ParallelFor<FunctorType, Kokkos::TeamPolicy<Properties...>,
         *this, grid, block, shmem_size_total,
         m_policy.space()
             .impl_internal_space_instance());  // copy to device and execute
+
+    if (m_scratch_pool_id >= 0) {
+      m_policy.space()
+          .impl_internal_space_instance()
+          ->release_team_scratch_space(m_scratch_pool_id);
+    }
   }
 
   ParallelFor(const FunctorType& arg_functor, const Policy& arg_policy)
@@ -606,14 +612,6 @@ class ParallelFor<FunctorType, Kokkos::TeamPolicy<Properties...>,
       Kokkos::Impl::throw_runtime_exception(error.str().c_str());
     }
   }
-
-  ~ParallelFor() {
-    if (m_scratch_pool_id >= 0) {
-      m_policy.space()
-          .impl_internal_space_instance()
-          ->release_team_scratch_space(m_scratch_pool_id);
-    }
-  }
 };
 
 template <class CombinedFunctorReducerType, class... Properties>
@@ -894,6 +892,12 @@ class ParallelReduce<CombinedFunctorReducerType,
         m_functor_reducer.get_reducer().init(m_result_ptr);
       }
     }
+
+    if (m_scratch_pool_id >= 0) {
+      m_policy.space()
+          .impl_internal_space_instance()
+          ->release_team_scratch_space(m_scratch_pool_id);
+    }
   }
 
   template <class ViewType>
@@ -1017,14 +1021,6 @@ class ParallelReduce<CombinedFunctorReducerType,
       Kokkos::Impl::throw_runtime_exception(error.str().c_str());
     }
   }
-
-  ~ParallelReduce() {
-    if (m_scratch_pool_id >= 0) {
-      m_policy.space()
-          .impl_internal_space_instance()
-          ->release_team_scratch_space(m_scratch_pool_id);
-    }
-  }
 };
 
 }  // namespace Impl

core/src/HIP/Kokkos_HIP_ParallelFor_Team.hpp

@@ -99,6 +99,12 @@ class ParallelFor<FunctorType, Kokkos::TeamPolicy<Properties...>, HIP> {
         *this, grid, block, shmem_size_total,
         m_policy.space().impl_internal_space_instance(),
         true);  // copy to device and execute
+
+    if (m_scratch_pool_id >= 0) {
+      m_policy.space()
+          .impl_internal_space_instance()
+          ->release_team_scratch_space(m_scratch_pool_id);
+    }
   }
 
   ParallelFor(FunctorType const& arg_functor, Policy const& arg_policy)
@@ -173,14 +179,6 @@ class ParallelFor<FunctorType, Kokkos::TeamPolicy<Properties...>, HIP> {
       Kokkos::Impl::throw_runtime_exception(error.str().c_str());
     }
   }
-
-  ~ParallelFor() {
-    if (m_scratch_pool_id >= 0) {
-      m_policy.space()
-          .impl_internal_space_instance()
-          ->release_team_scratch_space(m_scratch_pool_id);
-    }
-  }
 };
 
 }  // namespace Impl

core/src/HIP/Kokkos_HIP_ParallelReduce_Team.hpp

@@ -287,6 +287,12 @@ class ParallelReduce<CombinedFunctorReducerType,
         reducer.init(m_result_ptr);
       }
     }
+
+    if (m_scratch_pool_id >= 0) {
+      m_policy.space()
+          .impl_internal_space_instance()
+          ->release_team_scratch_space(m_scratch_pool_id);
+    }
   }
 
   template <class ViewType>
@@ -410,14 +416,6 @@ class ParallelReduce<CombinedFunctorReducerType,
       Kokkos::Impl::throw_runtime_exception(error.str().c_str());
     }
   }
-
-  ~ParallelReduce() {
-    if (m_scratch_pool_id >= 0) {
-      m_policy.space()
-          .impl_internal_space_instance()
-          ->release_team_scratch_space(m_scratch_pool_id);
-    }
-  }
 };
 
 }  // namespace Impl