SUNDIALS: Fix N_VInvTest_MultiFab (#5236)

The operation should be done on all components.

This also fixes another issue. N_VInvTest should return MPI-reduced
result, not local result. If we want local reduction result, that should
be N_InvTestLocal.

Close #5039.

Author: WeiqunZhang

Src/Extern/SUNDIALS/AMReX_NVector_MultiFab.cpp

@@ -532,26 +532,25 @@ int N_VInvTest_MultiFab(N_Vector x, N_Vector z)
     auto const& ma1 = mf_x->const_arrays();
     auto const& ma2 = mf_z->arrays();
 
-    GpuTuple<bool> mm = ParReduce(TypeList<ReduceOpLogicalAnd>{},
-                                    TypeList<bool>{},
-                                    *mf_x,  amrex::IntVect::TheZeroVector(),
-     [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept
-           -> GpuTuple<bool>
-     {
-         bool result = !(ma1[box_no](i,j,k) == amrex::Real(0.0));
-         ma2[box_no](i,j,k) = result ? amrex::Real(1.0) / ma1[box_no](i,j,k) : 0.0;
-         return { result };
-     });
+    bool val = ParReduce(TypeList<ReduceOpLogicalAnd>{},
+                         TypeList<int>{}, // We use int for logical and on device.
+                         *mf_x, amrex::IntVect::TheZeroVector(),
+                         mf_x->nComp(),
+    [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
+                        -> GpuTuple<int>
+    {
+        auto xx = ma1[box_no](i,j,k,n);
+        bool result = xx != Real(0.0);
+        ma2[box_no](i,j,k,n) = result ? Real(1.0)/xx : Real(0.0);
+        return { static_cast<int>(result) };
+    });
 
-    bool val = amrex::get<0>(mm);
+    ParallelAllReduce::And(val, ParallelContext::CommunicatorSub());
 
-    if (val == false)
-    {
-         return SUNFALSE;
-    }
-    else
-    {
-         return SUNTRUE;
+    if (val) {
+        return SUNTRUE;
+    } else {
+        return SUNFALSE;
     }
 }