Merge Bandicoot tests with other tests (#438)

* Normalize step size to the batch size used.

* Update history.

* Don't use if constexpr.

* Fix all step sizes in tests to account for new behavior.

* Retune DemonSGD tests to reduce failure probability.

* Merge Bandicoot tests into existing TEMPLATE_TEST_CASEs.

* Fix merge artifact.

* Fix compilation errors.

* Fix MOEAD test to be more robust.

* Oops, fix missing colon.

* Fix compilation issues.

* Remove arma:: for randu call.

* Try to fix AppVeyor build.

* Bump version number to fix mlpack integration build.

* Adapt FBS/FISTA/FASTA to bandicoot tests.

* Keep bandicoot implementation separate.

* Print which tests take a long time so I can prune down the runtime.

* Remove some tests that take a long long time.

* Filter out some more long-running GPU tests.

Author: rcurtin

.github/workflows/bandicoot-test.yml

@@ -51,4 +51,4 @@ jobs:
     - name: Test ensmallen
       run: |
           cd build/
-          ./ensmallen_tests
+          ./ensmallen_tests -d yes

include/ensmallen_bits/fasta/fasta.hpp

@@ -102,7 +102,7 @@ class FASTA
    */
   template<typename FunctionType, typename MatType, typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(FunctionType& function,
            MatType& iterate,

include/ensmallen_bits/fasta/fasta_impl.hpp

@@ -90,7 +90,7 @@ FASTA<BackwardStepType>::FASTA(BackwardStepType backwardStep,
 template<typename BackwardStepType>
 template<typename FunctionType, typename MatType, typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
+typename std::enable_if<IsMatrixType<GradType>::value,
     typename MatType::elem_type>::type
 FASTA<BackwardStepType>::Optimize(FunctionType& function,
                                   MatType& iterateIn,
@@ -358,7 +358,7 @@ FASTA<BackwardStepType>::Optimize(FunctionType& function,
     // proximal step.
 
     // Compute residual.  This is Eq. (40) in the paper.
-    const ElemType residual = arma::norm(g + (xHat - x) / currentStepSize, 2);
+    const ElemType residual = norm(g + (xHat - x) / currentStepSize, 2);
 
     // If this is the first iteration, store the residual as the first residual.
     if (i == 1)

include/ensmallen_bits/fbs/fbs.hpp

@@ -92,7 +92,7 @@ class FBS
    */
   template<typename FunctionType, typename MatType, typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(FunctionType& function,
            MatType& iterate,

include/ensmallen_bits/fbs/fbs_impl.hpp

@@ -44,7 +44,7 @@ FBS<BackwardStepType>::FBS(BackwardStepType backwardStep,
 template<typename BackwardStepType>
 template<typename FunctionType, typename MatType, typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
+typename std::enable_if<IsMatrixType<GradType>::value,
     typename MatType::elem_type>::type
 FBS<BackwardStepType>::Optimize(FunctionType& function,
                                 MatType& iterateIn,

include/ensmallen_bits/fbs/l1_constraint_impl.hpp

@@ -134,14 +134,19 @@ void L1Constraint::ProximalStep(MatType& coordinates,
   }
 
   const eT theta = (s - eT(lambda)) / rho;
-  coordinates.transform(
-      [theta](eT val)
-      {
-        if (val > 0)
-          return std::max(val - theta, eT(0));
-        else
-          return std::min(val + theta, eT(0));
-      });
+  // This is a single-line implementation of the .transform() below; we use the
+  // single-line implementation so it works with Bandicoot.
+  //
+  // coordinates.transform(
+  //     [theta](eT val)
+  //     {
+  //       if (val > 0)
+  //         return std::max(val - theta, eT(0));
+  //       else
+  //         return std::min(val + theta, eT(0));
+  //     });
+  coordinates = sign(coordinates) % clamp(
+      abs(coordinates) - theta, eT(0), std::numeric_limits<eT>::max());
 
   // Sanity check: ensure we actually ended up inside the L1 ball.  This might
   // not happen due to floating-point inaccuracies.  If so, try again.
@@ -168,7 +173,8 @@ template<typename MatType>
 inline arma::Col<typename MatType::elem_type> L1Constraint::ExtractNonzeros(
     const MatType& coordinates) const
 {
-  return arma::Col<typename MatType::elem_type>(vectorise(abs(coordinates)));
+  typedef typename MatType::elem_type ElemType;
+  return conv_to<arma::Col<ElemType>>::from(vectorise(abs(coordinates)));
 }
 
 template<typename eT>

include/ensmallen_bits/fbs/l1_penalty_impl.hpp

@@ -38,9 +38,24 @@ void L1Penalty::ProximalStep(MatType& coordinates,
   // Apply the backwards step coordinate-wise.  If `MatType` is sparse, this
   // only applies to nonzero elements, which is just fine.
   typedef typename MatType::elem_type eT;
-  coordinates.transform([this, stepSize](eT val) { return (val > eT(0)) ?
-      (std::max(eT(0), val - eT(lambda * stepSize))) :
-      (std::min(eT(0), val + eT(lambda * stepSize))); });
+
+  // This is equivalent to the following .transform() implementation (which is
+  // easier to read but will not work with Bandicoot):
+  //
+  //arma::Mat<typename MatType::elem_type> c2 = conv_to<arma::Mat<typename MatType::elem_type>>::from(coordinates);
+  //c2.transform([this, stepSize](eT val) { return (val > eT(0)) ?
+  //    (std::max(eT(0), val - eT(lambda * stepSize))) :
+  //    (std::min(eT(0), val + eT(lambda * stepSize))); });
+  // coordinates.transform([this, stepSize](eT val) { return (val > eT(0)) ?
+  //     (std::max(eT(0), val - eT(lambda * stepSize))) :
+  //     (std::min(eT(0), val + eT(lambda * stepSize))); });
+  //
+  coordinates = sign(coordinates) % clamp(
+      abs(coordinates) - eT(lambda * stepSize), eT(0),
+      std::numeric_limits<eT>::max());
+
+  //coordinates.print("coordinates");
+  //c2.print("c2");
 }
 
 } // namespace ens

include/ensmallen_bits/fista/fista.hpp

@@ -104,7 +104,7 @@ class FISTA
    */
   template<typename FunctionType, typename MatType, typename GradType,
            typename... CallbackTypes>
-  typename std::enable_if<IsArmaType<GradType>::value,
+  typename std::enable_if<IsMatrixType<GradType>::value,
       typename MatType::elem_type>::type
   Optimize(FunctionType& function,
            MatType& iterate,

include/ensmallen_bits/fista/fista_impl.hpp

@@ -74,7 +74,7 @@ FISTA<BackwardStepType>::FISTA(BackwardStepType backwardStep,
 template<typename BackwardStepType>
 template<typename FunctionType, typename MatType, typename GradType,
          typename... CallbackTypes>
-typename std::enable_if<IsArmaType<GradType>::value,
+typename std::enable_if<IsMatrixType<GradType>::value,
     typename MatType::elem_type>::type
 FISTA<BackwardStepType>::Optimize(FunctionType& function,
                                   MatType& iterateIn,

tests/ada_belief_test.cpp

@@ -20,19 +20,8 @@ using namespace ens;
 using namespace ens::test;
 
 TEMPLATE_TEST_CASE("AdaBelief_LogisticRegressionFunction", "[AdaBelief]",
-    ENS_ALL_TEST_TYPES)
+    ENS_ALL_CPU_TEST_TYPES)
 {
   AdaBelief adaBelief(0.032);
-  LogisticRegressionFunctionTest<TestType, arma::Row<size_t>>(adaBelief);
+  LogisticRegressionFunctionTest<TestType>(adaBelief);
 }
-
-#ifdef ENS_HAVE_COOT
-
-TEMPLATE_TEST_CASE("AdaBelief_LogisticRegressionFunction", "[AdaBelief]",
-    coot::mat, coot::fmat)
-{
-  AdaBelief adaBelief(0.032);
-  LogisticRegressionFunctionTest<TestType, coot::Row<size_t>>(adaBelief);
-}
-
-#endif