Upgrade Ensmallen to 1.11.1 (#6)

coatless · web-flow · commit 0ba1a613214c · 2018-11-30T08:22:20.000-06:00
* Ensmallen 1.11.1 release...

* Update package documentation...

* Update news.

* Avoid including a CRAN-RELEASE file in package build.
diff --git a/.Rbuildignore b/.Rbuildignore
@@ -1,3 +1,4 @@
+^CRAN-RELEASE$
 ^.*\.Rproj$
 ^\.Rproj\.user$
 ^\.travis\.yml$
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: RcppEnsmallen
 Title: Header-Only C++ Mathematical Optimization Library for 'Armadillo'
-Version: 0.1.10.0.1
+Version: 0.1.11.1.1
 Authors@R: c(
     person("James Joseph", "Balamuta", email = "balamut2@illinois.edu", 
            role = c("aut", "cre", "cph"), 
diff --git a/NEWS.md b/NEWS.md
@@ -1,9 +1,18 @@
+# RcppEnsmallen 0.1.11.1.1
+
+- Upgraded to ensmallen release 1.11.1 "Jet Lag" (2018-11-28)
+    - Minor documentation fixes.
+    - Add WNGrad optimizer.
+    - Fix header name in documentation samples.
+- Added citation information
+
 # RcppEnsmallen 0.1.10.0.1
 
 ## New
 
 - Support for compiling via _R_ the mathematical optimizers in the _C++_
   [ensmallen](http://ensmallen.org/docs.html) library.
+- Includes ensmallen release 1.10.0 "Corporate Catabolism" (2018-10-19)
 
 ## Special Thanks
 
diff --git a/inst/include/ensmallen.hpp b/inst/include/ensmallen.hpp
@@ -61,7 +61,6 @@
 
 #include "ensmallen_bits/problems/problems.hpp" // TODO: should move to another place
 
-// TODO: remove mlpack bits from each of these files
 #include "ensmallen_bits/ada_delta/ada_delta.hpp"
 #include "ensmallen_bits/ada_grad/ada_grad.hpp"
 #include "ensmallen_bits/adam/adam.hpp"
@@ -74,7 +73,7 @@
 
 #include "ensmallen_bits/fw/frank_wolfe.hpp"
 #include "ensmallen_bits/gradient_descent/gradient_descent.hpp"
-// #include "ensmallen_bits/grid_search/grid_search.hpp"
+#include "ensmallen_bits/grid_search/grid_search.hpp"
 #include "ensmallen_bits/iqn/iqn.hpp"
 #include "ensmallen_bits/katyusha/katyusha.hpp"
 #include "ensmallen_bits/lbfgs/lbfgs.hpp"
@@ -99,5 +98,6 @@
 #include "ensmallen_bits/smorms3/smorms3.hpp"
 #include "ensmallen_bits/spalera_sgd/spalera_sgd.hpp"
 #include "ensmallen_bits/svrg/svrg.hpp"
+#include "ensmallen_bits/wn_grad/wn_grad.hpp"
 
 #endif
diff --git a/inst/include/ensmallen_bits/ens_version.hpp b/inst/include/ensmallen_bits/ens_version.hpp
@@ -15,13 +15,13 @@
 #define ENS_VERSION_MAJOR 1
 // The minor version is two digits so regular numerical comparisons of versions
 // work right.  The first minor version of a release is always 10.
-#define ENS_VERSION_MINOR 10
-#define ENS_VERSION_PATCH 0
+#define ENS_VERSION_MINOR 11
+#define ENS_VERSION_PATCH 1
 // If this is a release candidate, it will be reflected in the version name
 // (i.e. the version name will be "RC1", "RC2", etc.).  Otherwise the version
 // name will typically be a seemingly arbitrary set of words that does not
 // contain the capitalized string "RC".
-#define ENS_VERSION_NAME  "Corporate Catabolism"
+#define ENS_VERSION_NAME "Jet Lag"
 
 namespace ens {
 
diff --git a/inst/include/ensmallen_bits/function/sfinae_utility.hpp b/inst/include/ensmallen_bits/function/sfinae_utility.hpp
@@ -102,12 +102,12 @@ struct MethodFormDetector<Class, MethodForm, 7>
   void operator()(MethodForm<Class, T1, T2, T3, T4, T5, T6, T7>);
 };
 
-} // namespace sfinae
-} // namespace ens
-
 //! Utility struct for checking signatures.
 template<typename U, U> struct SigCheck : std::true_type {};
 
+} // namespace sfinae
+} // namespace ens
+
 /*
  * Constructs a template supporting the SFINAE pattern.
  *
diff --git a/inst/include/ensmallen_bits/grid_search/grid_search_impl.hpp b/inst/include/ensmallen_bits/grid_search/grid_search_impl.hpp
@@ -24,31 +24,13 @@ double GridSearch::Optimize(
     const std::vector<bool>& categoricalDimensions,
     const arma::Row<size_t>& numCategories)
 {
-  if (categoricalDimensions.size() != iterate.n_rows)
-  {
-    std::ostringstream oss;
-    oss << "GridSearch::Optimize(): expected information about "
-        << iterate.n_rows << " dimensions in categoricalDimensions, "
-        << "but got " << categoricalDimensions.size();
-    throw std::invalid_argument(oss.str());
-  }
-
-  if (numCategories.n_elem != iterate.n_rows)
-  {
-    std::ostringstream oss;
-    oss << "GridSearch::Optimize(): expected numCategories to have length "
-        << "equal to number of dimensions (" << iterate.n_rows << ") but it has"
-        << " length " << numCategories.n_elem;
-    throw std::invalid_argument(oss.str());
-  }
-
   for (size_t i = 0; i < categoricalDimensions.size(); ++i)
   {
-    if (categoricalDimensions[i])
+    if (!categoricalDimensions[i])
     {
       std::ostringstream oss;
       oss << "GridSearch::Optimize(): the dimension " << i
-          << "is not categorical" << std::endl;
+          << " is not categorical" << std::endl;
       throw std::invalid_argument(oss.str());
     }
   }
diff --git a/inst/include/ensmallen_bits/sgd/decay_policies/no_decay.hpp b/inst/include/ensmallen_bits/sgd/decay_policies/no_decay.hpp
@@ -12,8 +12,8 @@
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
 
-#ifndef ENSMALLEN_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP
-#define ENSMALLEN_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP
+#ifndef ENSMALLEN_SGD_DECAY_POLICIES_NO_DECAY_HPP
+#define ENSMALLEN_SGD_DECAY_POLICIES_NO_DECAY_HPP
 
 namespace ens {
 
@@ -64,4 +64,4 @@ class NoDecay
 
 } // namespace ens
 
-#endif // ENSMALLEN_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP
+#endif // ENSMALLEN_SGD_DECAY_POLICIES_NO_DECAY_HPP
diff --git a/inst/include/ensmallen_bits/sgd/sgd.hpp b/inst/include/ensmallen_bits/sgd/sgd.hpp
@@ -71,9 +71,9 @@ namespace ens {
  * the first point in the dataset (presumably, the dataset is held internally in
  * the DecomposableFunctionType).
  *
- * @tparam UpdatePolicyType update policy used by SGD during the iterative update
- *     process. By default vanilla update policy (see ens::VanillaUpdate) is
- *     used.
+ * @tparam UpdatePolicyType update policy used by SGD during the iterative
+ *     update process. By default vanilla update policy (see ens::VanillaUpdate)
+ *     is used.
  * @tparam DecayPolicyType Decay policy used during the iterative update
  *     process to adjust the step size. By default the step size isn't going to
  *     be adjusted (i.e. NoDecay is used).
@@ -84,7 +84,7 @@ class SGD
 {
  public:
   /**
-   * Construct the SGD optimizer with the given function and parameters.  The
+   * Construct the SGD optimizer with the given function and parameters. The
    * defaults here are not necessarily good for the given problem, so it is
    * suggested that the values used be tailored to the task at hand.  The
    * maximum number of iterations refers to the maximum number of points that
diff --git a/inst/include/ensmallen_bits/sgd/sgd_impl.hpp b/inst/include/ensmallen_bits/sgd/sgd_impl.hpp
@@ -81,15 +81,15 @@ double SGD<UpdatePolicyType, DecayPolicyType>::Optimize(
 
       if (std::isnan(overallObjective) || std::isinf(overallObjective))
       {
-//        Log::Warn << "SGD: converged to " << overallObjective << "; terminating"
-//            << " with failure.  Try a smaller step size?" << std::endl;
+        Warn << "SGD: converged to " << overallObjective << "; terminating"
+            << " with failure.  Try a smaller step size?" << std::endl;
         return overallObjective;
       }
 
       if (std::abs(lastObjective - overallObjective) < tolerance)
       {
-//        Log::Info << "SGD: minimized within tolerance " << tolerance << "; "
-//            << "terminating optimization." << std::endl;
+        Info << "SGD: minimized within tolerance " << tolerance << "; "
+            << "terminating optimization." << std::endl;
         return overallObjective;
       }
 
@@ -127,8 +127,8 @@ double SGD<UpdatePolicyType, DecayPolicyType>::Optimize(
     currentFunction += effectiveBatchSize;
   }
 
-//  Log::Info << "SGD: maximum iterations (" << maxIterations << ") reached; "
-//      << "terminating optimization." << std::endl;
+  Info << "SGD: maximum iterations (" << maxIterations << ") reached; "
+      << "terminating optimization." << std::endl;
 
   // Calculate final objective.
   overallObjective = 0;
diff --git a/inst/include/ensmallen_bits/sgd/update_policies/nesterov_momentum_update.hpp b/inst/include/ensmallen_bits/sgd/update_policies/nesterov_momentum_update.hpp
@@ -20,10 +20,10 @@ namespace ens {
  * Learning with SGD can be slow. Applying Standard momentum can accelerate
  * the rate of convergence. Nesterov Momentum application can accelerate the
  * rate of convergence to O(1/k^2).
- * 
+ *
  * @code
  * @techreport{Nesterov1983,
- *   title       = {A Method Of Solving A Convex Programming Problem With 
+ *   title       = {A Method Of Solving A Convex Programming Problem With
  *                  Convergence Rate O(1/K^2)},
  *   author      = {Yuri Nesterov},
  *   institution = {Soviet Math. Dokl.},
@@ -39,8 +39,7 @@ class NesterovMomentumUpdate
    * Construct the Nesterov Momentum update policy with the given parameters.
    *
    */
-  NesterovMomentumUpdate(const double momentum = 0.5) :
-      momentum(momentum)
+  NesterovMomentumUpdate(const double momentum = 0.5) : momentum(momentum)
   {
     // Nothing to do.
   }
diff --git a/inst/include/ensmallen_bits/wn_grad/wn_grad.hpp b/inst/include/ensmallen_bits/wn_grad/wn_grad.hpp
@@ -0,0 +1,133 @@
+/**
+ * @file wn_grad.hpp
+ * @author Marcus Edel
+ *
+ * WNGrad is a general nonlinear update rule for the learning rate.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_WN_GRAD_WN_GRAD_HPP
+#define ENSMALLEN_WN_GRAD_WN_GRAD_HPP
+
+#include <ensmallen_bits/sgd/sgd.hpp>
+#include "wn_grad_update.hpp"
+
+namespace ens {
+
+/**
+ * WNGrad is a general nonlinear update rule for the learning rate, so that
+ * the learning rate can be initialized at a high value, and is subsequently
+ * decreased according to gradient observations.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{Wu2018,
+ *   author  = {{Wu}, X. and {Ward}, R. and {Bottou}, L.},
+ *   title   = {WNGrad: Learn the Learning Rate in Gradient Descent},
+ *   journal = {ArXiv e-prints},
+ *   year    = {2018},
+ *   url     = {https://arxiv.org/abs/1803.02865},
+ * }
+ * @endcode
+ *
+ * For WNGrad to work, a DecomposableFunctionType template parameter is
+ * required. This class must implement the following function:
+ *
+ *   size_t NumFunctions();
+ *   double Evaluate(const arma::mat& coordinates,
+ *                   const size_t i,
+ *                   const size_t batchSize);
+ *   void Gradient(const arma::mat& coordinates,
+ *                 const size_t i,
+ *                 arma::mat& gradient,
+ *                 const size_t batchSize);
+ *
+ * NumFunctions() should return the number of functions (\f$n\f$), and in the
+ * other two functions, the parameter i refers to which individual function (or
+ * gradient) is being evaluated.  So, for the case of a data-dependent function,
+ * such as NCA, NumFunctions() should return the number of points in the
+ * dataset, and Evaluate(coordinates, 0) will evaluate the objective function on
+ * the first point in the dataset (presumably, the dataset is held internally in
+ * the DecomposableFunctionType).
+ */
+class WNGrad
+{
+ public:
+  /**
+   * Construct the WNGrad optimizer with the given function and parameters. The
+   * defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *        function is visited in linear order.
+   */
+  WNGrad(const double stepSize = 0.562,
+         const size_t batchSize = 32,
+         const size_t maxIterations = 100000,
+         const double tolerance = 1e-5,
+         const bool shuffle = true);
+
+  /**
+   * Optimize the given function using WNGrad. The given starting point will
+   * be modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam DecomposableFunctionType Type of the function to be optimized.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @return Objective value of the final point.
+   */
+  template<typename DecomposableFunctionType>
+  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  {
+    return optimizer.Optimize(function, iterate);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+ private:
+  //! The WNGrad update policy.
+  SGD<WNGradUpdate> optimizer;
+};
+
+} // namespace ens
+
+// Include implementation.
+#include "wn_grad_impl.hpp"
+
+#endif
diff --git a/inst/include/ensmallen_bits/wn_grad/wn_grad_impl.hpp b/inst/include/ensmallen_bits/wn_grad/wn_grad_impl.hpp
@@ -0,0 +1,36 @@
+/**
+ * @file wn_grad_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of the WNGrad optimizer.
+ *
+ * ensmallen is free software; you may redistribute it and/or modify it under
+ * the terms of the 3-clause BSD license.  You should have received a copy of
+ * the 3-clause BSD license along with ensmallen.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef ENSMALLEN_WN_GRAD_WN_GRAD_IMPL_HPP
+#define ENSMALLEN_WN_GRAD_WN_GRAD_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "wn_grad.hpp"
+
+namespace ens {
+
+inline WNGrad::WNGrad(
+    const double stepSize,
+    const size_t batchSize,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              WNGradUpdate())
+{ /* Nothing to do. */ }
+
+} // namespace ens
+
+#endif
diff --git a/inst/include/ensmallen_bits/wn_grad/wn_grad_update.hpp b/inst/include/ensmallen_bits/wn_grad/wn_grad_update.hpp
diff --git a/man/RcppEnsmallen-package.Rd b/man/RcppEnsmallen-package.Rd

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+^CRAN-RELEASE$`
`1`	`2`	`^.*\.Rproj$`
`2`	`3`	`^\.Rproj\.user$`
`3`	`4`	`^\.travis\.yml$`
Original file line number	Diff line number	Diff line change
`@@ -81,15 +81,15 @@ double SGD<UpdatePolicyType, DecayPolicyType>::Optimize(`
`81`	`81`
`82`	`82`	`if (std::isnan(overallObjective) \|\| std::isinf(overallObjective))`
`83`	`83`	`{`
`84`		`-// Log::Warn << "SGD: converged to " << overallObjective << "; terminating"`
`85`		`-// << " with failure. Try a smaller step size?" << std::endl;`
	`84`	`+ Warn << "SGD: converged to " << overallObjective << "; terminating"`
	`85`	`+ << " with failure. Try a smaller step size?" << std::endl;`
`86`	`86`	`return overallObjective;`
`87`	`87`	`}`
`88`	`88`
`89`	`89`	`if (std::abs(lastObjective - overallObjective) < tolerance)`
`90`	`90`	`{`
`91`		`-// Log::Info << "SGD: minimized within tolerance " << tolerance << "; "`
`92`		`-// << "terminating optimization." << std::endl;`
	`91`	`+ Info << "SGD: minimized within tolerance " << tolerance << "; "`
	`92`	`+ << "terminating optimization." << std::endl;`
`93`	`93`	`return overallObjective;`
`94`	`94`	`}`
`95`	`95`
`@@ -127,8 +127,8 @@ double SGD<UpdatePolicyType, DecayPolicyType>::Optimize(`
`127`	`127`	`currentFunction += effectiveBatchSize;`
`128`	`128`	`}`
`129`	`129`
`130`		`-// Log::Info << "SGD: maximum iterations (" << maxIterations << ") reached; "`
`131`		`-// << "terminating optimization." << std::endl;`
	`130`	`+ Info << "SGD: maximum iterations (" << maxIterations << ") reached; "`
	`131`	`+ << "terminating optimization." << std::endl;`
`132`	`132`
`133`	`133`	`// Calculate final objective.`
`134`	`134`	`overallObjective = 0;`