Skip to content

Commit 99a7e6b

Browse files
apoorvalalapoorvalal
committed
"gradient free and nonsmooth optimizers"
1 parent 75184d9 commit 99a7e6b

File tree

4 files changed

+201
-0
lines changed

4 files changed

+201
-0
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,3 +166,4 @@ cython_debug/
166166
.DS_Store
167167
.claude/settings.local.json
168168
tmp.md
169+
ensmallen/*

‎pyensmallen/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@
2828
"AMSGrad",
2929
"OptimisticAdam",
3030
"Nadam",
31+
"CyclicDescent",
32+
"RandomDescent",
33+
"GreedyDescent",
34+
"SimulatedAnnealing",
3135
"linear_obj",
3236
"logistic_obj",
3337
"poisson_obj",

‎pyensmallen/additional_optimizers.hpp‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#pragma once
2+
3+
#include "utils.hpp"
4+
5+
// Wrapper for Coordinate Descent optimizer
6+
template<typename DescentPolicy = ens::RandomDescent>
7+
class PyCoordinateDescent
8+
{
9+
public:
10+
PyCoordinateDescent(double stepSize = 0.01,
11+
size_t maxIterations = 100000,
12+
double tolerance = 1e-5,
13+
size_t updateInterval = 1000)
14+
: optimizer(stepSize, maxIterations, tolerance, updateInterval) {}
15+
16+
PyCoordinateDescent(double stepSize,
17+
size_t maxIterations,
18+
double tolerance,
19+
size_t updateInterval,
20+
const DescentPolicy& descentPolicy)
21+
: optimizer(stepSize, maxIterations, tolerance, updateInterval, descentPolicy) {}
22+
23+
double getStepSize() const { return optimizer.StepSize(); }
24+
void setStepSize(double stepSize) { optimizer.StepSize() = stepSize; }
25+
26+
size_t getMaxIterations() const { return optimizer.MaxIterations(); }
27+
void setMaxIterations(size_t maxIterations) { optimizer.MaxIterations() = maxIterations; }
28+
29+
double getTolerance() const { return optimizer.Tolerance(); }
30+
void setTolerance(double tolerance) { optimizer.Tolerance() = tolerance; }
31+
32+
size_t getUpdateInterval() const { return optimizer.UpdateInterval(); }
33+
void setUpdateInterval(size_t updateInterval) { optimizer.UpdateInterval() = updateInterval; }
34+
35+
template<typename FunctionType>
36+
arma::mat Optimize(FunctionType& function, const arma::mat& initialPoint)
37+
{
38+
arma::mat coordinates = initialPoint;
39+
optimizer.Optimize(function, coordinates);
40+
return coordinates;
41+
}
42+
43+
private:
44+
ens::CD<DescentPolicy> optimizer;
45+
};
46+
47+
// Specific coordinate descent variants
48+
using PyCyclicDescent = PyCoordinateDescent<ens::CyclicDescent>;
49+
using PyRandomDescent = PyCoordinateDescent<ens::RandomDescent>;
50+
using PyGreedyDescent = PyCoordinateDescent<ens::GreedyDescent>;
51+
52+
// Wrapper for Simulated Annealing optimizer
53+
template<typename CoolingSchedule = ens::ExponentialSchedule>
54+
class PySimulatedAnnealing
55+
{
56+
public:
57+
PySimulatedAnnealing(size_t maxIterations = 1000000,
58+
double initT = 10000.0,
59+
size_t initMoves = 1000,
60+
size_t moveCtrlSweep = 100,
61+
double tolerance = 1e-5,
62+
size_t maxToleranceSweep = 3,
63+
double maxMoveCoef = 20,
64+
double initMoveCoef = 0.3,
65+
double gain = 0.3)
66+
: optimizer(CoolingSchedule(), maxIterations, initT, initMoves,
67+
moveCtrlSweep, tolerance, maxToleranceSweep,
68+
maxMoveCoef, initMoveCoef, gain) {}
69+
70+
PySimulatedAnnealing(const CoolingSchedule& coolingSchedule,
71+
size_t maxIterations = 1000000,
72+
double initT = 10000.0,
73+
size_t initMoves = 1000,
74+
size_t moveCtrlSweep = 100,
75+
double tolerance = 1e-5,
76+
size_t maxToleranceSweep = 3,
77+
double maxMoveCoef = 20,
78+
double initMoveCoef = 0.3,
79+
double gain = 0.3)
80+
: optimizer(coolingSchedule, maxIterations, initT, initMoves,
81+
moveCtrlSweep, tolerance, maxToleranceSweep,
82+
maxMoveCoef, initMoveCoef, gain) {}
83+
84+
size_t getMaxIterations() const { return optimizer.MaxIterations(); }
85+
void setMaxIterations(size_t maxIterations) { optimizer.MaxIterations() = maxIterations; }
86+
87+
double getTemperature() const { return optimizer.Temperature(); }
88+
void setTemperature(double temperature) { optimizer.Temperature() = temperature; }
89+
90+
size_t getInitMoves() const { return optimizer.InitMoves(); }
91+
void setInitMoves(size_t initMoves) { optimizer.InitMoves() = initMoves; }
92+
93+
size_t getMoveCtrlSweep() const { return optimizer.MoveCtrlSweep(); }
94+
void setMoveCtrlSweep(size_t moveCtrlSweep) { optimizer.MoveCtrlSweep() = moveCtrlSweep; }
95+
96+
double getTolerance() const { return optimizer.Tolerance(); }
97+
void setTolerance(double tolerance) { optimizer.Tolerance() = tolerance; }
98+
99+
size_t getMaxToleranceSweep() const { return optimizer.MaxToleranceSweep(); }
100+
void setMaxToleranceSweep(size_t maxToleranceSweep) { optimizer.MaxToleranceSweep() = maxToleranceSweep; }
101+
102+
double getGain() const { return optimizer.Gain(); }
103+
void setGain(double gain) { optimizer.Gain() = gain; }
104+
105+
template<typename FunctionType>
106+
arma::mat Optimize(FunctionType& function, const arma::mat& initialPoint)
107+
{
108+
arma::mat coordinates = initialPoint;
109+
optimizer.Optimize(function, coordinates);
110+
return coordinates;
111+
}
112+
113+
private:
114+
ens::SA<CoolingSchedule> optimizer;
115+
};
116+
117+
// Default simulated annealing with exponential cooling
118+
using PySimulatedAnnealingDefault = PySimulatedAnnealing<ens::ExponentialSchedule>;

‎pyensmallen/module.cpp‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include "newton_type.hpp"
66
#include "constrained.hpp"
77
#include "first_order.hpp"
8+
#include "additional_optimizers.hpp"
89

910
namespace py = pybind11;
1011

@@ -245,4 +246,81 @@ PYBIND11_MODULE(_pyensmallen, m)
245246
&PyAdamType<ens::NadamUpdate>::getResetPolicy,
246247
&PyAdamType<ens::NadamUpdate>::setResetPolicy)
247248
.def("optimize", &PyAdamType<ens::NadamUpdate>::Optimize);
249+
250+
// Coordinate Descent optimizers
251+
py::class_<PyCyclicDescent>(m, "CyclicDescent")
252+
.def(py::init<double, size_t, double, size_t>(),
253+
py::arg("stepSize") = 0.01,
254+
py::arg("maxIterations") = 100000,
255+
py::arg("tolerance") = 1e-5,
256+
py::arg("updateInterval") = 1000)
257+
.def_property("stepSize", &PyCyclicDescent::getStepSize,
258+
&PyCyclicDescent::setStepSize)
259+
.def_property("maxIterations", &PyCyclicDescent::getMaxIterations,
260+
&PyCyclicDescent::setMaxIterations)
261+
.def_property("tolerance", &PyCyclicDescent::getTolerance,
262+
&PyCyclicDescent::setTolerance)
263+
.def_property("updateInterval", &PyCyclicDescent::getUpdateInterval,
264+
&PyCyclicDescent::setUpdateInterval)
265+
.def("optimize", &PyCyclicDescent::Optimize<PyObjectiveFunction>);
266+
267+
py::class_<PyRandomDescent>(m, "RandomDescent")
268+
.def(py::init<double, size_t, double, size_t>(),
269+
py::arg("stepSize") = 0.01,
270+
py::arg("maxIterations") = 100000,
271+
py::arg("tolerance") = 1e-5,
272+
py::arg("updateInterval") = 1000)
273+
.def_property("stepSize", &PyRandomDescent::getStepSize,
274+
&PyRandomDescent::setStepSize)
275+
.def_property("maxIterations", &PyRandomDescent::getMaxIterations,
276+
&PyRandomDescent::setMaxIterations)
277+
.def_property("tolerance", &PyRandomDescent::getTolerance,
278+
&PyRandomDescent::setTolerance)
279+
.def_property("updateInterval", &PyRandomDescent::getUpdateInterval,
280+
&PyRandomDescent::setUpdateInterval)
281+
.def("optimize", &PyRandomDescent::Optimize<PyObjectiveFunction>);
282+
283+
py::class_<PyGreedyDescent>(m, "GreedyDescent")
284+
.def(py::init<double, size_t, double, size_t>(),
285+
py::arg("stepSize") = 0.01,
286+
py::arg("maxIterations") = 100000,
287+
py::arg("tolerance") = 1e-5,
288+
py::arg("updateInterval") = 1000)
289+
.def_property("stepSize", &PyGreedyDescent::getStepSize,
290+
&PyGreedyDescent::setStepSize)
291+
.def_property("maxIterations", &PyGreedyDescent::getMaxIterations,
292+
&PyGreedyDescent::setMaxIterations)
293+
.def_property("tolerance", &PyGreedyDescent::getTolerance,
294+
&PyGreedyDescent::setTolerance)
295+
.def_property("updateInterval", &PyGreedyDescent::getUpdateInterval,
296+
&PyGreedyDescent::setUpdateInterval)
297+
.def("optimize", &PyGreedyDescent::Optimize<PyObjectiveFunction>);
298+
299+
// Simulated Annealing optimizer
300+
py::class_<PySimulatedAnnealingDefault>(m, "SimulatedAnnealing")
301+
.def(py::init<size_t, double, size_t, size_t, double, size_t, double, double, double>(),
302+
py::arg("maxIterations") = 1000000,
303+
py::arg("initT") = 10000.0,
304+
py::arg("initMoves") = 1000,
305+
py::arg("moveCtrlSweep") = 100,
306+
py::arg("tolerance") = 1e-5,
307+
py::arg("maxToleranceSweep") = 3,
308+
py::arg("maxMoveCoef") = 20,
309+
py::arg("initMoveCoef") = 0.3,
310+
py::arg("gain") = 0.3)
311+
.def_property("maxIterations", &PySimulatedAnnealingDefault::getMaxIterations,
312+
&PySimulatedAnnealingDefault::setMaxIterations)
313+
.def_property("temperature", &PySimulatedAnnealingDefault::getTemperature,
314+
&PySimulatedAnnealingDefault::setTemperature)
315+
.def_property("initMoves", &PySimulatedAnnealingDefault::getInitMoves,
316+
&PySimulatedAnnealingDefault::setInitMoves)
317+
.def_property("moveCtrlSweep", &PySimulatedAnnealingDefault::getMoveCtrlSweep,
318+
&PySimulatedAnnealingDefault::setMoveCtrlSweep)
319+
.def_property("tolerance", &PySimulatedAnnealingDefault::getTolerance,
320+
&PySimulatedAnnealingDefault::setTolerance)
321+
.def_property("maxToleranceSweep", &PySimulatedAnnealingDefault::getMaxToleranceSweep,
322+
&PySimulatedAnnealingDefault::setMaxToleranceSweep)
323+
.def_property("gain", &PySimulatedAnnealingDefault::getGain,
324+
&PySimulatedAnnealingDefault::setGain)
325+
.def("optimize", &PySimulatedAnnealingDefault::Optimize<PyObjectiveFunction>);
248326
}

0 commit comments

Comments
 (0)