added example, deleted export of small functions

Author: AnotherSamWilson

NAMESPACE

@@ -1,16 +1,6 @@
 # Generated by roxygen2: do not edit by hand
 
 export(BayesianOptimization)
-export(CheckBounds)
-export(MinMaxScale)
-export(RandParams)
-export(UnMMScale)
-export(applyCluster)
-export(applyNoise)
-export(assignKern)
-export(calcAcq)
-export(maxAcq)
-export(updateGP)
 import(foreach)
 importFrom(GauPro,Exponential)
 importFrom(GauPro,GauPro_kernel_model)

R/BayesianOptimization.R

@@ -73,69 +73,87 @@
 #' \item{BestPars}{The best parameter set at each iteration}
 #' @references Jasper Snoek, Hugo Larochelle, Ryan P. Adams (2012) \emph{Practical Bayesian Optimization of Machine Learning Algorithms}
 #' @examples
+#' # Example 1 - Optimization of a Linear Function
+#' scoringFunction <- function(x) {
+#'   a <- exp(-(2-x)^2)*1.5
+#'   b <- exp(-(4-x)^2)*2
+#'   c <- exp(-(6-x)^2)*1
+#'   return(list(Score = a+b+c))
+#' }
+#'
+#' bounds <- list(x = c(0,8))
+#'
+#' Results <- BayesianOptimization(
+#'     FUN = scoringFunction
+#'   , bounds = bounds
+#'   , initPoints = 5
+#'   , nIters = 8
+#'   , gsPoints = 10
+#' )
+#'
 #' \dontrun{
+#' # Example 2 - Hyperparameter Tuning in xgboost
 #' library("xgboost")
-#' library("ParBayesianOptimization")
 #'
 #' data(agaricus.train, package = "xgboost")
 #'
-#' Folds <- list(  Fold1 = as.integer(seq(1,nrow(agaricus.train$data),by = 3))
-#'                 , Fold2 = as.integer(seq(2,nrow(agaricus.train$data),by = 3))
-#'                 , Fold3 = as.integer(seq(3,nrow(agaricus.train$data),by = 3)))
+#' Folds <- list( Fold1 = as.integer(seq(1,nrow(agaricus.train$data),by = 3))
+#'              , Fold2 = as.integer(seq(2,nrow(agaricus.train$data),by = 3))
+#'              , Fold3 = as.integer(seq(3,nrow(agaricus.train$data),by = 3)))
 #'
 #' scoringFunction <- function(max_depth, min_child_weight, subsample) {
 #'
 #'   dtrain <- xgb.DMatrix(agaricus.train$data,label = agaricus.train$label)
 #'
 #'   Pars <- list( booster = "gbtree"
-#'                 , eta = 0.01
-#'                 , max_depth = max_depth
-#'                 , min_child_weight = min_child_weight
-#'                 , subsample = subsample
-#'                 , objective = "binary:logistic"
-#'                 , eval_metric = "auc")
+#'               , eta = 0.01
+#'               , max_depth = max_depth
+#'               , min_child_weight = min_child_weight
+#'               , subsample = subsample
+#'               , objective = "binary:logistic"
+#'               , eval_metric = "auc")
 #'
-#'   xgbcv <- xgb.cv(params = Pars,
-#'                   data = dtrain
-#'                   , nround = 100
-#'                   , folds = Folds
-#'                   , prediction = TRUE
-#'                   , showsd = TRUE
-#'                   , early_stopping_rounds = 5
-#'                   , maximize = TRUE
-#'                   , verbose = 0)
+#'   xgbcv <- xgb.cv( params = Pars
+#'                  , data = dtrain
+#'                  , nround = 100
+#'                  , folds = Folds
+#'                  , prediction = TRUE
+#'                  , showsd = TRUE
+#'                  , early_stopping_rounds = 5
+#'                  , maximize = TRUE
+#'                  , verbose = 0)
 #'
-#'   return(list(Score = max(xgbcv$evaluation_log$test_auc_mean)
-#'               , nrounds = xgbcv$best_iteration
+#'   return(list( Score = max(xgbcv$evaluation_log$test_auc_mean)
+#'              , nrounds = xgbcv$best_iteration
 #'   )
 #'   )
 #' }
 #'
-#'
-#' bounds <- list( max_depth = c(2L, 10L)
-#'                 , min_child_weight = c(1, 100)
-#'                 , subsample = c(0.25, 1))
+#' bounds <- list(max_depth = c(2L, 10L)
+#'              , min_child_weight = c(1, 100)
+#'              , subsample = c(0.25, 1))
 #'
 #' kern <- "Matern52"
 #'
 #' acq <- "ei"
 #'
-#' ScoreResult <- BayesianOptimization(FUN = scoringFunction
-#'                                     , bounds = bounds
-#'                                     , initPoints = 10
-#'                                     , bulkNew = 1
-#'                                     , nIters = 12
-#'                                     , kern = kern
-#'                                     , acq = acq
-#'                                     , kappa = 2.576
-#'                                     , verbose = 1
-#'                                     , parallel = FALSE)
+#' ScoreResult <- BayesianOptimization(
+#'     FUN = scoringFunction
+#'   , bounds = bounds
+#'   , initPoints = 10
+#'   , bulkNew = 1
+#'   , nIters = 12
+#'   , kern = kern
+#'   , acq = acq
+#'   , kappa = 2.576
+#'   , verbose = 1
+#'   , parallel = FALSE
+#'   , gsPoints = 50)
 #' }
 #' @importFrom data.table data.table setDT setcolorder := as.data.table copy .I
 #' @importFrom utils head
 #' @importFrom GauPro GauPro_kernel_model Matern52 Matern32 Exponential Gaussian
 #' @export
-
 BayesianOptimization <- function(
     FUN
   , bounds

R/SmallFuncs.R

@@ -14,7 +14,6 @@
 #' @param bounds the bounds list
 #' @importFrom data.table between
 #' @return the number of values that are outside the bounds/
-#' @export
 CheckBounds <- function(x, Table, bounds) sum(!between(Table[[x]], lower = bounds[[x]][[1]], upper = bounds[[x]][[2]]))
 
 
@@ -29,7 +28,6 @@ CheckBounds <- function(x, Table, bounds) sum(!between(Table[[x]], lower = bound
 #' @importFrom stats runif
 #' @return a data.table of random parameters
 #' @keywords internal
-#' @export
 RandParams <- function(x, Rpoints, boundsDT) {
 
   B <- boundsDT[get("N") == x,]
@@ -55,7 +53,6 @@ RandParams <- function(x, Rpoints, boundsDT) {
 #' @param bounds the original bounds list
 #' @return a data.table the same length as \code{table} with scaled parameters
 #' @keywords internal
-#' @export
 MinMaxScale <- function(x, table, boundsDT) {
   B <- boundsDT[get("N") == x,]
   (table[[x]]-B$L) / (B$U-B$L)
@@ -72,7 +69,6 @@ MinMaxScale <- function(x, table, boundsDT) {
 #' @param bounds the original bounds list
 #' @return a data.table the same length as \code{table} with un-scaled parameters
 #' @keywords internal
-#' @export
 UnMMScale <- function(x, table, boundsDT) {
   B <- boundsDT[get("N") == x,]
 
@@ -91,7 +87,6 @@ UnMMScale <- function(x, table, boundsDT) {
 #' @param beta the log10(theta) the lengthscale parameter
 #' @return an GauPro_kernel_beta R6 class
 #' @keywords internal
-#' @export
 assignKern <- function(kern,beta) {
 
   if(kern == "Matern32"){kern <- Matern32$new(beta)

R/applyCluster.R

@@ -7,7 +7,6 @@
 #' @importFrom dbscan dbscan
 #' @importFrom data.table fintersect
 #' @return the number of values that are outside the bounds/
-#' @export
 
 applyCluster <- function(e = parent.frame()) {
 
@@ -21,7 +20,7 @@ applyCluster <- function(e = parent.frame()) {
 
   } else{
 
-    Clust <- dbscan(Lo2,eps = length(Lo2)*sqrt(2)/e$convThresh, minPts = 1)
+    Clust <- dbscan(Lo2,eps = length(Lo2)*sqrt(2)/1e3, minPts = 1)
 
     Lo2[,"Cluster" := Clust$cluster]
     clusterPoints <- copy(Lo2[Lo2[,.I[which.max(get("GP_Utility"))], by = get("Cluster")]$V1])

R/applyNoise.R

@@ -10,7 +10,6 @@
 #' @importFrom stats rbeta
 #' @return a data.table with the same number of rows as \code{table} with noise added
 #' @keywords internal
-#' @export
 #'
 applyNoise <- function( x
                       , table
@@ -36,4 +35,3 @@ applyNoise <- function( x
   return(Vec)
 
 }
-

R/calcAcq.R

@@ -12,7 +12,6 @@
 #' @importFrom stats dnorm pnorm
 #' @return The acquisition function value.
 #' @keywords internal
-#' @export
 
 calcAcq <- function(Params, GP, acq, y_max, kappa, eps) {
   # Utility Function Type

R/maxAcq.R

@@ -16,7 +16,6 @@
 #' @importFrom data.table as.data.table
 #' @import foreach
 #' @keywords internal
-#' @export
 
 maxAcq <- function(GP, TryOver, acq = "ucb", y_max, kappa, eps, parallel, ParMethod, convThresh) {
 

R/updateGP.R

@@ -10,7 +10,6 @@
 #' @param scaleList list of scaling parameters for the responses
 #' @return A data table of local optimums.
 #' @keywords internal
-#' @export
 
 updateGP <- function(GP, kern, X, Z, acq, scaleList, parallel) {
 

README.md

@@ -28,6 +28,37 @@ Idealy, we would use the information from prior model evaluations to guide us in
 4.  New parameter-score pairs are found
 5.  Repeat steps 2-4 until some stopping criteria is met
 
+Graphical Intuition
+-------------------
+
+As an example, let's say we are only tuning 1 hyperparameter in an xgboost model, min\_child weight in (0,1). We have initialized the process by randomly sampling the scoring function 6 times, and get the following results:
+
+|  min\_child\_weight|      Score|
+|-------------------:|----------:|
+|           0.6280082|  0.7133457|
+|           0.3276477|  0.8655448|
+|           0.7486012|  0.6814730|
+|           0.2425469|  1.0000000|
+|           0.0724098|  0.1308284|
+|           0.1579683|  0.5733343|
+
+How do we go about determining the best min\_child\_weight to try next? As it turns out, Gaussian processes can give us a very good definition for our prior distribution. Fitting a Gaussian process to the data above (indexed by min\_child\_weight), we can see the expected value accross our parameter bounds, as well as the uncertainty at different points:
+
+![](vignettes/GPpredictions.png)
+
+Before we can select our next candidate parameter to run the scoring function on, we need to determine how we define a "good" parameter inside this prior distribution. This is done by maximizing different functions within the Gaussian process. There are several functions to choose from:
+
+-   Upper Confidence Bound (ucb)
+-   Probability Of Improvement (poi)
+-   Expected Improvement (ei)
+-   Expected Improvement Per Second (eips)
+
+Continuing the example, we select to find the min\_child\_weight which maximizes the expected improvement according to the Gaussian process. As you can see, there are several good candidates:
+
+![](vignettes/expectedImprovement.png)
+
+An advanced feature of ParBayesianOptimization, which you can read about in the vignette advancedFeatures, describes how to use the `minClusterUtility` parameter to search over the different local maximums shown above. If not specified, only the global maximum would be sampled.
+
 Practical Example
 -----------------
 
@@ -110,12 +141,12 @@ ScoreResult <- BayesianOptimization(FUN = scoringFunction
 #> 
 #> Starting round number 1
 #>   1) Fitting Gaussian process...
-#>   2) Running global optimum search...
+#>   2) Running local optimum search...
 #>   3) Running scoring function 1 times in 1 thread(s)...
 #> 
 #> Starting round number 2
 #>   1) Fitting Gaussian process...
-#>   2) Running global optimum search...
+#>   2) Running local optimum search...
 #>   3) Running scoring function 1 times in 1 thread(s)...
 ```
 
@@ -124,24 +155,24 @@ The console informs us that the process initialized by running `scoringFunction`
 ``` r
 ScoreResult$ScoreDT
 #>     Iteration max_depth min_child_weight subsample Elapsed     Score nrounds
-#>  1:         0         8               96 0.5935750    0.70 0.9797757      32
-#>  2:         0         4               87 0.7504342    0.16 0.9779723       1
-#>  3:         0         2               76 0.9639782    0.12 0.9779723       1
-#>  4:         0         8               48 0.8497707    0.22 0.9889367       3
-#>  5:         0         6               17 0.3089911    0.58 0.9951817      19
-#>  6:         0         9               24 0.7529264    0.67 0.9974167      23
-#>  7:         0         7                1 0.3099921    0.31 0.9986540       5
-#>  8:         0         4               49 0.8077531    0.33 0.9901100       8
-#>  9:         0         3               35 0.9579958    0.16 0.9900173       2
-#> 10:         0         3               22 0.8018554    0.36 0.9953687      15
-#> 11:         1        10                1 0.8609687    0.24 0.9984757       1
-#> 12:         2         8                1 0.6257481    0.42 0.9986603       8
+#>  1:         0         3               21 0.4527621    0.37 0.9947540      10
+#>  2:         0         4               63 0.6965777    0.21 0.9782343       1
+#>  3:         0         3               45 0.9528065    0.27 0.9874717       3
+#>  4:         0         2               54 0.8177109    0.30 0.9869460       8
+#>  5:         0        10               20 0.7861882    2.53 0.9991937      97
+#>  6:         0        10               12 0.2594141    0.72 0.9962943      16
+#>  7:         0        10                6 0.4839067    0.57 0.9991430       9
+#>  8:         0         7               58 0.5443817    0.18 0.9779723       1
+#>  9:         0        10               22 0.5056945    1.79 0.9970410      52
+#> 10:         0        10               27 0.6718752    0.39 0.9943417       6
+#> 11:         1        10                1 0.9089677    0.28 0.9984757       1
+#> 12:         2        10                1 0.6652759    0.43 0.9986620       5
 ```
 
 ``` r
 ScoreResult$BestPars
 #>    Iteration max_depth min_child_weight subsample     Score nrounds elapsedSecs
-#> 1:         0         7                1 0.3099921 0.9986540       5      5 secs
-#> 2:         1         7                1 0.3099921 0.9986540       5     13 secs
-#> 3:         2         8                1 0.6257481 0.9986603       8     20 secs
+#> 1:         0        10               20 0.7861882 0.9991937      97      8 secs
+#> 2:         1        10               20 0.7861882 0.9991937      97     18 secs
+#> 3:         2        10               20 0.7861882 0.9991937      97     28 secs
 ```