Merge pull request #1355 from rstudio/optimizers-tf-v2.11-update

Updates for TF v2.11

Author: t-kalinowski

.github/workflows/R-CMD-check.yaml

@@ -4,7 +4,7 @@ on:
       - main
   pull_request:
   schedule:
-    - cron: '51 3 * * MON'
+    - cron: '51 3 * * Fri'
 
 name: R-CMD-check
 
@@ -20,28 +20,27 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {os: 'ubuntu-20.04'  , tf: 'default', r: 'release'}
+          - {os: 'ubuntu-latest' , tf: 'default', r: 'release'}
           - {os: 'windows-latest', tf: 'default', r: 'release'}
           - {os: 'macOS-latest'  , tf: 'default', r: 'release'}
 
-          - {os: 'ubuntu-20.04'  , tf: 'default', r: 'oldrel'}
+          - {os: 'ubuntu-latest' , tf: 'default', r: 'oldrel'}
           - {os: 'windows-latest', tf: 'default', r: 'oldrel'}
           - {os: 'macOS-latest'  , tf: 'default', r: 'oldrel'}
 
-          - {os: 'ubuntu-20.04'  , tf: 'default', r: 'oldrel-1'}
-          - {os: 'ubuntu-20.04', tf: 'default', r: '3.6'} # default R in ubuntu-20.04
-          - {os: 'ubuntu-20.04', tf: 'default', r: '3.5'} #
+          - {os: 'ubuntu-latest', tf: 'default', r: 'oldrel-1'}
+          - {os: 'ubuntu-latest', tf: 'default', r: '3.6'} # default R in ubuntu-20.04
+          - {os: 'ubuntu-latest', tf: 'default', r: '3.5'}
 
-          # - {os: 'ubuntu-20.04'  , tf: 'release', r: 'release'}
-          # - {os: 'windows-latest', tf: 'release', r: 'release'}
-          # - {os: 'macOS-latest'  , tf: 'release', r: 'release'}
+          - {os: 'ubuntu-20.04'  , tf: 'release', r: 'release'}
+          - {os: 'windows-latest', tf: 'release', r: 'release'}
+          - {os: 'macOS-latest'  , tf: 'release', r: 'release'}
 
-          - {os: 'ubuntu-20.04', tf: '2.9', r: 'release'}
-          - {os: 'ubuntu-20.04', tf: '2.8', r: 'release'}
-          - {os: 'ubuntu-20.04', tf: '2.7', r: 'release'}
-          - {os: 'ubuntu-20.04', tf: '2.6', r: 'release'}
-          - {os: 'ubuntu-20.04', tf: '2.5', r: 'release'}
-          - {os: 'ubuntu-20.04', tf: '2.4', r: 'release'}
+          - {os: 'ubuntu-latest', tf: '2.10', r: 'release'}
+          - {os: 'ubuntu-latest', tf: '2.9', r: 'release'}
+          - {os: 'ubuntu-latest', tf: '2.8', r: 'release'}
+          - {os: 'ubuntu-latest', tf: '2.7', r: 'release'}
+          - {os: 'ubuntu-latest', tf: '2.6', r: 'release'}
 
           # these are allowed to fail
           # - {os: 'ubuntu-20.04', tf: 'default', r: 'devel'}

DESCRIPTION

@@ -37,7 +37,6 @@ Imports:
     glue,
     methods,
     R6,
-    ellipsis,
     rlang
 Suggests:
     ggplot2,
@@ -50,5 +49,5 @@ Suggests:
     png,
     jpeg
 Roxygen: list(markdown = TRUE, r6 = FALSE)
-RoxygenNote: 7.2.1
+RoxygenNote: 7.2.3
 VignetteBuilder: knitr

NAMESPACE

@@ -526,6 +526,7 @@ export(optimizer_adadelta)
 export(optimizer_adagrad)
 export(optimizer_adam)
 export(optimizer_adamax)
+export(optimizer_ftrl)
 export(optimizer_nadam)
 export(optimizer_rmsprop)
 export(optimizer_sgd)
@@ -618,6 +619,7 @@ importFrom(reticulate,use_condaenv)
 importFrom(reticulate,use_python)
 importFrom(reticulate,use_virtualenv)
 importFrom(rlang,"%||%")
+importFrom(rlang,.data)
 importFrom(rlang,names2)
 importFrom(stats,predict)
 importFrom(tensorflow,as_tensor)

NEWS.md

@@ -1,5 +1,24 @@
 # keras (development version)
 
+- Default TensorFlow version installed by `install_keras()` is now 2.11. 
+
+- All optimizers have been updated for keras/tensorflow version 2.11.
+  Arguments to all the optimizers have changed. To access the previous
+  optimizer implementations, use the constructors available at
+  `keras$optimizers$legacy`. For example, use `keras$optimizers$legacy$Adam()`
+  for the previous implementation of `optimizer_adam()`.
+
+- New optimizer `optimizer_frtl()`.
+
+- updates to layers:
+  - `layer_attention()` gains `score_mode` and `dropout` arguments.
+  - `layer_discretization()` gains `output_mode` and `sparse` arguments.
+  - `layer_gaussian_dropout()` and `layer_gaussian_noise()` gain a `seed` argument.
+  - `layer_hashing()` gains `output_mode` and `sparse` arguments.
+  - `layer_integer_lookup()` gains `vocabulary_dtype` and `idf_weights` arguments.
+  - `layer_normalization()` gains an `invert` argument.
+  - `layer_string_lookup()` gains an `idf_weights` argument.
+
 - Fixed issue where `input_shape` supplied to custom layers defined with `new_layer_class()`  
   would result in an error (#1338)
 

R/history.R

@@ -59,6 +59,8 @@ print.keras_training_history <- function(x, ...) {
 #'   black and white.
 #' @param ... Additional parameters to pass to the [plot()] method.
 #'
+#' @importFrom rlang .data
+#'
 #' @export
 plot.keras_training_history <- function(x, y, metrics = NULL, method = c("auto", "ggplot2", "base"),
                                         smooth = getOption("keras.plot.history.smooth", TRUE),
@@ -95,11 +97,11 @@ plot.keras_training_history <- function(x, y, metrics = NULL, method = c("auto",
 
     if (do_validation) {
       if (theme_bw)
-        p <- ggplot2::ggplot(df, ggplot2::aes_(~epoch, ~value, color = ~data, fill = ~data, linetype = ~data, shape = ~data))
+        p <- ggplot2::ggplot(df, ggplot2::aes(.data$epoch, .data$value, color = .data$data, fill = .data$data, linetype = .data$data, shape = .data$data))
       else
-        p <- ggplot2::ggplot(df, ggplot2::aes_(~epoch, ~value, color = ~data, fill = ~data))
+        p <- ggplot2::ggplot(df, ggplot2::aes(.data$epoch, .data$value, color = .data$data, fill = .data$data))
     } else {
-      p <- ggplot2::ggplot(df, ggplot2::aes_(~epoch, ~value))
+      p <- ggplot2::ggplot(df, ggplot2::aes(.data$epoch, .data$value))
     }
 
     smooth_args <- list(se = FALSE, method = 'loess', na.rm = TRUE,

R/install.R

@@ -58,7 +58,7 @@ install_keras <- function(method = c("auto", "virtualenv", "conda"),
   )
 }
 
-default_version <- numeric_version("2.10")
+default_version <- numeric_version("2.11")
 
 default_extra_packages <- function(tensorflow_version = "default") {
   pkgs <- c(

R/layer-attention.R

@@ -1,35 +1,58 @@
 
-#' Creates attention layer
+
+#' Dot-product attention layer, a.k.a. Luong-style attention
 #'
-#' Dot-product attention layer, a.k.a. Luong-style attention.
+#' @details
+#' inputs are `query` tensor of shape `[batch_size, Tq, dim]`, `value` tensor
+#' of shape `[batch_size, Tv, dim]` and `key` tensor of shape
+#' `[batch_size, Tv, dim]`. The calculation follows the steps:
 #'
-#' @inheritParams layer_dense
+#' 1. Calculate scores with shape `[batch_size, Tq, Tv]` as a `query`-`key` dot
+#'    product: `scores = tf$matmul(query, key, transpose_b=TRUE)`.
+#' 2. Use scores to calculate a distribution with shape
+#'    `[batch_size, Tq, Tv]`: `distribution = tf$nn$softmax(scores)`.
+#' 3. Use `distribution` to create a linear combination of `value` with
+#'    shape `[batch_size, Tq, dim]`:
+#'    return `tf$matmul(distribution, value)`.
+#'
+#'
+#' @param inputs List of the following tensors:
+#'
+#'   -  query: Query Tensor of shape `[batch_size, Tq, dim]`.
+#'
+#'   -  value: Value Tensor of shape `[batch_size, Tv, dim]`.
 #'
-#' @param inputs a list of inputs first should be the query tensor, the second the value tensor
-#' @param use_scale If True, will create a scalar variable to scale the attention scores.
-#' @param causal Boolean. Set to True for decoder self-attention. Adds a mask such that position i cannot attend to positions j > i.
-#' This prevents the flow of information from the future towards the past.
+#'   -  key: Optional key Tensor of shape `[batch_size, Tv, dim]`. If not
+#'   given, will use value for both key and value, which is the most common
+#'   case.
+#'
+#' @param use_scale If `TRUE`, will create a scalar variable to scale the attention
+#' scores.
+#'
+#' @param dropout Float between 0 and 1. Fraction of the units to drop for the
+#' attention scores. Defaults to 0.0.
+#'
+#' @param score_mode Function to use to compute attention scores, one of
+#' `{"dot", "concat"}`. `"dot"` refers to the dot product between the query
+#' and key vectors. `"concat"` refers to the hyperbolic tangent of the
+#' concatenation of the query and key vectors.
+#'
+#' @param ... standard layer arguments (e.g., batch_size, dtype, name, trainable, weights)
 #'
 #' @family core layers
 #' @family attention layers
 #'
+#'
+#' @seealso
+#'   +  <https://www.tensorflow.org/api_docs/python/tf/keras/layers/Attention>
 #' @export
-layer_attention <- function(inputs,use_scale=FALSE, causal = FALSE, batch_size = NULL, dtype = NULL,
-                                name = NULL, trainable = NULL, weights = NULL) {
-  if (!is_tensorflow_implementation() || !tensorflow::tf_version() >= "1.14")
-    stop("layer_dense_features requires TensorFlow implementation and version >= 1.14")
-  create_layer(keras$layers$Attention, inputs, list(
-      use_scale = use_scale,
-      causal = causal,
-      batch_size = batch_size,
-      dtype = dtype,
-      name = name,
-      trainable = trainable,
-      weights = weights)
-    )
-
-
-}
+layer_attention <-
+  function(inputs, use_scale = FALSE, score_mode = "dot", ..., dropout = NULL)
+  {
+    args <- capture_args(match.call(), ignore = "inputs")
+    args$dropout <- dropout
+    create_layer(keras$layers$Attention, inputs, args)
+  }
 
 #' MultiHeadAttention layer
 #'

R/layers-noise.R

@@ -11,6 +11,10 @@
 #'
 #' @param stddev float, standard deviation of the noise distribution.
 #'
+#' @param seed Integer, optional random seed to enable deterministic behavior.
+#'
+#' @param ... standard layer arguments.
+#'
 #' @section Input shape: Arbitrary. Use the keyword argument `input_shape` (list
 #'   of integers, does not include the samples axis) when using this layer as
 #'   the first layer in a model.
@@ -20,19 +24,14 @@
 #' @family noise layers
 #'
 #' @export
-layer_gaussian_noise <- function(object, stddev, input_shape = NULL,
-                                 batch_input_shape = NULL, batch_size = NULL, dtype = NULL,
-                                 name = NULL, trainable = NULL, weights = NULL) {
-  create_layer(keras$layers$GaussianNoise, object, list(
-    stddev = stddev,
-    input_shape = normalize_shape(input_shape),
-    batch_input_shape = normalize_shape(batch_input_shape),
-    batch_size = as_nullable_integer(batch_size),
-    dtype = dtype,
-    name = name,
-    trainable = trainable,
-    weights = weights
-  ))
+layer_gaussian_noise <-
+function(object, stddev, seed = NULL, ...)
+{
+  args <- capture_args(match.call(),
+    modifiers = c(standard_layer_arg_modifiers,
+                  seed = as_nullable_integer),
+    ignore = "object")
+  create_layer(keras$layers$GaussianNoise, object, args)
 }
 
 #' Apply multiplicative 1-centered Gaussian noise.
@@ -44,6 +43,10 @@ layer_gaussian_noise <- function(object, stddev, input_shape = NULL,
 #' @param rate float, drop probability (as with `Dropout`). The multiplicative
 #'   noise will have standard deviation `sqrt(rate / (1 - rate))`.
 #'
+#' @param seed Integer, optional random seed to enable deterministic behavior.
+#'
+#' @param ... standard layer arguments.
+#'
 #' @section Input shape: Arbitrary. Use the keyword argument `input_shape` (list
 #'   of integers, does not include the samples axis) when using this layer as
 #'   the first layer in a model.
@@ -53,25 +56,24 @@ layer_gaussian_noise <- function(object, stddev, input_shape = NULL,
 #' @section References:
 #' - [Dropout: A Simple Way to Prevent Neural Networks from Overfitting Srivastava, Hinton, et al. 2014](https://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf)
 #'
+#' @seealso
+#'   +  <https://www.tensorflow.org/api_docs/python/tf/keras/layers/GaussianDropout>
+#'
 #' @family noise layers
 #'
 #' @export
-layer_gaussian_dropout <- function(object, rate, input_shape = NULL,
-                                   batch_input_shape = NULL, batch_size = NULL, dtype = NULL,
-                                   name = NULL, trainable = NULL, weights = NULL) {
-  create_layer(keras$layers$GaussianDropout, object, list(
-    rate = rate,
-    input_shape = normalize_shape(input_shape),
-    batch_input_shape = normalize_shape(batch_input_shape),
-    batch_size = as_nullable_integer(batch_size),
-    dtype = dtype,
-    name = name,
-    trainable = trainable,
-    weights = weights
-  ))
+layer_gaussian_dropout <-
+function(object, rate, seed = NULL, ...)
+{
+  args <- capture_args(match.call(),
+    modifiers = c(standard_layer_arg_modifiers,
+                  seed = as_nullable_integer),
+    ignore = "object")
+  create_layer(keras$layers$GaussianDropout, object, args)
 }
 
 
+
 #' Applies Alpha Dropout to the input.
 #'
 #' Alpha Dropout is a dropout that keeps mean and variance of inputs to their