Increase algorithm coverage for algorithms that need to modify the weights after training (compress API) or have an inference graph that differs from the training graph (decompress vs training).

PiperOrigin-RevId: 338149184

Author: alanchiao

tensorflow_model_optimization/python/core/common/keras/compression/algorithm.py

@@ -62,6 +62,24 @@ def init_training_weights_repr(
       `tf.keras.layers.Layer.add_weight`for each tf.Variable to create.
     """
 
+  def compress(self, training_weights: List[tf.Tensor]) -> List[tf.Tensor]:
+    """Define the operations to compress a single weight after training.
+
+    'Compress' can refer to making the weight more amenable to compression
+    or actually compress the weight.
+
+    The default is an identity.
+
+    Args:
+      training_weights: tf.Tensors representing all variables used during
+        training, for a single compressible weight, in the order returned in
+        `init_training_weights_repr`.
+
+    Returns:
+      List of tf.Tensors to set to compressed or more compressible form.
+    """
+    return training_weights
+
   def decompress(self, compressed_weights: List[tf.Tensor]) -> tf.Tensor:
     """Define the operations to decompress a single weight’s compressed form during inference.
 
@@ -80,6 +98,7 @@ def decompress(self, compressed_weights: List[tf.Tensor]) -> tf.Tensor:
   def training(self, training_weights: List[tf.Tensor]) -> tf.Tensor:
     """Define a piece of the forward pass during training, which operates on a single compressible weight.
 
+    TODO(tfmot): throw this error.
     The default throws an error when training occurs.
 
     Args:
@@ -96,3 +115,9 @@ def create_layer_for_training(
     layer: tf.keras.layers.Layer,
     algorithm: WeightCompressionAlgorithm) -> tf.keras.layers.Layer:
   return optimize.create_layer_for_training(layer, algorithm)
+
+
+def create_layer_for_inference(
+    layer_for_training: tf.keras.layers.Layer,
+    algorithm: WeightCompressionAlgorithm) -> tf.keras.layers.Layer:
+  return optimize.create_layer_for_inference(layer_for_training, algorithm)

tensorflow_model_optimization/python/core/common/keras/compression/algorithms/BUILD

@@ -22,3 +22,24 @@ py_test(
         # tensorflow dep1,
     ],
 )
+
+py_library(
+    name = "different_training_and_inference",
+    srcs = ["different_training_and_inference.py"],
+    srcs_version = "PY3",
+    deps = [
+        # tensorflow dep1,
+        "//tensorflow_model_optimization/python/core/common/keras/compression:algorithm",
+    ],
+)
+
+py_test(
+    name = "different_training_and_inference_test",
+    srcs = ["different_training_and_inference_test.py"],
+    python_version = "PY3",
+    deps = [
+        ":different_training_and_inference",
+        # numpy dep1,
+        # tensorflow dep1,
+    ],
+)

tensorflow_model_optimization/python/core/common/keras/compression/algorithms/different_training_and_inference.py

@@ -0,0 +1,101 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""SVD algorithm, where the training and inference graphs are different."""
+from typing import List
+
+import tensorflow as tf
+
+from tensorflow_model_optimization.python.core.common.keras.compression import algorithm
+
+
+class SVDParams(object):
+  """Define container for parameters for SVD algorithm."""
+
+  def __init__(self, rank):
+    self.rank = rank
+
+
+class SVD(algorithm.WeightCompressionAlgorithm):
+  """Define how to apply SVD algorithm."""
+
+  def __init__(self, params):
+    self.params = params
+
+  # TODO(tfmot): communicate that `pretrained_weight` will sometimes
+  # be a dummy tensor and sometimes be actual pretrained values during
+  # its actual usage.
+  def init_training_weights_repr(
+      self, pretrained_weight: tf.Tensor) -> List[algorithm.WeightRepr]:
+    return [
+        algorithm.WeightRepr(
+            name='w',
+            shape=pretrained_weight.shape,
+            initializer=tf.keras.initializers.Constant(pretrained_weight))
+    ]
+
+  def decompress(self, u: tf.Tensor, sv: tf.Tensor) -> tf.Tensor:
+    return tf.matmul(u, sv)
+
+  def compress(self, training_weights: List[tf.Tensor]) -> List[tf.Tensor]:
+    assert len(training_weights) == 1
+    weight = training_weights[0]
+
+    rank = self.params.rank
+    s, u, v = tf.linalg.svd(weight)
+
+    if len(weight.shape) == 2:
+      # FC Layer
+      s = s[:rank]
+      u = u[:, :rank]
+      v = v[:, :rank]
+    elif len(weight.shape) == 4:
+      # Conv2D Layer
+      s = s[:, :, :rank]
+      u = u[:, :, :, :rank]
+      v = v[:, :, :, :rank]
+    else:
+      raise NotImplementedError('Only for dimension=2 or 4 is supported.')
+
+    sv = tf.matmul(tf.linalg.diag(s), v, adjoint_b=True)
+
+    return [u, sv]
+
+  # TODO(tfmot): remove in this example, which is just post-training.
+  def training(self, training_weights: List[tf.Tensor]) -> tf.Tensor:
+    return training_weights[0]
+
+
+# TODO(tfmot): consider if we can simplify `create_model_for_training` and
+# `create_model_for_inference` into a single API for algorithm developers.
+def optimize(to_optimize: tf.keras.Model, params: SVDParams) -> tf.keras.Model:
+  """Model developer API for optimizing a model."""
+
+  def _create_layer_for_training(layer):
+    # Require layer to be built so that the SVD-factorized weights
+    # can be initialized from the weights.
+    if not layer.built:
+      raise ValueError(
+          'Applying SVD currently requires passing in a built model')
+
+    return algorithm.create_layer_for_training(layer, algorithm=SVD(params))
+
+  def _create_layer_for_inference(layer):
+    return algorithm.create_layer_for_inference(layer, algorithm=SVD(params))
+
+  intermediate_model = tf.keras.models.clone_model(
+      to_optimize, clone_function=_create_layer_for_training)
+
+  return tf.keras.models.clone_model(
+      intermediate_model, clone_function=_create_layer_for_inference)