Weight compression API implementation for simplest case where original weights and graph

differ from training/inference weights and graph, but training/inference graphs
are the same.

Test that weights are converted and that pretrained weights are preserved. The
TFLite prevention of constant folding currently doesn't work.

PiperOrigin-RevId: 338138622

Author: alanchiao

tensorflow_model_optimization/python/core/common/keras/compression/BUILD

@@ -0,0 +1,14 @@
+licenses(["notice"])
+
+py_library(
+    name = "algorithm",
+    srcs = ["algorithm.py"],
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow_model_optimization/python/core/common/keras/compression/internal:optimize",
+        #TODO(tfmot): remove when we stick to Python 3.7, which includes this by default.
+        # dataclasses dep1,
+        # tensorflow dep1,
+    ],
+)

tensorflow_model_optimization/python/core/common/keras/compression/README.md

@@ -0,0 +1,9 @@
+These APIs are not ready for public usage yet.
+
+algorithms/ : end-to-end tests that demonstrate usage of algorithm developer
+public API.
+
+algorithm.py : public API for algorithm developer
+
+internal/ : internal parts of library. These should not be used anywhere outside
+this directory.

tensorflow_model_optimization/python/core/common/keras/compression/algorithm.py

@@ -0,0 +1,98 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Public APIs for algorithm developer using weight compression API."""
+import abc
+from typing import List, Any
+import dataclasses
+
+import tensorflow as tf
+
+from tensorflow_model_optimization.python.core.common.keras.compression.internal import optimize
+
+
+@dataclasses.dataclass
+class WeightRepr:
+  """Dataclass that wraps `tf.keras.layers.Layer.add_weight` parameters."""
+  name: Any = None
+  shape: Any = None
+  dtype: Any = None
+  initializer: Any = None
+  regularizer: Any = None
+  trainable: Any = None
+  constraint: Any = None
+  partitioner: Any = None
+  use_resource: Any = None
+  synchronization: Any = tf.VariableSynchronization.AUTO
+  aggregation: Any = tf.compat.v1.VariableAggregation.NONE
+
+
+class WeightCompressionAlgorithm(metaclass=abc.ABCMeta):
+  """Interface for weight compression algorithm that acts on a per-layer basis.
+
+     This allows both options of either decompressing during inference or
+     decompressing prior to inference (where compression occurs by applying a
+     tool such as zip to the model file).
+
+     This interface is a purely functional one.
+  """
+
+  @abc.abstractmethod
+  def init_training_weights_repr(
+      self, pretrained_weight: tf.Tensor) -> List[WeightRepr]:
+    """Create training weight representations for initializing layer variables.
+
+    Args:
+      pretrained_weight: tf.Tensor of a pretrained weight of a layer that will
+        be compressed eventually.
+
+    Returns:
+      A list of `WeightRepr`, a container for arguments to
+      `tf.keras.layers.Layer.add_weight`for each tf.Variable to create.
+    """
+
+  def decompress(self, compressed_weights: List[tf.Tensor]) -> tf.Tensor:
+    """Define the operations to decompress a single weight’s compressed form during inference.
+
+    The default is an identity. TODO(): actually isn't.
+
+    Args:
+       compressed_weights: tf.Tensors representing a single weight’s compressed
+         form, coming from what’s returned in `compress`.
+
+    Returns:
+      A tf.Tensor representing the decompressed `compressed_weights`.
+    """
+    return compressed_weights[0]
+
+  @abc.abstractmethod
+  def training(self, training_weights: List[tf.Tensor]) -> tf.Tensor:
+    """Define a piece of the forward pass during training, which operates on a single compressible weight.
+
+    The default throws an error when training occurs.
+
+    Args:
+       training_weights: tf.Tensors representing any variables used during
+         training, for a single compressible weight, in the order returned in
+         `init_training_weights_repr`.
+
+    Returns:
+       tf.Tensor to set the compressible weight to.
+    """
+
+
+def create_layer_for_training(
+    layer: tf.keras.layers.Layer,
+    algorithm: WeightCompressionAlgorithm) -> tf.keras.layers.Layer:
+  return optimize.create_layer_for_training(layer, algorithm)

tensorflow_model_optimization/python/core/common/keras/compression/algorithms/BUILD

@@ -0,0 +1,24 @@
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])
+
+py_library(
+    name = "same_training_and_inference",
+    srcs = ["same_training_and_inference.py"],
+    srcs_version = "PY3",
+    deps = [
+        # tensorflow dep1,
+        "//tensorflow_model_optimization/python/core/common/keras/compression:algorithm",
+    ],
+)
+
+py_test(
+    name = "same_training_and_inference_test",
+    srcs = ["same_training_and_inference_test.py"],
+    python_version = "PY3",
+    deps = [
+        ":same_training_and_inference",
+        # numpy dep1,
+        # tensorflow dep1,
+    ],
+)

tensorflow_model_optimization/python/core/common/keras/compression/algorithms/same_training_and_inference.py

@@ -0,0 +1,95 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""SVD algorithm, where the training and inference graphs are the same."""
+from typing import List
+
+import tensorflow as tf
+
+from tensorflow_model_optimization.python.core.common.keras.compression import algorithm
+
+
+class SVDParams(object):
+  """Define container for parameters for SVD algorithm."""
+
+  def __init__(self, rank):
+    self.rank = rank
+
+
+class SVD(algorithm.WeightCompressionAlgorithm):
+  """Define how to apply SVD algorithm."""
+
+  def __init__(self, params):
+    self.params = params
+
+  # TODO(tfmot): communicate that `pretrained_weight` will sometimes
+  # be a dummy tensor and sometimes be actual pretrained values during
+  # its actual usage.
+  def init_training_weights_repr(
+      self, pretrained_weight: tf.Tensor) -> List[algorithm.WeightRepr]:
+    rank = self.params.rank
+    s, u, v = tf.linalg.svd(pretrained_weight)
+
+    if len(pretrained_weight.shape) == 2:
+      # FC Layer
+      s = s[:rank]
+      u = u[:, :rank]
+      v = v[:, :rank]
+    elif len(pretrained_weight.shape) == 4:
+      # Conv2D Layer
+      s = s[:, :, :rank]
+      u = u[:, :, :, :rank]
+      v = v[:, :, :, :rank]
+    else:
+      raise NotImplementedError('Only for dimension=2 or 4 is supported.')
+
+    sv = tf.matmul(tf.linalg.diag(s), v, adjoint_b=True)
+
+    # TODO(tfmot): note that it does not suffice to just have the initializer
+    # to derive the shape from, in the case of a constant initializer.
+    # The unit test fail without providing the shape.
+    return [
+        algorithm.WeightRepr(
+            name='u',
+            shape=u.shape,
+            initializer=tf.keras.initializers.Constant(u)),
+        algorithm.WeightRepr(
+            name='sv',
+            shape=sv.shape,
+            initializer=tf.keras.initializers.Constant(sv))
+    ]
+
+  def decompress(self, u: tf.Tensor, sv: tf.Tensor) -> tf.Tensor:
+    return tf.matmul(u, sv)
+
+  def training(self, training_weights: List[tf.Tensor]) -> tf.Tensor:
+    u = training_weights[0]
+    sv = training_weights[1]
+    return self.decompress(u, sv)
+
+
+def optimize(to_optimize: tf.keras.Model, params: SVDParams) -> tf.keras.Model:
+  """Model developer API for optimizing a model."""
+
+  def _optimize_layer(layer):
+    # Require layer to be built so that the SVD-factorized weights
+    # can be initialized from the weights.
+    if not layer.built:
+      raise ValueError(
+          'Applying SVD currently requires passing in a built model')
+
+    return algorithm.create_layer_for_training(layer, algorithm=SVD(params))
+
+  return tf.keras.models.clone_model(
+      to_optimize, clone_function=_optimize_layer)