check final ru satisfies target ru in all mp tests

Author: irenab

tests/keras_tests/feature_networks_tests/feature_networks/mixed_precision/requires_mixed_precision_test.py

@@ -92,7 +92,7 @@ def get_resource_utilization(self):
                                    total_memory=ru_data.total_memory - 1 if self.total_memory else np.inf,
                                    bops=int(ru_data.bops * 0.05) if self.bops else np.inf)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         if self.weights_memory or self.activation_memory or self.total_memory or self.bops:
             self.unit_test.assertTrue(any([i != 0 for i in quantization_info.mixed_precision_cfg]))
         else:

tests/keras_tests/feature_networks_tests/feature_networks/mixed_precision_tests.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
+import typing
 
+import abc
 
 import numpy as np
 import tensorflow as tf
@@ -94,10 +96,18 @@ def create_networks(self):
         model = keras.Model(inputs=inputs, outputs=outputs)
         return model
 
+    @typing.final
     def compare(self, quantized_model, float_model, input_x=None, quantization_info: UserInformation = None):
-        # This is a base test, so it does not check a thing. Only actual tests of mixed precision
-        # compare things to test.
-        raise NotImplementedError
+        # call concrete validation of the specific test
+        self._compare(quantized_model, float_model, input_x, quantization_info)
+        # make sure the final utilization satisfies the target constraints
+        self.unit_test.assertTrue(
+            self.get_resource_utilization().is_satisfied_by(quantization_info.final_resource_utilization))
+
+    @abc.abstractmethod
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info: UserInformation = None):
+        # test-specific validation, to be implemented by each test
+        raise NotImplementedError()
 
     def verify_quantization(self, quantized_model, input_x, weights_layers_idx, weights_layers_channels_size,
                             activation_layers_idx, unique_tensor_values):
@@ -120,8 +130,6 @@ def verify_quantization(self, quantized_model, input_x, weights_layers_idx, weig
         # verifying fake quant nodes output
         for layer_out in layer_outs:
             self.unit_test.assertTrue(np.unique(layer_out).flatten().shape[0] <= unique_tensor_values)
-
-
 class MixedPrecisionActivationSearchTest(MixedPrecisionActivationBaseTest):
     def __init__(self, unit_test):
         super().__init__(unit_test, activation_layers_idx=[1, 2, 4])
@@ -274,7 +282,7 @@ def create_networks(self):
     def get_resource_utilization(self):
         return ResourceUtilization(3071, 2079)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         # verify chosen activation bitwidth config
         # resource utilization is infinity -> should give best model - 8bits
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)
@@ -314,7 +322,7 @@ def get_tpc(self):
     def get_resource_utilization(self):
         return ResourceUtilization(activation_memory=6507)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         # verify chosen activation bitwidth config
         # resource utilization is infinity -> should give best model - 8bits
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)
@@ -361,7 +369,7 @@ def get_tpc(self):
     def get_resource_utilization(self):
         return ResourceUtilization(np.inf, 6407)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         # verify chosen activation bitwidth config
         # resource utilization is infinity -> should give best model - 8bits
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)
@@ -389,7 +397,7 @@ def create_networks(self):
         model = keras.Model(inputs=inputs, outputs=x)
         return model
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         # verify chosen activation bitwidth config
         # resource utilization is infinity -> should give best model - 8bits
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)
@@ -436,7 +444,7 @@ def create_networks(self):
         model = keras.Model(inputs=[inputs_1, inputs_2, inputs_3, inputs_4], outputs=outputs)
         return model
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         # verify chosen activation bitwidth config
         # resource utilization is infinity -> should give best model - 8bits
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)
@@ -458,7 +466,7 @@ def get_resource_utilization(self):
         # 17920: 8-bit weights, 6176: max cut of input+conv_bn
         return ResourceUtilization(np.inf, np.inf, total_memory=(17920 + 6176) * 4 / 8)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info: UserInformation = None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info: UserInformation = None):
         # verify chosen activation bitwidth config
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)[1:]
         activation_bits = [layer.activation_holder_quantizer.get_config()['num_bits'] for layer in holder_layers]
@@ -480,7 +488,7 @@ def get_resource_utilization(self):
         activation = 6176 * 4 / 8
         return ResourceUtilization(weights, activation, total_memory=weights + activation)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info: UserInformation = None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info: UserInformation = None):
         # verify chosen activation bitwidth config
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)[1:]
         activation_bits = [layer.activation_holder_quantizer.get_config()['num_bits'] for layer in holder_layers]
@@ -509,7 +517,7 @@ def get_resource_utilization(self):
         activation = 6176 * 4 / 8    # max cut of input + conv_bn
         return ResourceUtilization(weights, activation, total_memory=(weights + activation) / 2)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info: UserInformation = None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info: UserInformation = None):
         # verify chosen activation bitwidth config
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)[1:]
         activation_bits = [layer.activation_holder_quantizer.get_config()['num_bits'] for layer in holder_layers]
@@ -565,7 +573,7 @@ def create_networks(self):
         model = keras.Model(inputs=inputs, outputs=x)
         return model
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         # verify chosen activation bitwidth config
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)
         activation_bits = [layer.activation_holder_quantizer.get_config()['num_bits'] for layer in holder_layers]
@@ -602,7 +610,7 @@ def create_networks(self):
         model = keras.Model(inputs=inputs, outputs=x)
         return model
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         # verify chosen activation bitwidth config
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)
         activation_bits = [layer.activation_holder_quantizer.get_config()['num_bits'] for layer in
@@ -668,7 +676,7 @@ def get_tpc(self):
     def get_resource_utilization(self):
         return ResourceUtilization(np.inf, 5410)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         holder_layers = get_layers_from_model_by_type(quantized_model, KerasActivationQuantizationHolder)
 
         activation_bits = [layer.activation_holder_quantizer.get_config()['num_bits'] for layer in holder_layers]

tests/keras_tests/feature_networks_tests/feature_networks/weights_mixed_precision_tests.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
+import abc
 
+import typing
 
 import numpy as np
 import tensorflow as tf
@@ -65,10 +67,23 @@ def create_networks(self):
         model = keras.Model(inputs=inputs, outputs=outputs)
         return model
 
+    def get_resource_utilization(self):
+        raise NotImplementedError()
+
+    @typing.final
     def compare(self, quantized_model, float_model, input_x=None, quantization_info: UserInformation = None):
-        # This is a base test, so it does not check a thing. Only actual tests of mixed precision
-        # compare things to test.
-        raise NotImplementedError
+        # call concrete validation of the specific test
+        self._compare(quantized_model, float_model, input_x, quantization_info)
+        # make sure the final utilization satisfies the target constraints
+        target_ru = self.get_resource_utilization()
+        if target_ru.is_any_restricted():
+            self.unit_test.assertTrue(
+                target_ru.is_satisfied_by(quantization_info.final_resource_utilization))
+
+    @abc.abstractmethod
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info: UserInformation = None):
+        # test-specific validation, to be implemented by each test
+        raise NotImplementedError()
 
 
 class MixedPrecisionManuallyConfiguredTest(MixedPrecisionBaseTest):
@@ -95,7 +110,7 @@ def get_resource_utilization(self):
         # set manually)
         return ResourceUtilization(1)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         assert quantization_info.mixed_precision_cfg == [2, 1]
         conv_layers = get_layers_from_model_by_type(quantized_model, layers.Conv2D)
         self.unit_test.assertTrue(np.unique(conv_layers[0].weights[0]).flatten().shape[0] <= 4)
@@ -114,7 +129,7 @@ def get_mixed_precision_config(self):
         return mct.core.MixedPrecisionQuantizationConfig(num_of_images=1,
                                                          distance_weighting_method=self.distance_metric)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         conv_layers = get_layers_from_model_by_type(quantized_model, layers.Conv2D)
         self.unit_test.assertTrue(any([b != 0 for b in quantization_info.mixed_precision_cfg]),
                                   "At least one of the conv layers is expected to be quantized to meet the required "
@@ -147,7 +162,7 @@ def get_mixed_precision_config(self):
                                                          distance_weighting_method=self.distance_metric,
                                                          use_hessian_based_scores=True)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         conv_layers = get_layers_from_model_by_type(quantized_model, layers.Conv2D)
         self.unit_test.assertTrue(any([b != 0 for b in quantization_info.mixed_precision_cfg]),
                                   "At least one of the conv layers is expected to be quantized to meet the required "
@@ -220,7 +235,7 @@ def create_networks(self):
     def get_resource_utilization(self):
         return ResourceUtilization(1790)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         # We just needed to verify that the graph finalization is working without failing.
         # The actual quantization is not interesting for the sake of this test, so we just verify some
         # degenerated things to see that everything worked.
@@ -242,7 +257,7 @@ def get_resource_utilization(self):
         # Resource Utilization is for 4 bits on average
         return ResourceUtilization(17920 * 4 / 8)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         conv_layers = get_layers_from_model_by_type(quantized_model, layers.Conv2D)
         assert (quantization_info.mixed_precision_cfg == [1, 1]).all()
         for i in range(32):  # quantized per channel
@@ -283,7 +298,7 @@ def create_networks(self):
         model = keras.Model(inputs=inputs, outputs=outputs)
         return model
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         conv_layers = get_layers_from_model_by_type(quantized_model, layers.Conv2D)
         self.unit_test.assertTrue((quantization_info.mixed_precision_cfg != 0).any())
 
@@ -308,7 +323,7 @@ def get_resource_utilization(self):
         # Resource Utilization is for 2 bits on average
         return ResourceUtilization(17920 * 2 / 8)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         conv_layers = get_layers_from_model_by_type(quantized_model, layers.Conv2D)
         assert (quantization_info.mixed_precision_cfg == [2, 2]).all()
         for i in range(32):  # quantized per channel
@@ -335,7 +350,7 @@ def __init__(self, unit_test):
     def get_resource_utilization(self):
         return self.target_total_ru
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         # No need to verify quantization configuration here since this test is similar to other tests we have,
         # we're only interested in the ResourceUtilization
         self.unit_test.assertTrue(quantization_info.final_resource_utilization.activation_memory <=
@@ -351,7 +366,7 @@ def __init__(self, unit_test):
     def get_resource_utilization(self):
         return self.target_total_ru
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         # No need to verify quantization configuration here since this test is similar to other tests we have,
         # we're only interested in the ResourceUtilization
         self.unit_test.assertTrue(
@@ -373,7 +388,7 @@ def create_networks(self):
         model = keras.Model(inputs=inputs, outputs=x)
         return model
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         self.unit_test.assertTrue(len(quantization_info.mixed_precision_cfg) == 1)
         self.unit_test.assertTrue(quantization_info.mixed_precision_cfg[0] == 1)
 
@@ -426,7 +441,7 @@ def get_resource_utilization(self):
         # resource utilization is infinity -> should give best model - 8bits
         return ResourceUtilization(17919)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         conv_layers = get_layers_from_model_by_type(quantized_model, layers.Conv2D)
         assert (quantization_info.mixed_precision_cfg == [0, 1]).all()
         for i in range(32):  # quantized per channel
@@ -449,7 +464,7 @@ def get_mixed_precision_config(self):
     def get_resource_utilization(self):
         return ResourceUtilization(17919)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         conv_layers = get_layers_from_model_by_type(quantized_model, layers.Conv2D)
         assert any([(quantization_info.mixed_precision_cfg == [1, 0]).all(),
                     (quantization_info.mixed_precision_cfg == [0, 1]).all()])
@@ -526,7 +541,7 @@ def get_tpc(self):
     def get_resource_utilization(self):
         return ResourceUtilization(1535)
 
-    def compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
+    def _compare(self, quantized_model, float_model, input_x=None, quantization_info=None):
         wrapper_layers = get_layers_from_model_by_type(quantized_model, KerasQuantizationWrapper)
         weights_bits = wrapper_layers[0].weights_quantizers[KERNEL].num_bits
         self.unit_test.assertTrue(weights_bits == 4)