fix: Mixed type quantized matmul for after configure quantization update

If the quantization information is required to be updated after configure, in some IPs, the mixed data type support does not return the correct results. This is because if there is no native mixed quantized kernel support in arm_gemm, the operator changes the sign of Lhs. This wasn't accounted correctly in the after configure update.

Resolves: COMPMID-7833
Change-Id: I88bb88921c569a9ef3b4aa49d09015f9e10247e8
Signed-off-by: Gunes Bayir <[email protected]>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/14670
Benchmark: Arm Jenkins <[email protected]>
Reviewed-by: Dennis Wildmark <[email protected]>
Comments-Addressed: Arm Jenkins <[email protected]>
Tested-by: Arm Jenkins <[email protected]>

Author: gunes-arm

src/cpu/operators/CpuGemmLowpMatrixMultiplyCore.cpp

@@ -810,7 +810,21 @@ void CpuGemmLowpMatrixMultiplyCore::update_quantization_parameters(const GEMMLow
 {
     auto lowp_os = output_info;
     _gemm_info.set_gemmlowp_output_stage(lowp_os);
-    _asm_glue->update_quantization_parameters(output_info, a, b, is_prepared, negated_offsets);
+
+    const QuantizationInfo *a_to_use = &a;
+    QuantizationInfo        a_signed;
+
+    if (_flip_signedness)
+    {
+        const int32_t                 offset_correction = 128;
+        const UniformQuantizationInfo a_uniform         = a.uniform();
+
+        ARM_COMPUTE_ERROR_ON(a.scale().size() > 1);
+        a_signed = QuantizationInfo(a_uniform.scale, a_uniform.offset + offset_correction);
+        a_to_use = &a_signed;
+    }
+
+    _asm_glue->update_quantization_parameters(output_info, *a_to_use, b, is_prepared, negated_offsets);
     _is_prepared = is_prepared;
 }
 } // namespace cpu

src/runtime/NEON/functions/NEGEMMLowpMatrixMultiplyCore.cpp

@@ -146,7 +146,8 @@ void NEGEMMLowpMatrixMultiplyCore::update_quantization_parameters()
     output_info.gemmlowp_max_bound       = max_activation;
     output_info.is_quantized_per_channel = false;
     output_info.output_data_type         = dst->info()->data_type();
-    quantization::calculate_quantized_multipliers(iqinfo, wqinfo, oqinfo, output_info);
+    const Status status = quantization::calculate_quantized_multipliers(iqinfo, wqinfo, oqinfo, output_info);
+    ARM_COMPUTE_ERROR_ON(!bool(status));
 
     _impl->op->update_quantization_parameters(output_info, src->info()->quantization_info(),
                                               wei->info()->quantization_info(), true, true);

tests/validation/NEON/GEMMLowp.cpp

@@ -383,7 +383,8 @@ using NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQuantInfoAfterConfigureInt8Fix
     GEMMLowpGenericMatrixMultiplyCoreFusedOffsetOutputValidationFixture<Tensor, Accessor, NEGEMMLowpMatrixMultiplyCore, false, false, int8_t, int8_t, true>;
 FIXTURE_DATA_TEST_CASE(RunSmall, NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQuantInfoAfterConfigureInt8Fixture, framework::DatasetMode::ALL,
     combine(datasets::SmallGEMMLowpFusedOffsetOutputUint8Dataset(),
-        make("DataType", { DataType::QASYMM8_SIGNED }),
+        make("DataTypeA", { DataType::QASYMM8_SIGNED }),
+        make("DataTypeB", { DataType::QASYMM8_SIGNED }),
         make("reshape_b_only_on_first_run", { false }),
         make("updated_sq_info_after_config", { true }),
         QuantizedActivationFunctionsDataset
@@ -393,7 +394,8 @@ FIXTURE_DATA_TEST_CASE(RunSmall, NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQua
 }
 FIXTURE_DATA_TEST_CASE(RunLarge, NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQuantInfoAfterConfigureInt8Fixture, framework::DatasetMode::NIGHTLY,
     combine(datasets::LargeGEMMLowpFusedOffsetOutputUint8Dataset(),
-        make("DataType", { DataType::QASYMM8_SIGNED }),
+        make("DataTypeA", { DataType::QASYMM8_SIGNED }),
+        make("DataTypeB", { DataType::QASYMM8_SIGNED }),
         make("reshape_b_only_on_first_run", { false }),
         make("updated_sq_info_after_config", { true }),
         QuantizedActivationFunctionsDataset
@@ -408,7 +410,8 @@ using NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQuantInfoAfterConfigureUInt8Fi
     GEMMLowpGenericMatrixMultiplyCoreFusedOffsetOutputValidationFixture<Tensor, Accessor, NEGEMMLowpMatrixMultiplyCore, false, false, uint8_t, uint8_t, true>;
 FIXTURE_DATA_TEST_CASE(RunSmall, NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQuantInfoAfterConfigureUInt8Fixture, framework::DatasetMode::ALL,
     combine(datasets::SmallGEMMLowpFusedOffsetOutputUint8Dataset(),
-        make("DataType", { DataType::QASYMM8 }),
+        make("DataTypeA", { DataType::QASYMM8 }),
+        make("DataTypeB", { DataType::QASYMM8 }),
         make("reshape_b_only_on_first_run", { false }),
         make("updated_sq_info_after_config", { true }),
         QuantizedActivationFunctionsDataset
@@ -418,7 +421,8 @@ FIXTURE_DATA_TEST_CASE(RunSmall, NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQua
 }
 FIXTURE_DATA_TEST_CASE(RunLarge, NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQuantInfoAfterConfigureUInt8Fixture, framework::DatasetMode::NIGHTLY,
     combine(datasets::LargeGEMMLowpFusedOffsetOutputUint8Dataset(),
-        make("DataType", { DataType::QASYMM8 }),
+        make("DataTypeA", { DataType::QASYMM8 }),
+        make("DataTypeB", { DataType::QASYMM8 }),
         make("reshape_b_only_on_first_run", { false }),
         make("updated_sq_info_after_config", { true }),
         QuantizedActivationFunctionsDataset
@@ -427,6 +431,22 @@ FIXTURE_DATA_TEST_CASE(RunLarge, NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQua
     validate(Accessor(_target), _reference, tolerance_batched, large_test_tolerance_num);
 }
 TEST_SUITE_END() // QASYMM8
+
+TEST_SUITE(MixedQuantizedType)
+using NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQuantInfoAfterConfigureInt8Fixture =
+    GEMMLowpGenericMatrixMultiplyCoreFusedOffsetOutputValidationFixture<Tensor, Accessor, NEGEMMLowpMatrixMultiplyCore, false, false, uint8_t, int8_t, true>;
+FIXTURE_DATA_TEST_CASE(RunSmall, NEGEMMLowpMatrixMultiplyCoreForUpdatedStaticQuantInfoAfterConfigureInt8Fixture, framework::DatasetMode::ALL,
+    combine(datasets::SmallGEMMLowpFusedOffsetOutputUint8Dataset(),
+        make("DataTypeA", { DataType::QASYMM8 }),
+        make("DataTypeB", { DataType::QASYMM8_SIGNED }),
+        make("reshape_b_only_on_first_run", { false }),
+        make("updated_sq_info_after_config", { true }),
+        QuantizedActivationFunctionsDataset
+        ))
+{
+    validate(Accessor(_target), _reference, tolerance_batched);
+}
+TEST_SUITE_END() // MixedQuantizedType
 TEST_SUITE_END() // UpdateStaticQuantInfoAfterConfigure
 
 // Deqaunt tests involve returning FP32 from the MatrixMultiplyCore kernels and is only implemented in aarch64

tests/validation/fixtures/GEMMLowpFixture.h

@@ -111,7 +111,7 @@ TensorType compute_gemmlowp_target_for_updated_sq_info_after_config(const Tensor
 {
     ARM_COMPUTE_ASSERT((std::is_same<FunctionType, NEGEMMLowpMatrixMultiplyCore>::value == true));
     ARM_COMPUTE_ASSERT(is_data_type_quantized_asymmetric(data_type_a));
-    ARM_COMPUTE_ASSERT(data_type_a == data_type_b);
+    ARM_COMPUTE_ASSERT(is_data_type_quantized_asymmetric(data_type_b));
 
     // If unknown, set to sensible defaults
     if (data_type_output == DataType::UNKNOWN) {
@@ -531,25 +531,26 @@ class GEMMLowpGenericMatrixMultiplyCoreFusedOffsetOutputValidationFixture : publ
      * 2. The data type is quantized asymmetric
      *
      */
-    void setup(TensorShape shape_a, TensorShape shape_b, TensorShape shape_output, GEMMLowpOutputStageType output_stage_type, DataType data_type,
+    void setup(TensorShape shape_a, TensorShape shape_b, TensorShape shape_output, GEMMLowpOutputStageType output_stage_type, DataType data_type_a, DataType data_type_b,
                bool reshape_b_only_on_first_run, bool updated_sq_info_after_config = false, const ActivationLayerInfo& act_info = ActivationLayerInfo())
     {
         ARM_COMPUTE_ASSERT(output_stage_type != GEMMLowpOutputStageType::NONE);
-        ARM_COMPUTE_ASSERT(is_data_type_quantized_asymmetric(data_type));
+        ARM_COMPUTE_ASSERT(is_data_type_quantized_asymmetric(data_type_a));
+        ARM_COMPUTE_ASSERT(is_data_type_quantized_asymmetric(data_type_b));
 
         // Randomized dynamic quantization: randomize quantization info in a way that ensures no result saturation
         // most of the time
         QuantizationInfo a_qinfo;
         QuantizationInfo b_qinfo;
         QuantizationInfo output_qinfo;
         TensorFillInfo finfo;
-        setup_quantization<TI>(data_type, shape_a, shape_b, a_qinfo, b_qinfo, output_qinfo, finfo);
+        setup_quantization<TI>(data_type_a, shape_a, shape_b, a_qinfo, b_qinfo, output_qinfo, finfo);
 
         GEMMLowpOutputStageInfo output_stage;
-        init_gemmlowp_output_stage_info(data_type, a_qinfo, b_qinfo, output_qinfo, act_info, output_stage_type, output_stage);
+        init_gemmlowp_output_stage_info(data_type_a, a_qinfo, b_qinfo, output_qinfo, act_info, output_stage_type, output_stage);
 
-        _reference = compute_reference(shape_a, shape_b, shape_output, a_qinfo, b_qinfo, data_type, data_type, output_stage, finfo);
-        _target    = compute_target(shape_a, shape_b, shape_output, a_qinfo, b_qinfo, output_qinfo, data_type, data_type, output_stage, reshape_b_only_on_first_run, finfo, updated_sq_info_after_config, act_info);
+        _reference = compute_reference(shape_a, shape_b, shape_output, a_qinfo, b_qinfo, data_type_a, data_type_b, output_stage, finfo);
+        _target    = compute_target(shape_a, shape_b, shape_output, a_qinfo, b_qinfo, output_qinfo, data_type_a, data_type_b, output_stage, reshape_b_only_on_first_run, finfo, updated_sq_info_after_config, act_info);
     }
 
 protected:
@@ -687,7 +688,7 @@ class GEMMLowpMatrixMultiplyCoreFusedOffsetOutputValidationFixture : public GEMM
     void setup(TensorShape shape_a, TensorShape shape_b, TensorShape shape_output, GEMMLowpOutputStageType output_stage_type, DataType data_type, bool reshape_b_only_on_first_run)
     {
         GEMMLowpGenericMatrixMultiplyCoreFusedOffsetOutputValidationFixture<TensorType, AccessorType, FunctionType, reinterpret_input_as_3d, reinterpret_output_as_3d, TI, TW, run_twice>::setup(shape_a, shape_b,
-                shape_output, output_stage_type, data_type, reshape_b_only_on_first_run);
+                shape_output, output_stage_type, data_type, data_type, reshape_b_only_on_first_run);
     }
 };
 
@@ -697,7 +698,8 @@ class GEMMLowpBatchedMatrixMultiplyCoreFusedOffsetOutputFixture : public GEMMLow
 public:
     void setup(TensorShape shape_a, TensorShape shape_b, TensorShape shape_output, GEMMLowpOutputStageType output_stage_type, DataType data_type, bool reshape_b_only_on_first_run)
     {
-        GEMMLowpGenericMatrixMultiplyCoreFusedOffsetOutputValidationFixture<TensorType, AccessorType, FunctionType, reinterpret_input_as_3d, reinterpret_output_as_3d, TI, TW, run_twice>::setup(shape_a, shape_b, shape_output, output_stage_type, data_type, reshape_b_only_on_first_run);
+        GEMMLowpGenericMatrixMultiplyCoreFusedOffsetOutputValidationFixture<TensorType, AccessorType, FunctionType, reinterpret_input_as_3d, reinterpret_output_as_3d, TI, TW, run_twice>
+            ::setup(shape_a, shape_b, shape_output, output_stage_type, data_type, data_type, reshape_b_only_on_first_run);
     }
 };