add more parameter for Observer

Author: XiaobingSuper

intel_pytorch_extension_py/__init__.py

@@ -59,6 +59,16 @@ def get_auto_mix_precision():
     else:
         return None
 
+'''
+def quarry_int8_configure(model, inputs_shape):
+    dummy_input = torch.randn(input_shapes).to(DEVICE)
+    core.enable_mix_int8_fp32()
+    with torch.no_grad():
+        y = model(dummy_input)
+    observer_configures = core.get_int8_observer_configures()
+    return observer_configures
+'''
+
 def calibration_reset():
     if core.get_int8_calibration():
         core.calibration_reset()
@@ -93,13 +103,15 @@ def generator_context(*args, **kwargs):
         return generator_context
 
 class int8_calibration(_DecoratorContextManager):
-    def __init__(self, file_name):
+    def __init__(self, file_name, observer_configure=None):
+        #self.observer_configure = observer_configure
         self.configure_file = file_name
 
     def __enter__(self):
         if not core.get_mix_int8_fp32():
             raise ValueError("please first run enable_auto_mix_precision(torch.int8) before int8 calibration")
         core.enable_int8_calibration()
+        #core.set_int8_observer_configure(self.observer_configure)
 
     def __exit__(self, *args):
         core.disable_int8_calibration()

torch_ipex/csrc/auto_opt_config.h

@@ -60,46 +60,63 @@ class AutoOptConfig {
     return calibration_step_;
   }
 
-  inline void insert_or_updata_observer(std::string op_name, std::vector<float> max_values) {
+  inline void insert_or_updata_observer(std::string op_name,
+    std::vector<float> input_min_max_values, std::vector<float> output_min_max_values) {
     num_ops_id++;
     if (observers_.size() < num_ops_id) {
-      //Operator op = {num_ops_id - 1, op_n};
-      Observer new_observer = {num_ops_id - 1, op_name, max_values};
+      // this path is that user not set int8 op's configure, using default configures
+      Observer new_observer = {num_ops_id - 1, op_name, input_min_max_values, output_min_max_values};
       observers_.push_back(new_observer);
     } else {
-      for (auto i = 0; i < max_values.size(); i++)
-        observers_[num_ops_id -1].max_values[i] = std::max(observers_[num_ops_id -1].max_values[i], max_values[i]);
+      // user has set configure or have run one interation
+      auto input_pre = observers_[num_ops_id - 1].Input_min_max_values;
+      auto output_pre = observers_[num_ops_id - 1].Output_min_max_values;
+      if (observers_[num_ops_id - 1].Algorithm == "min_max") {
+        observers_[num_ops_id - 1].Input_min_max_values[0] = std::min(input_pre[0], input_min_max_values[0]);
+        observers_[num_ops_id - 1].Input_min_max_values[1] = std::max(input_pre[1], input_min_max_values[1]);
+        observers_[num_ops_id - 1].Output_min_max_values[0] = std::min(output_pre[0], output_min_max_values[0]);
+        observers_[num_ops_id - 1].Output_min_max_values[1] = std::max(output_pre[1], output_min_max_values[1]);
+      } else if(observers_[num_ops_id -1].Algorithm == "moving_averager_min_max"){
+        auto c = observers_[num_ops_id - 1].Averaging_constant;
+        observers_[num_ops_id - 1].Input_min_max_values[0] = (1 - c) * input_pre[0] + c * input_min_max_values[0];
+        observers_[num_ops_id - 1].Input_min_max_values[1] = (1 - c) * input_pre[1] + c * input_min_max_values[1];
+        observers_[num_ops_id - 1].Output_min_max_values[0] = (1 - c) * output_pre[0] + c * output_min_max_values[0];
+        observers_[num_ops_id - 1].Output_min_max_values[1] = (1 - c) * output_pre[1] + c * output_min_max_values[1];
+      }
     }
   }
 
+  /*
   inline void print_observer() {
     for (auto i = 0; i< observers_.size(); i++) {
       for (auto j = 0; j < observers_[i].max_values.size(); j++)
         std::cout<<observers_[i].max_values[j]<<std::endl;
     }
   }
-
+*/
   inline void print_indicator() {
     for (auto i = 0; i< indicators_.size(); i++) {
       auto scales = indicators_[i].get_indicator_scales();
       for (auto j = 0; j< scales.size(); j++)
           std::cout<<scales[j]<<std::endl;
     }
   }
-
+ 
   inline void add_indicators() {
     num_ops_id = 0;
     // default used is s8
     for (auto i = 0; i < observers_.size(); i++) {
       std::vector<float> scales;
-      std::vector<bool> uint8_used;
-      for (auto j = 0; j < observers_[i].max_values.size(); j++) {
-        scales.push_back(127.5 / observers_[i].max_values[j]);
-        uint8_used.push_back(false);
-      }
-        // zero_points not used now, zero_points = 0 for u8 and 128 for s8.
-        //zero_point = 128;
-      Indicator new_indicator(observers_[i].Id, observers_[i].Name, scales, uint8_used, true);
+      std::vector<float> input_values = observers_[i].Input_min_max_values;
+      std::vector<float> output_values = observers_[i].Output_min_max_values;
+
+      scales.push_back(127.5 / std::max(std::abs(input_values[0]), input_values[1]));
+      scales.push_back(127.5 / std::max(std::abs(output_values[0]), output_values[1]));
+      // zero_points not used now, zero_points = 0 for u8 and 128 for s8.
+      //zero_point = 128;
+      Indicator new_indicator(observers_[i].Id, observers_[i].Name, observers_[i].Algorithm,
+        observers_[i].Weight_granularity, scales, {observers_[i].Input_dtype_uint8, observers_[i].Output_dtype_uint8},
+        observers_[i].Quantized);
       indicators_.push_back(new_indicator);
     }
     observers_.clear();

torch_ipex/csrc/cpu/DevOPs.cpp

@@ -73,7 +73,6 @@ at::Tensor AtenIpexCPUDev::dil_convolution(
 
   dil_input = dbl::comm::try_gen_dil_tensor(input);
   if (bias.defined()) {
-      std::cout<<"convolution has bias"<<std::endl;
     CHECK_DNNL_OP_PRE_COND(bias);
     if (!check_auto_mix_int8_fp32()) {
       dbl::comm::reorder_to_bf16_for_mix_prec(bias, true);

torch_ipex/csrc/init_python_bindings.cpp

@@ -141,7 +141,7 @@ void InitIpexModuleBindings(py::module m) {
   m.def("get_int8_calibration", []() { return AutoOptConfig::singleton().get_int8_calibration(); });
   m.def("calibration_reset", []() { AutoOptConfig::singleton().calibration_reset(); });
   m.def("add_indicators", []() { AutoOptConfig::singleton().add_indicators(); });
-  m.def("print_observer", []() { AutoOptConfig::singleton().print_observer(); });
+  //m.def("print_observer", []() { AutoOptConfig::singleton().print_observer(); });
   m.def("print_indicator", []() { AutoOptConfig::singleton().print_indicator(); });
   m.def("get_int8_configures", []() {
       py::list output_list;
@@ -150,6 +150,8 @@ void InitIpexModuleBindings(py::module m) {
         py::dict d;
         d["id"] = indicator.get_indicator_id();
         d["name"] = indicator.get_indicator_name();
+        d["algorithm"] = indicator.get_indicator_algorithm();
+        d["weight_granularity"] = indicator.get_indicator_weight_granularity();
         std::vector<float> scales = indicator.get_indicator_scales();
         d["input_scale"] = scales[0];
         d["output_scale"] = scales[1];
@@ -166,12 +168,14 @@ void InitIpexModuleBindings(py::module m) {
       for (py::handle i : l) {
         int64_t id = py::cast<std::int64_t>(i["id"]);
         std::string op_name = py::cast<std::string>(i["name"]);
+        std::string algorithm = py::cast<std::string>(i["algorithm"]);
+        std::string weight_granularity = py::cast<std::string>(i["weight_granularity"]);
         float input_scale = py::cast<float>(i["input_scale"]);
         float output_scale = py::cast<float>(i["output_scale"]);
         bool input_uint8_used = py::cast<bool>(i["input_uint8_used"]);
         bool output_uint8_used = py::cast<bool>(i["output_uint8_used"]);
         bool quantized  = py::cast<bool>(i["quantized"]);
-        Indicator temp(id, op_name, {input_scale, output_scale},
+        Indicator temp(id, op_name, algorithm, weight_granularity, {input_scale, output_scale},
           {input_uint8_used, output_uint8_used}, quantized);
         indicators.push_back(temp);
       }

torch_ipex/csrc/quantization/Observer.h

@@ -8,15 +8,28 @@ namespace int8 {
 struct Observer {
   int64_t Id;
   std::string Name;
-  // the max_values of input and output for one op
-  std::vector<float> max_values;
+  std::vector<float> Input_min_max_values;
+  std::vector<float> Output_min_max_values;
+  // default uising min/max to compute the quantization parameters,
+  // only support min_max, MovingAverageMinMax and other none per_channel merthod
+  std::string Algorithm = "min_max";
+  float Averaging_constant = 0.01;  // for MovingAverage method
+  // only useful for conv, onednn only support per_channel foo conv's weight,
+  // default is per_tensor
+  std::string Weight_granularity = "per_tensor";
+  // ture means input will be quantized to int8, otherwise quantized to uint8.
+  bool Input_dtype_uint8 = false;
+  bool Output_dtype_uint8 = false;
+  bool Quantized = true;
 };
 
 class Indicator {
   public:
-    Indicator(int64_t id = 0, std::string name = "", std::vector<float> scales = std::vector<float>(2, 1),
-        std::vector<bool> uint8_used = std::vector<bool>(2, false) , bool quantized = true):
-      Id(id), Name(name), Scales(scales), Uint8_used(uint8_used), Quantized(quantized) {}
+    Indicator(int64_t id = 0, std::string name = "", std::string algorithm = "min_max",
+      std::string weight_granularity = "per_tensor", std::vector<float> scales = std::vector<float>(2, 1),
+      std::vector<bool> uint8_used = std::vector<bool>(2, false),bool quantized = true):
+      Id(id), Name(name), Algorithm(algorithm), Weight_granularity(weight_granularity),
+      Scales(scales), Uint8_used(uint8_used), Quantized(quantized) {}
 
   int64_t get_indicator_id() {
     return Id;
@@ -26,6 +39,14 @@ class Indicator {
     return Name;
   }
 
+  std::string get_indicator_algorithm() {
+    return Algorithm;
+  }
+
+  std::string get_indicator_weight_granularity() {
+    return Weight_granularity;
+  }
+
   std::vector<float> get_indicator_scales() {
     return Scales;
   }
@@ -53,6 +74,8 @@ class Indicator {
   private:
     int64_t Id;
     std::string Name;
+    std::string Algorithm;
+    std::string Weight_granularity;
     std::vector<float> Scales;
     std::vector<bool> Uint8_used;
     bool Quantized;

torch_ipex/csrc/utils.cpp

@@ -129,15 +129,16 @@ bool check_int8_calibration() {
 }
 
 void insert_or_updata_observer(const at::Tensor& self, const at::Tensor& output, std::string op_name) {
-  std::vector<float> max_values;
-  auto value = self.abs().max().item<float>();
-  max_values.push_back(value);
+  std::vector<float> input_min_max_values, output_min_max_values;
+  input_min_max_values.push_back(self.abs().min().item<float>());
+  input_min_max_values.push_back(self.abs().max().item<float>());
   if (output.defined()) {
-    max_values.push_back(output.abs().max().item<float>());
+    output_min_max_values.push_back(output.abs().min().item<float>());
+    output_min_max_values.push_back(output.abs().max().item<float>());
   } else {
-    max_values.push_back(value);
+    output_min_max_values = input_min_max_values;
   }
-  AutoOptConfig::singleton().insert_or_updata_observer(op_name, max_values);
+  AutoOptConfig::singleton().insert_or_updata_observer(op_name, input_min_max_values, output_min_max_values);
 }
 
 std::tuple<std::vector<float>, bool> get_indicator_scales(std::vector<bool> uint8_used) {