add quantize-computation for eltwise and deconvolution layers

Author: snoopyisadog

include/caffe/layers/eltwise_layer.hpp

@@ -41,6 +41,7 @@ class EltwiseLayer : public Layer<Dtype> {
 
   EltwiseParameter_EltwiseOp op_;
   vector<Dtype> coeffs_;
+  vector<Dtype> input_scale_; //CUSTOMIZATION
   vector<int> input_zero_point_; //CUSTOMIZATION
   Blob<int> max_idx_;
   Blob<int> min_idx_; //CUSTOMIZATION

src/caffe/layers/deconv_layer.cpp

@@ -43,8 +43,32 @@ void DeconvolutionLayer<Dtype>::compute_output_shape() {
 template <typename Dtype>
 void DeconvolutionLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
       const vector<Blob<Dtype>*>& top) {
+  const Dtype input_scale = this->input_scale_;
+  const Dtype output_scale = this->output_scale_;
+  const Dtype weight_scale = this->weight_scale_;
+  const Dtype bias_scale = this->bias_scale_;
+  const int input_zero_point = this->input_zero_point_;
+  const int output_zero_point = this->output_zero_point_;
+  const int weight_zero_point = this->weight_zero_point_;
+  const int bias_zero_point = this->bias_zero_point_;
+  const Dtype saturate = this->saturate_;
+  const bool quant_in = (input_scale != Dtype(1.0) || input_zero_point != 0);
+  const bool quant_out = (output_scale != Dtype(1.0) || output_zero_point != 0);
+  const bool quant_w = (weight_scale != Dtype(1.0) || weight_zero_point != 0);
+  const bool quant_b = (this->bias_term_&& (bias_scale != Dtype(1.0) || bias_zero_point != 0));
+  if (quant_w) {
+    Dtype *qw = this->blobs_[0]->mutable_cpu_data();
+    caffe_cpu_dequantize<Dtype>(this->blobs_[0]->count(), qw, weight_scale, weight_zero_point);
+  }
+  if (quant_b) {
+    Dtype *qb = this->blobs_[1]->mutable_cpu_data();
+    caffe_cpu_dequantize<Dtype>(this->blobs_[1]->count(), qb, bias_scale, bias_zero_point);
+  }
   const Dtype* weight = this->blobs_[0]->cpu_data();
   for (int i = 0; i < bottom.size(); ++i) {
+    if (quant_in) {
+      caffe_cpu_dequantize<Dtype>(bottom[i]->count(), bottom[i]->mutable_cpu_data(), input_scale, input_zero_point);
+    }
     const Dtype* bottom_data = bottom[i]->cpu_data();
     Dtype* top_data = top[i]->mutable_cpu_data();
     for (int n = 0; n < this->num_; ++n) {
@@ -55,6 +79,30 @@ void DeconvolutionLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
         this->forward_cpu_bias(top_data + n * this->top_dim_, bias);
       }
     }
+    const int count_t = top[i]->count();
+    if (quant_out) {
+      caffe_cpu_quantize<Dtype>(count_t, top_data, output_scale, output_zero_point);
+    }
+    if (saturate == ConvolutionParameter_SaturateMethod_Signed)
+      caffe_cpu_signed_saturate(count_t, top_data);
+    if (saturate == ConvolutionParameter_SaturateMethod_Unsigned)
+      caffe_cpu_unsigned_saturate(count_t, top_data);
+    if (saturate == ConvolutionParameter_SaturateMethod_Signed_8bit)
+      caffe_cpu_signed_8bit_saturate(count_t, top_data);
+    if (saturate == ConvolutionParameter_SaturateMethod_Unsigned_8bit)
+      caffe_cpu_unsigned_8bit_saturate(count_t, top_data);
+    if (quant_in) {
+      caffe_cpu_quantize<Dtype>(bottom[i]->count(), bottom[i]->mutable_cpu_data(), input_scale, input_zero_point);
+    }
+  }
+  // restore quantized weight/bias
+  if (quant_w) {
+    Dtype *qw = this->blobs_[0]->mutable_cpu_data();
+    caffe_cpu_quantize<Dtype>(this->blobs_[0]->count(), qw, weight_scale, weight_zero_point);
+  }
+  if (quant_b) {
+    Dtype *qb = this->blobs_[1]->mutable_cpu_data();
+    caffe_cpu_quantize<Dtype>(this->blobs_[1]->count(), qb, bias_scale, bias_zero_point);
   }
 }
 

src/caffe/layers/eltwise_layer.cpp

@@ -27,6 +27,12 @@ void EltwiseLayer<Dtype>::LayerSetUp(const vector<Blob<Dtype>*>& bottom,
       coeffs_[i] = this->layer_param().eltwise_param().coeff(i);
     }
   }
+  input_scale_ = vector<Dtype>(bottom.size(), 0);
+  if (this->layer_param().eltwise_param().input_scale_size()) {
+    for (int i = 0; i < bottom.size(); ++i) {
+      input_scale_[i] = this->layer_param().eltwise_param().input_scale(i);
+    }
+  }
   input_zero_point_ = vector<int>(bottom.size(), 0);
   if (this->layer_param().eltwise_param().input_zero_point_size()) {
     for (int i = 0; i < bottom.size(); ++i) {
@@ -97,6 +103,14 @@ void EltwiseLayer<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
 template <typename Dtype>
 void EltwiseLayer<Dtype>::Forward_cpu(
     const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
+  std::vector<bool> quant_in(bottom.size(), false);
+  for (int i = 0; i < quant_in.size(); ++i) {
+    quant_in[i] = (input_scale_[i] != Dtype(1.0) || input_zero_point_[i] != 0);
+    if (quant_in[i]) {
+      caffe_cpu_dequantize<Dtype>(bottom[i]->count(), bottom[i]->mutable_cpu_data(),
+        input_scale_[i], input_zero_point_[i]);
+    }
+  }// <<-- dequantize the input blobs
   const Dtype* bottom_data = bottom[0]->cpu_data();
   const Dtype* eltwise_data = NULL; //CUSTOMIZATION
   if(bottom.size() > 1)
@@ -276,6 +290,25 @@ void EltwiseLayer<Dtype>::Forward_cpu(
   default:
     LOG(FATAL) << "Unknown elementwise operation.";
   }
+  // quantize the output blob
+  if (output_scale_ != Dtype(1.0) || output_zero_point_ != 0) {
+    caffe_cpu_quantize<Dtype>(count, top_data, output_scale_, output_zero_point_);
+  }
+  if (saturate_ == EltwiseParameter_SaturateMethod_Signed)
+    caffe_cpu_signed_saturate(count, top_data);
+  if (saturate_ == EltwiseParameter_SaturateMethod_Unsigned)
+    caffe_cpu_unsigned_saturate(count, top_data);
+  if (saturate_ == EltwiseParameter_SaturateMethod_Signed_8bit)
+    caffe_cpu_signed_8bit_saturate(count, top_data);
+  if (saturate_ == EltwiseParameter_SaturateMethod_Unsigned_8bit)
+    caffe_cpu_unsigned_8bit_saturate(count, top_data);
+  // restore the quantized input blobs -->>
+  for (int i = 0; i < quant_in.size(); ++i) {
+    if (quant_in[i]) {
+      caffe_cpu_quantize<Dtype>(bottom[i]->count(), bottom[i]->mutable_cpu_data(),
+        input_scale_[i], input_zero_point_[i]);
+    }
+  }
 }
 
 template <typename Dtype>