Mul, Sub and Add Layer with broadcasting support

app/Accuracy/accuracy_check.cpp

@@ -7,7 +7,7 @@
 #include "layers/OutputLayer.hpp"
 #include "layers/PoolingLayer.hpp"
 
-using namespace itlab_2023;
+using namespace it_lab_ai;
 
 int main() {
   std::string image_path = IMAGE1_PATH;

app/AccuracyImgNet/accimgnet.cpp

@@ -10,7 +10,7 @@
 #include "layers/OutputLayer.hpp"
 #include "layers/PoolingLayer.hpp"
 
-using namespace itlab_2023;
+using namespace it_lab_ai;
 
 bool cmp_by_first(const std::pair<size_t, std::string>& a,
                   const std::pair<size_t, std::string>& b) {

app/Graph/acc_check_mnist.cpp

@@ -5,7 +5,7 @@
 #include "build.cpp"
 #include "build.hpp"
 
-using namespace itlab_2023;
+using namespace it_lab_ai;
 
 int main(int argc, char* argv[]) {
   bool parallel = false;

app/Graph/build.cpp

@@ -86,7 +86,7 @@ void build_graph(Tensor& input, Tensor& output, bool comments,
       Tensor tmp_bias = make_tensor(tensor.get_bias());
       Tensor tmp_tensor =
           Tensor(Shape({tensor.get_shape()[1], tensor.get_shape()[0]}),
-                 itlab_2023::Type::kFloat);
+                 it_lab_ai::Type::kFloat);
       // kernel is always transposed ?
       for (size_t h = 0; h < tensor.get_shape()[0]; h++) {
         for (size_t w = 0; w < tensor.get_shape()[1]; w++) {

app/Graph/graph_build.cpp

@@ -2,7 +2,7 @@
 #include "build.hpp"
 
 namespace fs = std::filesystem;
-using namespace itlab_2023;
+using namespace it_lab_ai;
 
 int main(int argc, char* argv[]) {
   std::string image_folder = IMAGE1_PATH;

include/Weights_Reader/reader_weights.hpp

@@ -5,7 +5,7 @@
 #include "layers/Tensor.hpp"
 
 using json = nlohmann::json;
-using namespace itlab_2023;
+using namespace it_lab_ai;
 
 json read_json(const std::string& filename);
 void extract_values_from_json(const json& j, std::vector<float>& values);

include/graph/graph.hpp

@@ -10,7 +10,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 class Graph {
   int BiggestSize_;
@@ -148,4 +148,4 @@ class Graph {
   std::vector<Tensor> getWEIGHTS() { return weights_; }
 #endif
 };
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/BinaryOpLayer.hpp

@@ -0,0 +1,72 @@
+#pragma once
+#include <algorithm>
+#include <memory>
+#include <stdexcept>
+#include <utility>
+#include <vector>
+
+#include "Tensor.hpp"
+#include "layers/Layer.hpp"
+
+namespace it_lab_ai {
+
+class BinaryOpLayer : public Layer {
+ public:
+  enum class Operation : uint8_t { kMul, kAdd, kSub };
+
+  BinaryOpLayer() = default;
+  explicit BinaryOpLayer(Operation op) : op_(op) {}
+
+  static std::string get_name() { return "Binary Operation Layer"; }
+  void run(const Tensor& input, Tensor& output) override;
+  void run(const Tensor& A, const Tensor& B, Tensor& output);
+  static bool is_scalar_tensor(const Tensor& t);
+
+#ifdef ENABLE_STATISTIC_WEIGHTS
+  Tensor get_weights() override {
+    std::vector<int> v = {0};
+    return make_tensor(v);
+  }
+#endif
+
+ private:
+  Operation op_ = Operation::kMul;
+  std::shared_ptr<void> impl_;
+
+  template <typename ValueType>
+  void run_with_scalar_impl(const Tensor& input, ValueType scalar,
+                            Tensor& output) const;
+  void run_with_scalar(const Tensor& input, float scalar, Tensor& output);
+
+  static bool can_broadcast(const Shape& shape_A, const Shape& shape_B);
+  static Shape calculate_broadcasted_shape(const Shape& shape_A,
+                                           const Shape& shape_B);
+  static std::vector<size_t> get_strides(const Shape& shape);
+  static size_t get_broadcasted_index(size_t flat_index,
+                                      const Shape& input_shape,
+                                      const Shape& output_shape);
+
+  template <typename ValueType>
+  class BinaryOpLayerImpl;
+};
+
+template <typename ValueType>
+class BinaryOpLayer::BinaryOpLayerImpl : public LayerImpl<ValueType> {
+ public:
+  BinaryOpLayerImpl() = delete;
+  explicit BinaryOpLayerImpl(BinaryOpLayer::Operation op);
+
+  std::vector<ValueType> run(
+      const std::vector<ValueType>& input) const override {
+    (void)input;
+    throw std::runtime_error("BinaryOpLayer requires two inputs");
+  }
+
+  std::vector<ValueType> run(const std::vector<ValueType>& inputA,
+                             const std::vector<ValueType>& inputB) const;
+
+ private:
+  BinaryOpLayer::Operation op_;
+};
+
+}  // namespace it_lab_ai

include/layers/ConvLayer.hpp

@@ -6,7 +6,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 class ConvolutionalLayer : public Layer {
  private:
@@ -402,4 +402,4 @@ void Conv4DSTL(const Tensor& input, const Tensor& kernel_, const Tensor& bias_,
   output = make_tensor<ValueType>(one_d_vector, sh);
 }
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/DropOutLayer.hpp

@@ -3,7 +3,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 class DropOutLayer : public Layer {
  private:
@@ -16,4 +16,4 @@ class DropOutLayer : public Layer {
   void run(const Tensor& input, Tensor& output) override;
 };
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/EWLayer.hpp

@@ -6,7 +6,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 template <typename T>
 T relu(const T& value) {
@@ -93,4 +93,4 @@ std::vector<ValueType> EWLayerImpl<ValueType>::run(
   return res;
 }
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/FCLayer.hpp

@@ -7,7 +7,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 class FCLayer : public Layer {
  private:
@@ -128,4 +128,4 @@ std::vector<ValueType> FCLayerImpl<ValueType>::run(
   return output_values;
 }
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/FlattenLayer.hpp

@@ -3,7 +3,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 std::vector<size_t> reorder(std::vector<size_t> order_vec,
                             std::vector<size_t> order);
@@ -51,4 +51,4 @@ void Flatten4D(const Tensor& input, Tensor& output,
                        Shape({input.get_shape().count()}));
 }
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/InputLayer.hpp

@@ -4,7 +4,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 enum LayInOut : uint8_t {
   kNchw,  // 0
@@ -179,4 +179,4 @@ class InputLayer : public Layer {
   }
 };
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/Layer.hpp

@@ -9,7 +9,7 @@
 #include "layers/Tensor.hpp"
 #include "oneapi/tbb.h"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 enum LayerType : uint8_t {
   kInput,
@@ -65,4 +65,4 @@ class LayerImpl {
   Shape outputShape_;
 };
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/OutputLayer.hpp

@@ -7,7 +7,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 class OutputLayer : public Layer {
  public:
@@ -111,4 +111,4 @@ std::pair<std::vector<std::string>, std::vector<ValueType>> top_k_vec(
   return std::make_pair(res_labels, res_input);
 }
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/PoolingLayer.hpp

@@ -6,7 +6,7 @@
 
 #include "layers/Layer.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 enum PoolingType : uint8_t { kAverage, kMax };
 
@@ -266,4 +266,4 @@ std::vector<ValueType> PoolingLayerImplTBB<ValueType>::run(
   return res;
 }
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/Shape.hpp

@@ -8,7 +8,7 @@
 #include <stdexcept>
 #include <vector>
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 class Shape {
  public:
@@ -40,9 +40,19 @@ class Shape {
   size_t dims() const noexcept { return dims_.size(); }
   size_t get_index(const std::vector<size_t>& coords) const;
   friend std::ostream& operator<<(std::ostream& os, const Shape& shape);
+  bool operator==(const Shape& other) const noexcept {
+    if (dims_.size() != other.dims_.size()) {
+      return false;
+    }
+    return std::equal(dims_.begin(), dims_.end(), other.dims_.begin());
+  }
+
+  bool operator!=(const Shape& other) const noexcept {
+    return !(*this == other);
+  }
 
  private:
   std::vector<size_t> dims_;
 };
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/layers/Tensor.hpp

@@ -8,7 +8,7 @@
 
 #include "layers/Shape.hpp"
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 enum class Type : uint8_t { kUnknown, kInt, kFloat };
 
@@ -207,4 +207,4 @@ Tensor make_tensor(const std::vector<T>& values, const Shape& shape,
 }
 std::ostream& operator<<(std::ostream& out, const Tensor& t);
 
-}  // namespace itlab_2023
+}  // namespace it_lab_ai

include/perf/benchmarking.hpp

@@ -9,7 +9,7 @@
 #include <stdexcept>
 #include <vector>
 
-namespace itlab_2023 {
+namespace it_lab_ai {
 
 template <typename DurationContainerType, typename DurationType, class Function,
           typename... Args>
@@ -114,4 +114,4 @@ T accuracy_norm(T* test, T* ref, const size_t size) {
   // typename T should have friend sqrt() function
   return std::sqrt(res);
 }
-}  // namespace itlab_2023
+}  // namespace it_lab_ai