Add linear_ layer for neural networks (#3074)

Author: Cydral

dlib/dnn/layers.h

@@ -2143,6 +2143,206 @@ namespace dlib
         >
     using fc_no_bias = add_layer<fc_<num_outputs,FC_NO_BIAS>, SUBNET>;
 
+// ----------------------------------------------------------------------------------------
+    
+    enum linear_bias_mode { LINEAR_HAS_BIAS = 0, LINEAR_NO_BIAS = 1 };
+
+    template <
+        unsigned long num_outputs_,
+        linear_bias_mode bias_mode_
+    >
+    class linear_
+    {
+        static_assert(num_outputs_ > 0, "The number of outputs from a linear_ layer must be > 0");
+
+    public:
+        linear_() :
+            num_outputs(num_outputs_),
+            num_inputs(0),
+            learning_rate_multiplier(1),
+            bias_mode(bias_mode_) {
+        }
+
+        double get_learning_rate_multiplier() const { return learning_rate_multiplier; }
+        void set_learning_rate_multiplier(double val) { learning_rate_multiplier = val; }
+
+        unsigned long get_num_inputs() const { return num_inputs; }
+        unsigned long get_num_outputs() const { return num_outputs; }
+        void set_num_outputs(long num)
+        {
+            DLIB_CASSERT(num > 0, "The number of outputs must be > 0, but num == " << num);
+            if (num != (long)num_outputs)
+            {
+                DLIB_CASSERT(get_layer_params().size() == 0,
+                    "You can't change the number of filters in linear_ if the parameter tensor has already been allocated.");
+                num_outputs = num;
+            }
+        }
+        linear_bias_mode get_bias_mode() const { return bias_mode; }
+
+        template <typename SUBNET>
+        void setup(const SUBNET& sub)
+        {
+            num_inputs = sub.get_output().nc();
+            if (bias_mode == LINEAR_HAS_BIAS)
+                params.set_size(num_inputs + 1, num_outputs);
+            else
+                params.set_size(num_inputs, num_outputs);
+
+            dlib::rand rnd(std::rand());
+            randomize_parameters(params, num_inputs + num_outputs, rnd);
+            weights = alias_tensor(num_inputs, num_outputs);
+
+            if (bias_mode == LINEAR_HAS_BIAS) {
+                biases = alias_tensor(1, num_outputs);
+                biases(params, weights.size()) = 0;
+            }
+        }
+
+        template <typename SUBNET>
+        void forward(const SUBNET& sub, resizable_tensor& output)
+        {
+            const auto& prev_output = sub.get_output();
+            DLIB_CASSERT((long)num_inputs == prev_output.nc(),
+                "The size of the input tensor to this linear layer doesn't match the size the linear layer was trained with.");            
+            output.set_size(prev_output.num_samples(), prev_output.k(), prev_output.nr(), num_outputs);
+
+            auto o = alias_tensor(output.num_samples() * output.k() * output.nr(), num_outputs)(output, 0);
+            auto so = alias_tensor(prev_output.num_samples() * prev_output.k() * prev_output.nr(), num_inputs)(prev_output, 0);
+
+            auto w = weights(params, 0);
+            tt::gemm(0, (tensor&)o, 1, so, false, w, false);
+
+            if (bias_mode == LINEAR_HAS_BIAS)
+            {
+                auto b = biases(params, weights.size());
+                tt::add(1, (tensor&)o, 1, b);
+            }
+        }
+
+        template <typename SUBNET>
+        void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad)
+        {
+            auto gi = alias_tensor(gradient_input.num_samples() * gradient_input.k() * gradient_input.nr(), num_outputs)(gradient_input, 0);
+            if (learning_rate_multiplier != 0)
+            {
+                const auto& prev_output = sub.get_output();
+                auto pw = weights(params_grad, 0);
+                auto so = alias_tensor(prev_output.num_samples() * prev_output.k() * prev_output.nr(), num_inputs)(prev_output, 0);
+                tt::gemm(0, pw, learning_rate_multiplier, so, true, gi, false);
+
+                if (bias_mode == LINEAR_HAS_BIAS)
+                {
+                    auto pb = biases(params_grad, weights.size());
+                    tt::assign_bias_gradient(pb, gi);
+                }
+            }
+
+            const auto& prev_gradient = sub.get_gradient_input();
+            auto sgi = alias_tensor(prev_gradient.num_samples() * prev_gradient.k() * prev_gradient.nr(), num_inputs)(prev_gradient, 0);
+            auto w = weights(params, 0);
+            tt::gemm(1, (tensor&)sgi, 1, gi, false, w, true);
+        }
+
+        alias_tensor_instance get_weights() { return weights(params, 0); }
+        alias_tensor_const_instance get_weights() const { return weights(params, 0); }
+        alias_tensor_instance get_biases()
+        {
+            static_assert(bias_mode == LINEAR_HAS_BIAS, "This linear_ layer doesn't have a bias vector "
+                "to be retrieved, as per template parameter 'bias_mode'.");
+            return biases(params, weights.size());
+        }
+        alias_tensor_const_instance get_biases() const
+        {
+            static_assert(bias_mode == LINEAR_HAS_BIAS, "This linear_ layer doesn't have a bias vector "
+                "to be retrieved, as per template parameter 'bias_mode'.");
+            return biases(params, weights.size());
+        }
+
+        inline dpoint map_input_to_output(const dpoint& p) const { return p; }
+        inline dpoint map_output_to_input(const dpoint& p) const { return p; }
+
+        const tensor& get_layer_params() const { return params; }
+        tensor& get_layer_params() { return params; }
+
+        friend void serialize(const linear_& item, std::ostream& out)
+        {
+            serialize("linear_", out);
+            serialize(item.num_outputs, out);
+            serialize(item.num_inputs, out);
+            serialize(item.params, out);
+            serialize(item.weights, out);
+            serialize(item.biases, out);
+            serialize((int)item.bias_mode, out);
+            serialize(item.learning_rate_multiplier, out);
+        }
+
+        friend void deserialize(linear_& item, std::istream& in)
+        {
+            std::string version;
+            deserialize(version, in);
+            if (version == "linear_")
+            {
+                deserialize(item.num_outputs, in);
+                deserialize(item.num_inputs, in);
+                deserialize(item.params, in);
+                deserialize(item.weights, in);
+                deserialize(item.biases, in);
+                int bmode;
+                deserialize(bmode, in);
+                item.bias_mode = static_cast<linear_bias_mode>(bmode);
+                if (bias_mode_ != item.bias_mode) throw serialization_error("Wrong bias_mode found while deserializing dlib::linear_");
+                deserialize(item.learning_rate_multiplier, in);
+            }
+            else
+            {
+                throw serialization_error("Unexpected version '" + version + "' found while deserializing dlib::linear_.");
+            }
+        }
+
+        friend std::ostream& operator<<(std::ostream& out, const linear_& item)
+        {
+            out << "linear\t (num_outputs=" << item.num_outputs;
+            if (item.bias_mode == LINEAR_HAS_BIAS)
+                out << ", bias=true";
+            else
+                out << ", bias=false";
+            out << ")";
+            out << " learning_rate_mult=" << item.learning_rate_multiplier;
+            return out;
+        }
+
+        friend void to_xml(const linear_& item, std::ostream& out)
+        {
+            out << "<linear"
+                << " num_outputs='" << item.num_outputs << "'"
+                << " bias='" << ((item.bias_mode == LINEAR_HAS_BIAS) ? "true" : "false") << "'"
+                << " learning_rate_mult='" << item.learning_rate_multiplier << "'>\n";
+            out << mat(item.params);
+            out << "</linear>\n";
+        }
+
+    private:
+        unsigned long num_inputs;
+        unsigned long num_outputs;
+        double learning_rate_multiplier;
+        linear_bias_mode bias_mode;
+        resizable_tensor params;
+        alias_tensor weights, biases;
+    };
+
+    template <
+        unsigned long num_outputs,
+        typename SUBNET
+    >
+    using linear = add_layer<linear_<num_outputs, LINEAR_HAS_BIAS>, SUBNET>;
+
+    template <
+        unsigned long num_outputs,
+        typename SUBNET
+    >
+    using linear_no_bias = add_layer<linear_<num_outputs, LINEAR_NO_BIAS>, SUBNET>;
+
 // ----------------------------------------------------------------------------------------
 
     class dropout_