Update

Author: Cydral

dlib/dnn/transformer.h

@@ -32,29 +32,29 @@ namespace dlib
     namespace canonical_transformer
     {
 
-        template <long seq_len, long d_model, long num_heads, typename SUBNET>
-        using query = reshape_to<num_heads, seq_len, d_model / num_heads,
+        template <long d_model, long num_heads, typename SUBNET>
+        using query = reshape_to<num_heads, -1, d_model / num_heads,
             linear_no_bias<d_model, SUBNET>>;
 
-        template <long seq_len, long d_model, long num_heads, typename SUBNET>
-        using key = reshape_to<num_heads, seq_len, d_model / num_heads,
+        template <long d_model, long num_heads, typename SUBNET>
+        using key = reshape_to<num_heads, -1, d_model / num_heads,
             linear_no_bias<d_model, SUBNET>>;
 
-        template <long seq_len, long d_model, long num_heads, typename SUBNET>
-        using value = reshape_to<num_heads, seq_len, d_model / num_heads,
+        template <long d_model, long num_heads, typename SUBNET>
+        using value = reshape_to<num_heads, -1, d_model / num_heads,
             linear_no_bias<d_model, SUBNET>>;
 
         template <template <typename> class ACT, template <typename> class DO,
-            long seq_len, long d_model, long num_heads, typename SUBNET>
+            long d_model, long num_heads, typename SUBNET>
         using multihead_attention =
-            DO<linear_no_bias<d_model, reshape_to<1, seq_len, d_model,
+            DO<linear_no_bias<d_model, reshape_to<1, -1, d_model,
             multm_prev3<softmaxm<tril_mask<
             scale_weights<d_model / num_heads,
             multm_prev4<
-            rope<query<seq_len, d_model, num_heads, skip1<
+            rope<query<d_model, num_heads, skip1<
             tag4<transpose<
-            rope<key<seq_len, d_model, num_heads, skip2<
-            tag3<value<seq_len, d_model, num_heads,
+            rope<key<d_model, num_heads, skip2<
+            tag3<value<d_model, num_heads,
             tag2<SUBNET>>>>>>>>>>>>>>>>>>>;
 
         template <template <typename> class ACT, template <typename> class DO,
@@ -68,29 +68,29 @@ namespace dlib
             tag7<silu<linear<(d_model * 2) / 7, tag6<SUBNET>>>>>>>>>;
 
         template <template <typename> class ACT, template <typename> class DO,
-            long seq_len, long d_model, long num_heads, typename SUBNET>
+            long d_model, long num_heads, typename SUBNET>
         using transformer_block = 
             add_prev5<std_ffn<ACT, DO, d_model, rms_norm<tag5<
-            add_prev1<multihead_attention<ACT, DO, seq_len, d_model, num_heads, rms_norm<tag1<SUBNET>>>>>>>>;
+            add_prev1<multihead_attention<ACT, DO, d_model, num_heads, rms_norm<tag1<SUBNET>>>>>>>>;
 
         template<long remaining_layers, template <typename> class ACT, template <typename> class DO,
-            long seq_len, long d_model, long num_heads, typename SUBNET, typename enabled = void>
+            long d_model, long num_heads, typename SUBNET, typename enabled = void>
         struct transformer_stack_impl
         {
-            using type = transformer_block<ACT, DO, seq_len, d_model, num_heads,
-                typename transformer_stack_impl<remaining_layers - 1, ACT, DO, seq_len, d_model, num_heads, SUBNET>::type>;
+            using type = transformer_block<ACT, DO, d_model, num_heads,
+                typename transformer_stack_impl<remaining_layers - 1, ACT, DO, d_model, num_heads, SUBNET>::type>;
         };
 
         template<template <typename> class ACT, template <typename> class DO,
-            long seq_len, long d_model, long num_heads, typename SUBNET>
-        struct transformer_stack_impl<0, ACT, DO, seq_len, d_model, num_heads, SUBNET, void>
+            long d_model, long num_heads, typename SUBNET>
+        struct transformer_stack_impl<0, ACT, DO, d_model, num_heads, SUBNET, void>
         {
             using type = tag10<SUBNET>;
         };
 
         template<long num_layers, template <typename> class ACT, template <typename> class DO,
-            long seq_len, long d_model, long num_heads, typename SUBNET>
-        using transformer_stack = typename transformer_stack_impl<num_layers, ACT, DO, seq_len, d_model, num_heads, SUBNET>::type;
+            long d_model, long num_heads, typename SUBNET>
+        using transformer_stack = typename transformer_stack_impl<num_layers, ACT, DO, d_model, num_heads, SUBNET>::type;
 
     } // namespace std_transformer
 
@@ -179,7 +179,6 @@ namespace dlib
         using l_net_type = L_NET;
 
         explicit hrm_() :
-            seq_len(0),
             hidden_dim(0),
             learning_rate_multiplier(1.0)
         {
@@ -190,7 +189,6 @@ namespace dlib
             l_net(other.l_net),
             z_h_init(other.z_h_init),
             z_l_init(other.z_l_init),
-            seq_len(other.seq_len),
             hidden_dim(other.hidden_dim),
             learning_rate_multiplier(other.learning_rate_multiplier)
         {
@@ -203,7 +201,6 @@ namespace dlib
                 l_net = other.l_net;
                 z_h_init = other.z_h_init;
                 z_l_init = other.z_l_init;
-                seq_len = other.seq_len;
                 hidden_dim = other.hidden_dim;
                 learning_rate_multiplier = other.learning_rate_multiplier;
             }
@@ -215,8 +212,7 @@ namespace dlib
         {
             const tensor& input = sub.get_output();
 
-            // Store dimensions for initialization
-            seq_len = input.nr();
+            // Store dimension for initialization
             hidden_dim = input.nc();
 
             // Initialize hidden states with truncated normal (std=1, trunc=2)
@@ -229,6 +225,7 @@ namespace dlib
             const tensor& x = sub.get_output();
             const long batch_size = x.num_samples();
             const long k = x.k();
+            const long seq_len = x.nr();
 
             // Allocate working tensors with proper batch size
             z_h_current.copy_size(x);
@@ -356,7 +353,6 @@ namespace dlib
             serialize(item.l_net, out);
             serialize(item.z_h_init, out);
             serialize(item.z_l_init, out);
-            serialize(item.seq_len, out);
             serialize(item.hidden_dim, out);
             serialize(item.learning_rate_multiplier, out);
         }
@@ -372,7 +368,6 @@ namespace dlib
             deserialize(item.l_net, in);
             deserialize(item.z_h_init, in);
             deserialize(item.z_l_init, in);
-            deserialize(item.seq_len, in);
             deserialize(item.hidden_dim, in);
             deserialize(item.learning_rate_multiplier, in);
         }
@@ -449,7 +444,6 @@ namespace dlib
         resizable_tensor z_l_init;
 
         // Dimensions and learning rate
-        long seq_len;
         long hidden_dim;
         double learning_rate_multiplier;
 
@@ -473,7 +467,7 @@ namespace dlib
 
     // Gate network: produces raw logits for expert selection
     template <long num_experts, template <typename> class DO, typename SUBNET>
-    using gate = fc<num_experts, DO<leaky_relu<fc<num_experts * 8, SUBNET>>>>;
+    using gate = fc<num_experts, DO<leaky_relu<fc<num_experts * 8, avg_pool_everything<SUBNET>>>>>;
 
     struct training_mode_tag {};
     struct inference_mode_tag {};

examples/slm_advanced_train_ex.cpp

@@ -66,7 +66,6 @@ namespace dlib
      * @param num_layers Number of transformer layers
      * @param num_heads Number of attention heads
      * @param embedding_dim Dimension of token embeddings
-     * @param max_seq_len Maximum sequence length
      * @param activation_func Activation function type
      * @param dropout_policy Dropout regularization policy
      */
@@ -75,7 +74,6 @@ namespace dlib
         long num_layers = 6,
         long num_heads = 8,
         long embedding_dim = 512,
-        long max_seq_len = 300,
         template <typename> class activation_func = gelu,
         template <typename> class dropout_policy = dropout_10
     >
@@ -85,7 +83,6 @@ namespace dlib
         static constexpr long NUM_LAYERS = num_layers;
         static constexpr long NUM_HEADS = num_heads;
         static constexpr long EMBEDDING_DIM = embedding_dim;
-        static constexpr long MAX_SEQ_LEN = max_seq_len;
 
         // Compile-time validation of model configuration
         struct validation {
@@ -98,10 +95,10 @@ namespace dlib
         template<bool is_training>
         using network_type = std::conditional_t<is_training,
             classification_head<VOCAB_SIZE, EMBEDDING_DIM,
-            transformer_stack<NUM_LAYERS, activation_func, dropout_policy, MAX_SEQ_LEN, EMBEDDING_DIM, NUM_HEADS,
+            transformer_stack<NUM_LAYERS, activation_func, dropout_policy, EMBEDDING_DIM, NUM_HEADS,
             embeddings<VOCAB_SIZE, EMBEDDING_DIM, input<matrix<int, 0, 1>>>>>,
             classification_head<VOCAB_SIZE, EMBEDDING_DIM,
-            transformer_stack<NUM_LAYERS, activation_func, multiply, MAX_SEQ_LEN, EMBEDDING_DIM, NUM_HEADS,
+            transformer_stack<NUM_LAYERS, activation_func, multiply, EMBEDDING_DIM, NUM_HEADS,
             embeddings<VOCAB_SIZE, EMBEDDING_DIM, input<matrix<int, 0, 1>>>>>>;
 
         struct model_info {
@@ -111,8 +108,7 @@ namespace dlib
                     << "- vocabulary size: " << VOCAB_SIZE << "\n"
                     << "- layers: " << NUM_LAYERS << "\n"
                     << "- attention heads: " << NUM_HEADS << "\n"
-                    << "- embedding dimension: " << EMBEDDING_DIM << "\n"
-                    << "- sequence length: " << MAX_SEQ_LEN;
+                    << "- embedding dimension: " << EMBEDDING_DIM;
                 return ss.str();
             }
         };
@@ -309,8 +305,7 @@ int main(int argc, char** argv)
             num_tokens,     // vocab_size
             num_layers,     // number of layers
             num_heads,      // number of attention heads
-            embedding_dim,  // embedding dimension
-            max_seq_len     // maximum sequence length
+            embedding_dim   // embedding dimension
         >;
 
         // Load internal dataset

examples/slm_chatbot_ex.cpp

@@ -54,10 +54,10 @@ namespace dlib
 
     // Complete transformer block with MoE-based feed-forward layer
     template <template <typename> class ACT, template <typename> class DO,
-        long seq_len, long d_model, long num_heads, typename MODE, typename SUBNET>
+        long d_model, long num_heads, typename MODE, typename SUBNET>
     using trans_moe_block =
         moe_ffn<expert_net_type<DO, d_model>, 4, 0, MODE, DO,
-        add_prev1<multihead_attention<ACT, DO, seq_len, d_model, num_heads, rms_norm<tag1<SUBNET>>>>>;
+        add_prev1<multihead_attention<ACT, DO, d_model, num_heads, rms_norm<tag1<SUBNET>>>>>;
 
     // Classification head for next-token prediction in conversational context
     template <long num_logits, typename SUBNET>
@@ -66,7 +66,6 @@ namespace dlib
     // Chatbot model configuration
     template<
         long vocab_size = 2000,
-        long max_seq_len = 128,
         long num_layers = 3,
         long num_heads = 6,
         long embedding_dim = 192,
@@ -75,7 +74,6 @@ namespace dlib
     >
     struct chatbot_config {
         static constexpr long VOCAB_SIZE = vocab_size;
-        static constexpr long MAX_SEQ_LEN = max_seq_len;
         static constexpr long NUM_LAYERS = num_layers;
         static constexpr long NUM_HEADS = num_heads;
         static constexpr long EMBEDDING_DIM = embedding_dim;
@@ -90,13 +88,13 @@ namespace dlib
         // Network component definitions for training (with dropout)
         template <typename SUBNET>
         using t_transformer_block =
-            trans_moe_block<activation_func, dropout_policy, MAX_SEQ_LEN, EMBEDDING_DIM, NUM_HEADS,
+            trans_moe_block<activation_func, dropout_policy, EMBEDDING_DIM, NUM_HEADS,
             training_mode_tag, SUBNET>;
 
         // Network component definitions for inference (using multiply)
         template <typename SUBNET>
         using i_transformer_block =
-            trans_moe_block<activation_func, multiply, MAX_SEQ_LEN, EMBEDDING_DIM, NUM_HEADS,
+            trans_moe_block<activation_func, multiply, EMBEDDING_DIM, NUM_HEADS,
             inference_mode_tag, SUBNET>;
 
         // Complete network type selector based on training/inference mode
@@ -117,7 +115,6 @@ namespace dlib
                     << "- Layers: " << NUM_LAYERS << " transformer layers with MoE\n"
                     << "- Attention heads: " << NUM_HEADS << "\n"
                     << "- Embedding dimension: " << EMBEDDING_DIM << "\n"
-                    << "- Context window: " << MAX_SEQ_LEN << " tokens\n"
                     << "- Experts per layer: 4 (auto top-n selection)";
                 return ss.str();
             }
@@ -246,7 +243,7 @@ int main(int argc, char** argv)
         // Configuration parameters
         const long vocab_size = 3500;
         const long max_seq_len = 128;
-        using config = chatbot_config<vocab_size, max_seq_len>;
+        using config = chatbot_config<vocab_size>;
         using train_net = config::network_type<true>;
         using infer_net = config::network_type<false>;
         cout << config::model_info::describe() << "\n\n";

examples/slm_mixture_of_experts_ex.cpp

@@ -57,16 +57,16 @@ namespace dlib
         Architecture:
         1. Multi-head self-attention (from canonical_transformer)
         2. MoE feed-forward layer with multiple expert networks
-
+        f
         This replaces the standard transformer feed-forward layer with a
         mixture-of-experts that can specialize different experts for different
         types of patterns in the input.
     !*/
     template <template <typename> class ACT, template <typename> class DO,
-        long seq_len, long d_model, long num_heads, typename MODE, typename SUBNET>
+        long d_model, long num_heads, typename MODE, typename SUBNET>
     using trans_moe_block =
         moe_ffn<expert_net_type<DO, d_model>, 4, 0, MODE, DO,
-        add_prev1<multihead_attention<ACT, DO, seq_len, d_model, num_heads, rms_norm<tag1<SUBNET>>>>>;
+        add_prev1<multihead_attention<ACT, DO, d_model, num_heads, rms_norm<tag1<SUBNET>>>>>;
 
     /*!
         Classification head for next-token prediction.
@@ -80,7 +80,6 @@ namespace dlib
         long num_layers = 6,
         long num_heads = 8,
         long embedding_dim = 512,
-        long max_seq_len = 300,
         template <typename> class activation_func = gelu,
         template <typename> class dropout_policy = dropout_10
     >
@@ -89,7 +88,6 @@ namespace dlib
         static constexpr long NUM_LAYERS = num_layers;
         static constexpr long NUM_HEADS = num_heads;
         static constexpr long EMBEDDING_DIM = embedding_dim;
-        static constexpr long MAX_SEQ_LEN = max_seq_len;
 
         struct validation {
             static_assert(VOCAB_SIZE > 0, "Vocabulary size must be positive");
@@ -101,13 +99,13 @@ namespace dlib
         // Network component definitions for training (with dropout)
         template <typename SUBNET>
         using t_transformer_block =
-            trans_moe_block<activation_func, dropout_policy, MAX_SEQ_LEN, EMBEDDING_DIM, NUM_HEADS,
+            trans_moe_block<activation_func, dropout_policy, EMBEDDING_DIM, NUM_HEADS,
             training_mode_tag, SUBNET>;
 
         // Network component definitions for inference (using multiply)
         template <typename SUBNET>
         using i_transformer_block =
-            trans_moe_block<activation_func, multiply, MAX_SEQ_LEN, EMBEDDING_DIM, NUM_HEADS,
+            trans_moe_block<activation_func, multiply, EMBEDDING_DIM, NUM_HEADS,
             inference_mode_tag, SUBNET>;
 
         // Complete network type selector based on training/inference mode
@@ -128,7 +126,6 @@ namespace dlib
                     << "- Layers: " << NUM_LAYERS << "\n"
                     << "- Attention heads: " << NUM_HEADS << "\n"
                     << "- Embedding dimension: " << EMBEDDING_DIM << "\n"
-                    << "- Sequence length: " << MAX_SEQ_LEN << "\n"
                     << "- Architecture: Transformer with MoE feed-forward layers\n"
                     << "- Experts per layer: 4 (auto top-n selection)";
                 return ss.str();
@@ -602,8 +599,7 @@ int main(int argc, char** argv)
             num_tokens,     // vocab_size
             num_layers,     // number of layers
             num_heads,      // number of attention heads
-            embedding_dim,  // embedding dimension
-            max_seq_len     // maximum sequence length
+            embedding_dim   // embedding dimension
         > ;
 
         // Load internal dataset
@@ -938,19 +934,20 @@ int main(int argc, char** argv)
                 << tokenized_segments.size() << ") for generation\n";
             const auto& selected_segment = tokenized_segments[segment_idx];
 
+            long prompt_seq_len = max_seq_len;
             if (selected_segment.size() < (size_t)max_seq_len) {
-                cerr << "Error: Selected segment has only " << selected_segment.size()
+                cerr << "Warning: Selected segment has only " << selected_segment.size()
                     << " tokens, need at least " << max_seq_len << ".\n";
-                return 0;
+                prompt_seq_len = (selected_segment.size() * 2) / 3;
             }
 
-            // Extract prompt tokens (first max_seq_len tokens of the segment)
+            // Extract prompt tokens (first prompt_seq_len tokens of the segment)
             std::vector<int> prompt_tokens(selected_segment.begin(),
-                selected_segment.begin() + max_seq_len);
+                selected_segment.begin() + prompt_seq_len);
             cout << "Using " << prompt_tokens.size() << " tokens for initial prompt.\n";
 
             // Setup inference context
-            inference_context llm_context(max_seq_len, 4, tokenizer.get_special_token_id("<pad>"));
+            inference_context llm_context(max_seq_len*2, 4, tokenizer.get_special_token_id("<pad>"));
             llm_context.add_tokens(prompt_tokens);
             auto input_seq = llm_context.get_input_window();
 
@@ -969,7 +966,7 @@ int main(int argc, char** argv)
             cout << "Starting autoregressive generation...\n";
 
             // Generation parameters
-            const size_t tokens_to_generate = selected_segment.size() - max_seq_len;
+            const size_t tokens_to_generate = selected_segment.size() - prompt_seq_len;
             std::vector<int> generated_tokens;
             generated_tokens.reserve(tokens_to_generate);
 
@@ -1021,7 +1018,7 @@ int main(int argc, char** argv)
             cout << "\n=== Validation: comparing generated vs. original segment ===\n";
 
             // Extract reference tokens (the part we tried to regenerate)
-            std::vector<int> reference_tokens(selected_segment.begin() + max_seq_len,
+            std::vector<int> reference_tokens(selected_segment.begin() + prompt_seq_len,
                 selected_segment.end());
 
             // Limit comparison to the length of generated tokens