@@ -282,6 +282,51 @@ Davis E. King. <a href="http://jmlr.csail.mit.edu/papers/volume10/king09a/king09
282282 <name >inception</name >
283283 <link >dlib/dnn/layers_abstract.h.html#inception</link >
284284 </item >
285+
286+ <item >
287+ <name >input_tensor</name >
288+ <link >dlib/dnn/layers_abstract.h.html#input_tensor_</link >
289+ </item >
290+ <item >
291+ <name >dropout_rate</name >
292+ <link >dlib/dnn/layers_abstract.h.html#dropout_rate_</link >
293+ </item >
294+ <item >
295+ <name >rms_norm</name >
296+ <link >dlib/dnn/layers_abstract.h.html#rms_norm_</link >
297+ </item >
298+ <item >
299+ <name >transpose</name >
300+ <link >dlib/dnn/layers_abstract.h.html#transpose_</link >
301+ </item >
302+ <item >
303+ <name >tril</name >
304+ <link >dlib/dnn/layers_abstract.h.html#tril_</link >
305+ </item >
306+ <item >
307+ <name >positional_encodings</name >
308+ <link >dlib/dnn/layers_abstract.h.html#positional_encodings_</link >
309+ </item >
310+ <item >
311+ <name >embeddings</name >
312+ <link >dlib/dnn/layers_abstract.h.html#embeddings_</link >
313+ </item >
314+ <item >
315+ <name >multm_prev</name >
316+ <link >dlib/dnn/layers_abstract.h.html#multm_prev_</link >
317+ </item >
318+ <item >
319+ <name >slice</name >
320+ <link >dlib/dnn/layers_abstract.h.html#slice_</link >
321+ </item >
322+ <item >
323+ <name >linear</name >
324+ <link >dlib/dnn/layers_abstract.h.html#linear_</link >
325+ </item >
326+ <item >
327+ <name >reshape_to</name >
328+ <link >dlib/dnn/layers_abstract.h.html#reshape_to_</link >
329+ </item >
285330 </sub >
286331 </item >
287332 <item nolink =" true" >
@@ -551,6 +596,7 @@ Davis E. King. <a href="http://jmlr.csail.mit.edu/papers/volume10/king09a/king09
551596 <item >count_ranking_inversions</item >
552597 <item >learn_platt_scaling</item >
553598 <item >process_sample</item >
599+ <item >bpe_tokenizer</item >
554600
555601
556602
@@ -2563,6 +2609,22 @@ Davis E. King. <a href="http://jmlr.csail.mit.edu/papers/volume10/king09a/king09
25632609 </description >
25642610 </component >
25652611
2612+ <!-- ************************************************************************* -->
2613+
2614+ <component >
2615+ <name >bpe_tokenizer</name >
2616+ <file >dlib/tokenizer.h</file >
2617+ <spec_file link =" true" >dlib/tokenizer/bpe_tokenizer_abstract.h</spec_file >
2618+ <description >
2619+ This class implements a Byte Pair Encoding (BPE) tokenizer, which is a subword
2620+ tokenization algorithm commonly used in natural language processing (NLP). The
2621+ BPE algorithm iteratively merges the most frequent pairs of bytes or characters
2622+ to form a vocabulary of subword units. This approach is particularly useful for
2623+ handling out-of-vocabulary words and reducing the size of the vocabulary while
2624+ maintaining the ability to represent any text.
2625+ </description >
2626+ </component >
2627+
25662628 <!-- ************************************************************************* -->
25672629
25682630 <component >
0 commit comments