Fix Class Name for PretrainedTokenizer (#71)

* Fix Class Name for PreTrainedTokenizer.php

* Fix Folder Name & Also Namespace.

Author: takielias

docs/tokenizers.md

@@ -76,7 +76,7 @@ any tokenizer from the Hugging Face model hub. Here's how to create a tokenizer
 class:
 
 ```php
-use Codewithkyrian\Transformers\PretrainedTokenizers\AutoTokenizer;
+use Codewithkyrian\Transformers\PreTrainedTokenizers\AutoTokenizer;
 
 $tokenizer = AutoTokenizer::fromPretrained('Xenova/toxic-bert');
 ```
@@ -135,11 +135,11 @@ class, you can still skip that step and use the specific tokenizer class directl
 have more control over the tokenization process. Here's an example of how to use the `BertTokenizer` class directly:
 
 ```php
-use Codewithkyrian\Transformers\PretrainedTokenizers\BertTokenizer;
+use Codewithkyrian\Transformers\PreTrainedTokenizers\BertTokenizer;
 
 $tokenizer = BertTokenizer::fromPretrained('Xenova/toxic-bert');
 ```
 
 The `fromPretrained` method of the `BertTokenizer` class accepts the same arguments as the `AutoTokenizer` class. You
 can
-see all available model specific tokenizers in the `Codewithkyrian\Transformers\PretrainedTokenizers` namespace.
+see all available model specific tokenizers in the `Codewithkyrian\Transformers\PreTrainedTokenizers` namespace.

examples/tokenizers/apply-chat-template.php

@@ -2,7 +2,7 @@
 
 declare(strict_types=1);
 
-use Codewithkyrian\Transformers\PretrainedTokenizers\AutoTokenizer;
+use Codewithkyrian\Transformers\PreTrainedTokenizers\AutoTokenizer;
 
 require_once './bootstrap.php';
 
@@ -17,4 +17,4 @@
 
 $text = $tokenizer->applyChatTemplate($messages, addGenerationPrompt: true, tokenize: false);
 
-dd($text);
+dd($text);

src/Commands/DownloadModelCommand.php

@@ -7,7 +7,7 @@
 
 use Codewithkyrian\Transformers\Models\Auto\AutoModel;
 use Codewithkyrian\Transformers\Pipelines\Task;
-use Codewithkyrian\Transformers\PretrainedTokenizers\AutoTokenizer;
+use Codewithkyrian\Transformers\PreTrainedTokenizers\AutoTokenizer;
 use Codewithkyrian\Transformers\Transformers;
 use Exception;
 use Symfony\Component\Console\Attribute\AsCommand;

src/Generation/Streamers/Streamer.php

@@ -5,7 +5,7 @@
 
 namespace Codewithkyrian\Transformers\Generation\Streamers;
 
-use Codewithkyrian\Transformers\PretrainedTokenizers\PretrainedTokenizer;
+use Codewithkyrian\Transformers\PreTrainedTokenizers\PreTrainedTokenizer;
 
 /**
  * Base streamer from which all streamers inherit.
@@ -16,7 +16,7 @@ abstract class Streamer
     protected bool $skipPrompt = false;
     protected bool $nextTokensArePrompt;
 
-    protected PretrainedTokenizer $tokenizer;
+    protected PreTrainedTokenizer $tokenizer;
     protected mixed $onStreamCallback = null;
     protected mixed $onStreamEndCallback = null;
     protected StreamMode $streamMode = StreamMode::PARTIAL;
@@ -26,7 +26,7 @@ public static function make(): static
         return new static();
     }
 
-    public function setTokenizer(PretrainedTokenizer $tokenizer): static
+    public function setTokenizer(PreTrainedTokenizer $tokenizer): static
     {
         $this->tokenizer = $tokenizer;
         return $this;
@@ -67,4 +67,4 @@ abstract public function put(mixed $value): void;
 
     abstract public function end(): void;
 
-}
+}

src/Generation/Streamers/TextStreamer.php

@@ -4,7 +4,7 @@
 
 namespace Codewithkyrian\Transformers\Generation\Streamers;
 
-use Codewithkyrian\Transformers\PretrainedTokenizers\PretrainedTokenizer;
+use Codewithkyrian\Transformers\PreTrainedTokenizers\PreTrainedTokenizer;
 use InvalidArgumentException;
 
 /**

src/Pipelines/Pipeline.php

@@ -6,7 +6,7 @@
 
 use Codewithkyrian\Transformers\Exceptions\UnsupportedTaskException;
 use Codewithkyrian\Transformers\Models\Pretrained\PretrainedModel;
-use Codewithkyrian\Transformers\PretrainedTokenizers\PretrainedTokenizer;
+use Codewithkyrian\Transformers\PreTrainedTokenizers\PreTrainedTokenizer;
 use Codewithkyrian\Transformers\Processors\Processor;
 use Codewithkyrian\Transformers\Tensor\Tensor;
 use Codewithkyrian\Transformers\Utils\Image;
@@ -16,7 +16,7 @@ class Pipeline
     public function __construct(
         protected string|Task       $task,
         protected PretrainedModel   $model,
-        public ?PretrainedTokenizer $tokenizer = null,
+        public ?PreTrainedTokenizer $tokenizer = null,
         protected ?Processor        $processor = null,
     )
     {

src/Pipelines/Task.php

@@ -23,8 +23,8 @@
 use Codewithkyrian\Transformers\Models\Auto\AutoModelForVision2Seq;
 use Codewithkyrian\Transformers\Models\Auto\AutoModelForZeroShotObjectDetection;
 use Codewithkyrian\Transformers\Models\Pretrained\PretrainedModel;
-use Codewithkyrian\Transformers\PretrainedTokenizers\AutoTokenizer;
-use Codewithkyrian\Transformers\PretrainedTokenizers\PretrainedTokenizer;
+use Codewithkyrian\Transformers\PreTrainedTokenizers\AutoTokenizer;
+use Codewithkyrian\Transformers\PreTrainedTokenizers\PreTrainedTokenizer;
 use Codewithkyrian\Transformers\Processors\AutoProcessor;
 use Codewithkyrian\Transformers\Processors\Processor;
 
@@ -59,7 +59,7 @@ enum Task: string
     case ASR = 'asr';
 
 
-    public function pipeline(PretrainedModel $model, ?PretrainedTokenizer $tokenizer, ?Processor $processor): Pipeline
+    public function pipeline(PretrainedModel $model, ?PreTrainedTokenizer $tokenizer, ?Processor $processor): Pipeline
     {
         return match ($this) {
             self::SentimentAnalysis,
@@ -214,7 +214,7 @@ public function autoTokenizer(
         ?string   $cacheDir = null,
         string    $revision = 'main',
         ?callable $onProgress = null
-    ): ?PretrainedTokenizer
+    ): ?PreTrainedTokenizer
     {
         return match ($this) {
 

src/Pipelines/ZeroShotClassificationPipeline.php

@@ -7,7 +7,7 @@
 
 use Codewithkyrian\Transformers\Models\Output\SequenceClassifierOutput;
 use Codewithkyrian\Transformers\Models\Pretrained\PretrainedModel;
-use Codewithkyrian\Transformers\PretrainedTokenizers\PretrainedTokenizer;
+use Codewithkyrian\Transformers\PreTrainedTokenizers\PreTrainedTokenizer;
 use Codewithkyrian\Transformers\Utils\Math;
 use function Codewithkyrian\Transformers\Utils\timeUsage;
 
@@ -59,7 +59,7 @@ class ZeroShotClassificationPipeline extends Pipeline
 
     protected mixed $contradictionId;
 
-    public function __construct(Task|string $task, PretrainedModel $model, ?PretrainedTokenizer $tokenizer = null, ?string $processor = null)
+    public function __construct(Task|string $task, PretrainedModel $model, ?PreTrainedTokenizer $tokenizer = null, ?string $processor = null)
     {
         parent::__construct($task, $model, $tokenizer, $processor);
 
@@ -140,4 +140,4 @@ public function __invoke(array|string $inputs, ...$args): array
 
         return $isBatched ? $toReturn : $toReturn[0];
     }
-}
+}

…PretrainedTokenizers/AlbertTokenizer.php …PreTrainedTokenizers/AlbertTokenizer.phpsrc/PretrainedTokenizers/AlbertTokenizer.php renamed to src/PreTrainedTokenizers/AlbertTokenizer.php src/PretrainedTokenizers/AlbertTokenizer.php renamed to src/PreTrainedTokenizers/AlbertTokenizer.php

@@ -3,12 +3,12 @@
 declare(strict_types=1);
 
 
-namespace Codewithkyrian\Transformers\PretrainedTokenizers;
+namespace Codewithkyrian\Transformers\PreTrainedTokenizers;
 
 /**
  * Albert tokenizer
  */
-class AlbertTokenizer extends PretrainedTokenizer
+class AlbertTokenizer extends PreTrainedTokenizer
 {
     protected bool $returnTokenTypeIds = true;
-}
+}

…c/PretrainedTokenizers/AutoTokenizer.php …c/PreTrainedTokenizers/AutoTokenizer.phpsrc/PretrainedTokenizers/AutoTokenizer.php renamed to src/PreTrainedTokenizers/AutoTokenizer.php src/PretrainedTokenizers/AutoTokenizer.php renamed to src/PreTrainedTokenizers/AutoTokenizer.php

@@ -3,7 +3,7 @@
 declare(strict_types=1);
 
 
-namespace Codewithkyrian\Transformers\PretrainedTokenizers;
+namespace Codewithkyrian\Transformers\PreTrainedTokenizers;
 
 use Codewithkyrian\Transformers\Tokenizers\Tokenizer;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -75,15 +75,15 @@ class AutoTokenizer
      * @param string $revision
      * @param mixed $legacy
      * @param OutputInterface|null $output
-     * @return PretrainedTokenizer|null
+     * @return PreTrainedTokenizer|null
      */
     public static function fromPretrained(
         string           $modelNameOrPath,
         ?string          $cacheDir = null,
         string           $revision = 'main',
         mixed            $legacy = null,
         ?callable $onProgress = null
-    ): ?PretrainedTokenizer
+    ): ?PreTrainedTokenizer
     {
         ['tokenizerJson' => $tokenizerJson, 'tokenizerConfig' => $tokenizerConfig] =
             Tokenizer::load($modelNameOrPath, $cacheDir, $revision, $legacy, $onProgress);
@@ -107,4 +107,4 @@ public static function fromPretrained(
 
         return new $cls($tokenizerJson, $tokenizerConfig);
     }
-}
+}