Merge pull request #95 from DirectoryTree/add-number-token

Parse `Number` tokens in tokenizer

Author: stevebauman

src/Connection/ImapTokenizer.php

@@ -9,6 +9,7 @@
 use DirectoryTree\ImapEngine\Connection\Tokens\ListClose;
 use DirectoryTree\ImapEngine\Connection\Tokens\ListOpen;
 use DirectoryTree\ImapEngine\Connection\Tokens\Literal;
+use DirectoryTree\ImapEngine\Connection\Tokens\Number;
 use DirectoryTree\ImapEngine\Connection\Tokens\QuotedString;
 use DirectoryTree\ImapEngine\Connection\Tokens\ResponseCodeClose;
 use DirectoryTree\ImapEngine\Connection\Tokens\ResponseCodeOpen;
@@ -120,8 +121,8 @@ public function nextToken(): ?Token
             return $this->readLiteral();
         }
 
-        // Otherwise, parse an atom.
-        return $this->readAtom();
+        // Otherwise, parse a number or atom.
+        return $this->readNumberOrAtom();
     }
 
     /**
@@ -300,6 +301,64 @@ protected function readLiteral(): Literal
         return new Literal($literal);
     }
 
+    /**
+     * Reads a number or atom token.
+     */
+    protected function readNumberOrAtom(): Token
+    {
+        $position = $this->position;
+
+        // First char must be a digit to even consider a number.
+        if (! ctype_digit($this->buffer[$position] ?? '')) {
+            return $this->readAtom();
+        }
+
+        // Walk forward to find the end of the digit run.
+        while (ctype_digit($this->buffer[$position] ?? '')) {
+            $position++;
+
+            $this->ensureBuffer($position - $this->position + 1);
+        }
+
+        $next = $this->buffer[$position] ?? null;
+
+        // If next is EOF or a delimiter, it's a Number.
+        if ($next === null || $this->isDelimiter($next)) {
+            return $this->readNumber();
+        }
+
+        // Otherwise it's an Atom.
+        return $this->readAtom();
+    }
+
+    /**
+     * Reads a number token.
+     *
+     * A number consists of one or more digit characters and represents a numeric value.
+     */
+    protected function readNumber(): Number
+    {
+        $start = $this->position;
+
+        while (true) {
+            $this->ensureBuffer(1);
+
+            $char = $this->currentChar();
+
+            if ($char === null) {
+                break;
+            }
+
+            if (! ctype_digit($char)) {
+                break;
+            }
+
+            $this->advance();
+        }
+
+        return new Number(substr($this->buffer, $start, $this->position - $start));
+    }
+
     /**
      * Reads an atom token.
      *
@@ -311,6 +370,7 @@ protected function readAtom(): Atom
 
         while (true) {
             $this->ensureBuffer(1);
+
             $char = $this->currentChar();
 
             if ($char === null) {

src/Connection/Tokens/Number.php

@@ -0,0 +1,5 @@
+<?php
+
+namespace DirectoryTree\ImapEngine\Connection\Tokens;
+
+class Number extends Token {}

src/MessageQuery.php

@@ -8,7 +8,7 @@
 use DirectoryTree\ImapEngine\Connection\ImapQueryBuilder;
 use DirectoryTree\ImapEngine\Connection\Responses\MessageResponseParser;
 use DirectoryTree\ImapEngine\Connection\Responses\UntaggedResponse;
-use DirectoryTree\ImapEngine\Connection\Tokens\Atom;
+use DirectoryTree\ImapEngine\Connection\Tokens\Token;
 use DirectoryTree\ImapEngine\Enums\ImapFetchIdentifier;
 use DirectoryTree\ImapEngine\Enums\ImapFlag;
 use DirectoryTree\ImapEngine\Exceptions\ImapCommandException;
@@ -305,7 +305,7 @@ protected function search(): Collection
         ]);
 
         return new Collection(array_map(
-            fn (Atom $token) => $token->value,
+            fn (Token $token) => $token->value,
             $response->tokensAfter(2)
         ));
     }

tests/Unit/Connection/ImapTokenizerTest.php

@@ -8,6 +8,7 @@
 use DirectoryTree\ImapEngine\Connection\Tokens\ListClose;
 use DirectoryTree\ImapEngine\Connection\Tokens\ListOpen;
 use DirectoryTree\ImapEngine\Connection\Tokens\Literal;
+use DirectoryTree\ImapEngine\Connection\Tokens\Number;
 use DirectoryTree\ImapEngine\Connection\Tokens\QuotedString;
 use DirectoryTree\ImapEngine\Connection\Tokens\ResponseCodeClose;
 use DirectoryTree\ImapEngine\Connection\Tokens\ResponseCodeOpen;
@@ -257,7 +258,7 @@
     expect($token->value)->toBe('UIDNEXT');
 
     $token = $tokenizer->nextToken();
-    expect($token)->toBeInstanceOf(Atom::class);
+    expect($token)->toBeInstanceOf(Number::class);
     expect($token->value)->toBe('1000');
 
     $token = $tokenizer->nextToken();
@@ -334,6 +335,28 @@
     ],
 ]);
 
+test('tokenizer handles edge cases', function (string $feed, Token ...$expectedTokens) {
+    $stream = new FakeStream;
+    $stream->open();
+
+    $stream->feed($feed);
+
+    $tokenizer = new ImapTokenizer($stream);
+
+    foreach ($expectedTokens as $expectedToken) {
+        $actualToken = $tokenizer->nextToken();
+
+        expect($actualToken)->toBeInstanceOf(get_class($expectedToken));
+        expect($actualToken->value)->toBe($expectedToken->value);
+    }
+})->with([
+    ['UID 48273)', new Atom('UID'), new Number('48273'), new ListClose(')')],
+    ['* 23 EXISTS', new Atom('*'), new Number('23'), new Atom('EXISTS')],
+    ['OK (0.002 secs)', new Atom('OK'), new ListOpen('('), new Atom('0.002'), new Atom('secs'), new ListClose(')')],
+    ['OK 404NotFound', new Atom('OK'), new Atom('404NotFound')],
+    ['A1 OK [UIDNEXT 1000]', new Atom('A1'), new Atom('OK'), new ResponseCodeOpen('['), new Atom('UIDNEXT'), new Number('1000'), new ResponseCodeClose(']')],
+]);
+
 test('all tokens implement the token interface', function (string $data) {
     $stream = new FakeStream;
     $stream->open();