Multibyte string support

Author: GrahamCampbell

composer.json

@@ -13,7 +13,8 @@
   ],
 
   "require": {
-    "php": ">=5.4.0"
+    "php": ">=5.4.0",
+    "symfony/polyfill-mbstring": "^1.4"
   },
 
   "require-dev": {

src/FnDispatcher.php

@@ -62,7 +62,7 @@ private function fn_contains(array $args)
         if (is_array($args[0])) {
             return in_array($args[1], $args[0]);
         } elseif (is_string($args[1])) {
-            return strpos($args[0], $args[1]) !== false;
+            return mb_strpos($args[0], $args[1], 0, 'UTF-8') !== false;
         } else {
             return null;
         }
@@ -72,7 +72,7 @@ private function fn_ends_with(array $args)
     {
         $this->validate('ends_with', $args, [['string'], ['string']]);
         list($search, $suffix) = $args;
-        return $suffix === '' || substr($search, -strlen($suffix)) === $suffix;
+        return $suffix === '' || mb_substr($search, -mb_strlen($suffix, 'UTF-8'), null, 'UTF-8') === $suffix;
     }
 
     private function fn_floor(array $args)
@@ -112,7 +112,7 @@ private function fn_keys(array $args)
     private function fn_length(array $args)
     {
         $this->validate('length', $args, [['string', 'array', 'object']]);
-        return is_string($args[0]) ? strlen($args[0]) : count((array) $args[0]);
+        return is_string($args[0]) ? mb_strlen($args[0], 'UTF-8') : count((array) $args[0]);
     }
 
     private function fn_max(array $args)
@@ -207,7 +207,7 @@ private function fn_starts_with(array $args)
     {
         $this->validate('starts_with', $args, [['string'], ['string']]);
         list($search, $prefix) = $args;
-        return $prefix === '' || strpos($search, $prefix) === 0;
+        return $prefix === '' || mb_strpos($search, $prefix, 0, 'UTF-8') === 0;
     }
 
     private function fn_type(array $args)
@@ -240,7 +240,7 @@ private function fn_to_number(array $args)
         if ($type == 'number') {
             return $value;
         } elseif ($type == 'string' && is_numeric($value)) {
-            return strpos($value, '.') ? (float) $value : (int) $value;
+            return mb_strpos($value, '.', 0, 'UTF-8') ? (float) $value : (int) $value;
         } else {
             return null;
         }
@@ -282,7 +282,7 @@ private function fn_map(array $args)
 
     private function typeError($from, $msg)
     {
-        if (strpos($from, ':')) {
+        if (mb_strpos($from, ':', 0, 'UTF-8')) {
             list($fn, $pos) = explode(':', $from);
             throw new \RuntimeException(
                 sprintf('Argument %d of %s %s', $pos, $fn, $msg)

src/Lexer.php

@@ -340,7 +340,7 @@ public function tokenize($input)
         eof:
         $tokens[] = [
             'type'  => self::T_EOF,
-            'pos'   => strlen($input),
+            'pos'   => mb_strlen($input, 'UTF-8'),
             'value' => null
         ];
 

src/Utils.php

@@ -214,7 +214,7 @@ private static function adjustSlice($length, $start, $stop, $step)
     private static function sliceIndices($subject, $start, $stop, $step)
     {
         $type = gettype($subject);
-        $len = $type == 'string' ? strlen($subject) : count($subject);
+        $len = $type == 'string' ? mb_strlen($subject, 'UTF-8') : count($subject);
         list($start, $stop, $step) = self::adjustSlice($len, $start, $stop, $step);
 
         $result = [];

tests/ComplianceTest.php

@@ -61,6 +61,7 @@ public function testPassesCompliance(
 
         $file = __DIR__ . '/compliance/' . $file . '.json';
         $failure .= "\n{$compiledStr}php bin/jp.php --file {$file} --suite {$suite} --case {$case}\n\n"
+            . "Result: " . $this->prettyJson($evalResult) . "\n\n"
             . "Expected: " . $this->prettyJson($result) . "\n\n";
         $failure .= 'Associative? ' . var_export($asAssoc, true) . "\n\n";
 

tests/compliance/functions.json

@@ -179,6 +179,10 @@
       "expression": "length('abc')",
       "result": 3
     },
+    {
+      "expression": "length('✓foo')",
+      "result": 4
+    },
     {
       "expression": "length('')",
       "result": 0