Encoder: invalid UTF-8 sequences are replaced with U+FFFD [Closes #76]

Author: dg

src/Neon/Encoder.php

@@ -65,8 +65,14 @@ public function valueToNode(mixed $val, bool $blockMode = false): Node
 			$node->items = $this->arrayToNodes($val, $blockMode);
 			return $node;
 
-		} elseif (is_string($val) && Lexer::requiresDelimiters($val)) {
-			return new Node\StringNode($val, $this->indentation);
+		} elseif (is_string($val)) {
+			if (preg_match('//u', $val) === false) {
+				trigger_error('Invalid UTF-8 sequence in string, replaced with U+FFFD', E_USER_WARNING);
+				$val = json_decode(json_encode($val, JSON_INVALID_UTF8_SUBSTITUTE));
+			}
+			return Lexer::requiresDelimiters($val)
+				? new Node\StringNode($val, $this->indentation)
+				: new Node\LiteralNode($val);
 
 		} else {
 			return new Node\LiteralNode($val);

tests/Neon/Encoder.phpt

@@ -185,3 +185,10 @@ Assert::same(
 	"inner:\n    msg: '''\n        string\n        with newline\n    '''\n\n",
 	Neon::encode(['inner' => ['msg' => "string\nwith newline"]], true, '    '),
 );
+
+
+// Invalid UTF-8 is replaced with U+FFFD and triggers warning
+Assert::error(function () {
+	$result = Neon::encode("\x80");
+	Assert::same("\u{FFFD}", $result);
+}, E_USER_WARNING, 'Invalid UTF-8 sequence in string, replaced with U+FFFD');