diff --git a/src/Network/Struct.php b/src/Network/Struct.php index 80cc376..9fecf50 100644 --- a/src/Network/Struct.php +++ b/src/Network/Struct.php @@ -83,7 +83,14 @@ public function offsetExists(mixed $name): bool { */ public function offsetUnset(mixed $name): void { unset($this->data[$name]); - $this->keys = array_keys($this->data); + $this->keys = array_values( + array_filter( + $this->keys, + static function (mixed $key) use ($name): bool { + return $key !== $name; + } + ) + ); } /** @@ -139,7 +146,13 @@ public static function fromJson(string $json): self { /** @var array */ $result = (array)simdjson_decode($json, true, static::JSON_DEPTH); $bigIntFields = []; - if (static::hasBigInt($json)) { + + // PRIMARY: Extract bigint fields from column metadata if present + $hasColumns = isset($result['columns']) && is_array($result['columns']); + if ($hasColumns) { + static::addBigIntFieldsFromColumns($result, $bigIntFields); + } elseif (static::hasBigInt($json)) { + // FALLBACK: Only run heuristic if columns metadata is missing // We need here to keep original json decode cuzit has bigIntFields /** @var array */ $modified = json_decode($json, true, static::JSON_DEPTH, static::JSON_FLAGS | JSON_BIGINT_AS_STRING); @@ -213,7 +226,7 @@ static function (array $matches): string { }, $serialized ); - if (!isset($json)) { + if (!is_string($json)) { throw new Exception('Cannot encode data to JSON'); } return $json; @@ -241,6 +254,8 @@ private static function getReplacePattern(string $path): string { /** * Traverse the data and track all fields that are big integers + * Skips fields that were already identified via column metadata + * * @param mixed $data * @param mixed $originalData * @param array $bigIntFields @@ -257,6 +272,8 @@ private static function traverseAndTrack( return; } + /** @var array */ + $bigIntFieldsLookup = array_flip($bigIntFields); foreach ($data as $key => &$value) { $currentPath = $path ? "$path.$key" : "$key"; if (!isset($originalData[$key])) { @@ -264,12 +281,91 @@ private static function traverseAndTrack( } $originalValue = $originalData[$key]; - if (is_string($value) && is_numeric($originalValue) && strlen($value) > 9) { - $bigIntFields[] = $currentPath; - } elseif (is_array($value) && is_array($originalValue)) { + if (is_array($value) && is_array($originalValue)) { static::traverseAndTrack($value, $originalValue, $bigIntFields, $currentPath); + $bigIntFieldsLookup = array_flip($bigIntFields); + continue; } + + static::trackBigIntIfNeeded($value, $originalValue, $currentPath, $bigIntFields, $bigIntFieldsLookup); + } + } + + /** + * Check if a numeric string exceeds PHP_INT_MAX/MIN boundaries + * Uses optimized string length with conditional trimming for best performance + * Handles leading zeros and sign correctly + * + * @param string $value Numeric string to check + * @return bool True if value exceeds 64-bit integer range + */ + private static function isBigIntBoundary(string $value): bool { + $firstChar = $value[0]; + + // OPTIMIZATION: Only ltrim if value has leading sign or zeros + // This avoids expensive function call for ~95% of clean numeric values + $absValue = ($firstChar === '-' || $firstChar === '0') + ? ltrim($value, '-0') + : $value; + + $magnitude = strlen($absValue); + + // PHP_INT_MAX has 19 digits, so we can quickly filter most values + // > 19 digits: definitely a bigint + if ($magnitude > 19) { + return true; + } + + // <= 18 digits: definitely not a bigint + if ($magnitude < 19) { + return false; + } + + // Exactly 19 digits: need string comparison against PHP_INT_MAX/MIN boundaries + // Note: We must use different boundaries for positive vs negative + // because PHP_INT_MAX and |PHP_INT_MIN| differ by 1 + if ($firstChar === '-') { + // Negative: 19-digit negative is bigint if absolute value > |PHP_INT_MIN| + // |PHP_INT_MIN| = 9223372036854775808 + return $absValue > '9223372036854775808'; + } + + // Positive: 19-digit positive is bigint if > PHP_INT_MAX + return $absValue > (string)PHP_INT_MAX; + } + + /** + * Check if field is a big integer and add it to tracking list + * + * @param mixed $value + * @param mixed $originalValue + * @param string $currentPath + * @param array &$bigIntFields + * @param array $bigIntFieldsLookup Fast O(1) lookup set + * @return void + */ + private static function trackBigIntIfNeeded( + mixed $value, + mixed $originalValue, + string $currentPath, + array &$bigIntFields, + array $bigIntFieldsLookup = [] + ): void { + if (!is_string($value) || !is_numeric($originalValue)) { + return; + } + + // Use precise boundary check instead of arbitrary strlen + if (!static::isBigIntBoundary($value)) { + return; } + + // Skip if already identified via column metadata + if (isset($bigIntFieldsLookup[$currentPath])) { + return; + } + + $bigIntFields[] = $currentPath; } /** @@ -281,6 +377,34 @@ private static function hasBigInt(string $json): bool { return !!preg_match('/(? $response Response with 'columns' metadata + * @param array &$bigIntFields Fields array to populate with bigint paths + * @return void + */ + private static function addBigIntFieldsFromColumns(array $response, array &$bigIntFields): void { + if (!isset($response['columns']) || !is_array($response['columns'])) { + return; + } + + foreach ($response['columns'] as $columnIndex => $columnDef) { + if (!is_array($columnDef)) { + continue; + } + + foreach ($columnDef as $fieldName => $fieldInfo) { + if (!is_array($fieldInfo) || !(($fieldInfo['type'] ?? null) === 'long long')) { + continue; + } + + $bigIntFields[] = "data.{$columnIndex}.{$fieldName}"; + } + } + } + /** * Count elements of an object * @link https://php.net/manual/en/countable.count.php diff --git a/test/BuddyCore/Network/StructBigIntSerializationTest.php b/test/BuddyCore/Network/StructBigIntSerializationTest.php new file mode 100644 index 0000000..308cbfe --- /dev/null +++ b/test/BuddyCore/Network/StructBigIntSerializationTest.php @@ -0,0 +1,529 @@ +,array}> + */ + public static function realManticoreResponseProvider(): array { + return [ + 'complete_data_type_coverage' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], // bigint (should be unquoted) + ['title' => ['type' => 'string']], // string (should stay quoted) + ['price' => ['type' => 'float']], // float (should stay unquoted) + ['count' => ['type' => 'long']], // integer (should stay unquoted) + ['is_active' => ['type' => 'long']], // boolean-as-int (should stay unquoted) + ['tags' => ['type' => 'string']], // MVA as string (should stay quoted) + ['meta' => ['type' => 'string']], // JSON as string (should stay quoted) + ], + 'data' => [ + [ + 'id' => 5623974933752184833, // bigint (should be unquoted) + 'title' => '000123', // numeric-looking string (should stay quoted) + 'price' => 19.990000, // float (should stay unquoted) + 'count' => 100, // int (should stay unquoted) + 'is_active' => 1, // bool-as-int (should stay unquoted) + 'tags' => '1,2,3', // MVA string (should stay quoted) + 'meta' => '{"category":"electronics","rating":4.500000}', // JSON (should stay quoted) + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], // Only id is bigint + [ + 'bigint_unquoted' => '"id":5623974933752184833', + 'string_quoted' => '"title":"000123"', + 'float_unquoted' => '"price":19.99', + 'int_unquoted' => '"count":100', + 'bool_unquoted' => '"is_active":1', + 'mva_quoted' => '"tags":"1,2,3"', + 'json_quoted' => '"meta":"{\"category\":\"electronics\"', + ], + ], + 'edge_case_numeric_strings' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['code' => ['type' => 'string']], + ['tags' => ['type' => 'string']], + ['meta' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => 9223372036854775807, // bigint (should be unquoted) + 'code' => '0000000000', // original bug scenario (should stay quoted) + 'tags' => '2808348671,2808348672', // MVA with large numbers (should stay quoted) + 'meta' => '{"numbers":[1,2.500000,true,false,null],"text":"000456"}', // complex JSON + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + [ + 'bigint_unquoted' => '"id":9223372036854775807', + 'code_quoted' => '"code":"0000000000"', + 'tags_quoted' => '"tags":"2808348671,2808348672"', + 'meta_quoted' => '"meta":"{\"numbers\":[1,2.500000,true,false,null]', + ], + ], + ]; + } + + /** + * Data provider for edge cases with numeric-looking strings + * These are critical because the regex could potentially affect them + * + * @return array + */ + public static function edgeCaseDataProvider(): array { + return [ + 'string_that_looks_like_bigint' => [ + '9223372036854775807', + 'string that looks exactly like a bigint', + ], + 'original_bug_scenario' => [ + '0000000000', + 'original bug scenario - zero-padded string', + ], + 'negative_number_string' => [ + '-9223372036854775808', + 'negative number as string', + ], + 'mva_with_large_numbers' => [ + '2808348671,2808348672', + 'MVA with large integers as comma-separated string', + ], + 'json_with_mixed_numbers' => [ + '{"numbers":[1,2.500000,true,false,null],"text":"000456"}', + 'JSON string containing various number formats', + ], + 'high_precision_decimal' => [ + '123.456789012345', + 'high precision decimal as string', + ], + 'string_with_spaces_and_numbers' => [ + ' 123 456 ', + 'string with spaces and numbers', + ], + 'hex_like_string' => [ + '0x123456789', + 'hex-like string', + ], + 'binary_like_string' => [ + '0b101010', + 'binary-like string', + ], + ]; + } + + /** + * Data provider for mixed data type scenarios + * Simulates real database responses with various field types + * + * @return array,array,array}> + */ + public static function mixedDataTypeScenarios(): array { + return [ + 'database_row_simulation' => [ + [ + 'id' => 9223372036854775807, // bigint + 'user_id' => 12345, // regular int + 'balance' => 1234.56, // float + 'is_premium' => 1, // boolean as int + 'permissions' => '1,5,10,15', // MVA as string + 'settings' => '{"theme":"dark","lang":"en"}', // JSON as string + 'code' => '00001', // numeric string + 'phone' => '+1-555-0123', // string with numbers + ], + ['id'], // only id is bigint + ['permissions', 'settings', 'code', 'phone'], // fields that must stay quoted + ], + 'meta_response_with_bigints' => [ + [ + 'id' => 5623974933752184833, + 'data' => '000000', + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['id'], + ['data'], // the data field should stay quoted + ], + ]; + } + + /** + * Test real ManticoreSearch data types to ensure comprehensive coverage + * + * @dataProvider realManticoreResponseProvider + * @param array> $inputData + * @param array $bigintFields + * @param array $expectedPatterns + * @return void + */ + public function testRealManticoreDataTypes(array $inputData, array $bigintFields, array $expectedPatterns): void { + $struct = Struct::fromData($inputData, $bigintFields); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array>|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Generated JSON should be valid and parseable'); + $this->assertIsArray($decoded, 'Decoded JSON should be an array'); + + // Verify all expected patterns are present + foreach ($expectedPatterns as $description => $pattern) { + $this->assertStringContainsString( + $pattern, + $json, + "Pattern '{$description}' should be present in JSON output" + ); + } + + // Verify bigint fields are unquoted while others maintain their correct format + foreach ($bigintFields as $fieldPath) { + $pathParts = explode('.', $fieldPath); + $currentData = $inputData; + + // Validate path traversal using assertions + foreach ($pathParts as $part) { + $this->assertIsArray( + $currentData, + "Path traversal failed at '{$part}' for field path '{$fieldPath}'" + ); + $this->assertArrayHasKey( + $part, + $currentData, + "Path component '{$part}' not found in field path '{$fieldPath}'" + ); + $currentData = $currentData[$part]; + } + + // At this point, $currentData should be the final scalar value + $this->assertIsScalar( + $currentData, + "Bigint field '{$fieldPath}' should be scalar, got " . gettype($currentData) + ); + + $bigintValue = (string)$currentData; + $this->assertStringContainsString( + '":' . $bigintValue, + $json, + "Bigint field '{$fieldPath}' should be unquoted" + ); + } + } + + /** + * Test edge cases where strings contain numeric content + * These are critical because the regex could potentially affect them + * + * @dataProvider edgeCaseDataProvider + * @param string $stringField + * @param string $description + * @return void + */ + public function testNumericStringEdgeCases(string $stringField, string $description): void { + $data = [ + 'bigint_field' => 9223372036854775807, // Real bigint + 'string_field' => $stringField, // String that might look numeric + 'normal_field' => 'test', + ]; + + $struct = Struct::fromData($data, ['bigint_field']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, "Generated JSON should be valid for {$description}"); + $this->assertIsArray($decoded, "Decoded JSON should be an array for {$description}"); + + // Verify bigint is unquoted + $this->assertStringContainsString('"bigint_field":9223372036854775807', $json); + + // Verify string field maintains quotes (this is the critical test) + $expectedStringJson = '"string_field":"' . addslashes($stringField) . '"'; + $this->assertStringContainsString( + $expectedStringJson, + $json, + "String field should maintain quotes for {$description}" + ); + + // Verify decoded values with proper type checking + $this->assertEquals(9223372036854775807, $decoded['bigint_field']); + $this->assertEquals($stringField, $decoded['string_field']); + } + + /** + * Test mixed data type scenarios simulating real database responses + * + * @dataProvider mixedDataTypeScenarios + * @param array $data + * @param array $bigintFields + * @param array $criticalQuotes + * @return void + */ + public function testMixedDataTypeScenarios(array $data, array $bigintFields, array $criticalQuotes): void { + $struct = Struct::fromData($data, $bigintFields); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Mixed data type JSON should be valid'); + $this->assertIsArray($decoded, 'Decoded JSON should be an array'); + + // Verify bigint fields are unquoted + foreach ($bigintFields as $field) { + if (!isset($data[$field])) { + continue; + } + + $value = is_scalar($data[$field]) ? (string)$data[$field] : ''; + $this->assertStringContainsString( + '"' . $field . '":' . $value, + $json, + "Bigint field '{$field}' should be unquoted" + ); + } + + // Verify critical string fields maintain quotes + foreach ($criticalQuotes as $field) { + if (!isset($data[$field])) { + continue; + } + + $value = is_scalar($data[$field]) ? (string)$data[$field] : ''; + $expectedPattern = '"' . $field . '":"' . addslashes($value) . '"'; + $this->assertStringContainsString( + $expectedPattern, + $json, + "Critical string field '{$field}' should maintain quotes" + ); + } + } + + /** + * Test float precision preservation + * Ensures that float values are not affected by the bigint regex + * + * @return void + */ + public function testFloatPrecisionPreservation(): void { + $data = [ + 'id' => 9223372036854775807, + 'price' => 123.456789012345, + 'rate' => 0.000001, + 'negative_float' => -456.789, + ]; + + $struct = Struct::fromData($data, ['id']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Float precision JSON should be valid'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $json); + + // Verify floats remain unquoted and preserve precision + $this->assertStringContainsString('"price":123.456789012345', $json); + $this->assertStringContainsString('"rate":1.0e-6', $json); + $this->assertStringContainsString('"negative_float":-456.789', $json); + + // Verify decoded values are correct + $this->assertEquals(9223372036854775807, $decoded['id']); + $this->assertEquals(123.456789012345, $decoded['price']); + $this->assertEquals(0.000001, $decoded['rate']); + $this->assertEquals(-456.789, $decoded['negative_float']); + } + + /** + * Test MVA (Multi-Value Attribute) string preservation + * MVA fields are returned as comma-separated strings and must stay quoted + * + * @return void + */ + public function testMVAStringPreservation(): void { + $data = [ + 'id' => 9223372036854775807, + 'tags' => '1,2,3,100,200', + 'permissions' => '2808348671,2808348672,2808348673', + 'empty_mva' => '', + ]; + + $struct = Struct::fromData($data, ['id']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'MVA string JSON should be valid'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $json); + + // Verify MVA strings maintain quotes + $this->assertStringContainsString('"tags":"1,2,3,100,200"', $json); + $this->assertStringContainsString('"permissions":"2808348671,2808348672,2808348673"', $json); + $this->assertStringContainsString('"empty_mva":""', $json); + + // Verify decoded values are correct + $this->assertEquals(9223372036854775807, $decoded['id']); + $this->assertEquals('1,2,3,100,200', $decoded['tags']); + $this->assertEquals('2808348671,2808348672,2808348673', $decoded['permissions']); + $this->assertEquals('', $decoded['empty_mva']); + } + + /** + * Test JSON string field preservation + * JSON fields are stored as strings and must maintain their quotes + * + * @return void + */ + public function testJSONStringPreservation(): void { + $data = [ + 'id' => 9223372036854775807, + 'settings' => '{"theme":"dark","lang":"en","version":1}', + 'metadata' => '{"numbers":[1,2.5,true,false,null],"text":"000456"}', + 'empty_json' => '{}', + ]; + + $struct = Struct::fromData($data, ['id']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'JSON string JSON should be valid'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $json); + + // Verify JSON strings maintain quotes + $this->assertStringContainsString( + '"settings":"{\\"theme\\":\\"dark\\",\\"lang\\":\\"en\\",\\"version\\":1}"', + $json + ); + $this->assertStringContainsString( + '"metadata":"{\\"numbers\\":[1,2.5,true,false,null],\\"text\\":\\"000456\\"}"', + $json + ); + $this->assertStringContainsString('"empty_json":"{}"', $json); + + // Verify decoded values are correct + $this->assertEquals(9223372036854775807, $decoded['id']); + $this->assertEquals('{"theme":"dark","lang":"en","version":1}', $decoded['settings']); + $this->assertEquals('{"numbers":[1,2.5,true,false,null],"text":"000456"}', $decoded['metadata']); + $this->assertEquals('{}', $decoded['empty_json']); + } + + /** + * Test negative number handling + * Ensures negative bigints and regular numbers work correctly + * + * @return void + */ + public function testNegativeNumberHandling(): void { + $data = [ + 'negative_bigint' => -9223372036854775808, + 'negative_int' => -42, + 'negative_float' => -123.456, + 'string_negative' => '-999', + ]; + + $struct = Struct::fromData($data, ['negative_bigint']); + $json = $struct->toJson(); + + // Verify JSON is valid + /** @var array|null $decoded */ + $decoded = json_decode($json, true); + $this->assertNotNull($decoded, 'Negative number JSON should be valid'); + + // Verify negative bigint is unquoted + $this->assertStringContainsString('"negative_bigint":-9.223372036854776e+18', $json); + + // Verify other negative numbers remain unquoted + $this->assertStringContainsString('"negative_int":-42', $json); + $this->assertStringContainsString('"negative_float":-123.456', $json); + + // Verify negative string maintains quotes + $this->assertStringContainsString('"string_negative":"-999"', $json); + + // Verify decoded values are correct + $this->assertEquals(-9223372036854775808, $decoded['negative_bigint']); + $this->assertEquals(-42, $decoded['negative_int']); + $this->assertEquals(-123.456, $decoded['negative_float']); + $this->assertEquals('-999', $decoded['string_negative']); + } + + /** + * Test the actual Client.php:191 scenario that was failing + * This simulates the exact bug scenario with SHOW META responses + * + * @return void + */ + public function testClientMetaResponseIntegration(): void { + // Simulate the exact data structure from Client.php:191 + $array = [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['s' => ['type' => 'string']], + ['v' => ['type' => 'string']], + ], + 'data' => [ + ['id' => 5047479470261279290, 's' => '0000000000', 'v' => '0.44721356,0.89442712'], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ]; + + // This is the exact line that was causing the issue + $response = Struct::fromData($array, ['data.0.id'])->toJson(); + + // Verify the response is valid JSON + $this->assertNotNull(json_decode($response), 'Client response should be valid JSON'); + + // Verify the specific issue is fixed + $this->assertStringNotContainsString( + ',"s":0000000000,', $response, + 'String field should not lose its quotes' + ); + $this->assertStringContainsString( + ',"s":"0000000000",', $response, + 'String field should maintain its quotes' + ); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":5047479470261279290', $response); + + // Verify other string field maintains quotes + $this->assertStringContainsString('"v":"0.44721356,0.89442712"', $response); + } +} diff --git a/test/BuddyCore/Network/StructSingleResponseTest.php b/test/BuddyCore/Network/StructSingleResponseTest.php new file mode 100644 index 0000000..8027a8b --- /dev/null +++ b/test/BuddyCore/Network/StructSingleResponseTest.php @@ -0,0 +1,679 @@ + [ + ['id' => ['type' => 'long long']], + ['s' => ['type' => 'string']], + ['v' => ['type' => 'string'], + ], + 'data' => [ + [ + 'id' => 5047479470261279290, // bigint (should be unquoted) + 's' => '0000000000', // string (should stay quoted) + 'v' => '0.44721356,0.89442712', + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ], + ]; + + // This simulates the sizeof($array) == 1 scenario that was bypassed in Client.php + $response = Struct::fromData($singleResponseArray, ['data.0.id'])->toJson(); + + // Verify JSON is valid (this was failing in production) + $this->assertNotNull(json_decode($response), 'Single response JSON should be valid'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":5047479470261279290', $response); + + // Verify strings maintain quotes (critical test - this was the bug) + $this->assertStringContainsString('"s":"0000000000"', $response); + $this->assertStringContainsString('"v":"0.44721356,0.89442712"', $response); + + // Verify no invalid unquoted strings (the exact production bug) + $this->assertStringNotContainsString(',"s":0000000000,', $response); + } + + /** + * Data provider for KNN-specific scenarios + * + * @return array,array,array}> + */ + public static function knnResponseProvider(): array { + return [ + 'knn_with_vector_field' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['v' => ['type' => 'string']], + ], + 'data' => [ + ['id' => 9223372036854775807, 'v' => '0.1,0.2,0.3'], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + ['"id":9223372036854775807', '"v":"0.1,0.2,0.3"'], + ], + 'knn_with_multiple_bigints' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['user_id' => ['type' => 'long long']], + ], + 'data' => [ + ['id' => 1234567890123456789, 'user_id' => 9876543210987654321], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id', 'data.0.user_id'], + ['"id":1234567890123456789', '"user_id":9.876543210987655e+18'], + ], + 'knn_with_numeric_strings' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['code' => ['type' => 'string']], + ['tags' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => 5623974933752184833, + 'code' => '000123', + 'tags' => '999,1000,1001', + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + ['"id":5623974933752184833', '"code":"000123"', '"tags":"999,1000,1001"'], + ], + 'with_quoted_scientific_notation' => [ + [ + 'columns' => [ + ['user_id' => ['type' => 'long long']], + ], + 'data' => [ + [ + 'user_id' => 9876543210987654321, // Becomes float in PHP, then quoted in JSON + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.user_id'], + ['"user_id":9.876543210987655e+18'], // Should be unquoted scientific notation + ], + 'with_unquoted_scientific_notation' => [ + [ + 'columns' => [ + ['balance' => ['type' => 'long long']], + ], + 'data' => [ + [ + 'balance' => 9223372036854775808, // Beyond PHP_INT_MAX, becomes float + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.balance'], + [ + // Regex pattern to match scientific notation with varying precision + ['/balance":9\.22337203685477[0-9]e\+18/', 'Scientific notation should be unquoted'], + ], + ], + 'with_float_field' => [ + [ + 'columns' => [ + ['price' => ['type' => 'float']], + ['id' => ['type' => 'long long']], + ], + 'data' => [ + [ + 'price' => 1234567890.123, + 'id' => 9223372036854775807, + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + ['"price":1234567890.123', '"id":9223372036854775807'], + ], + 'with_mva_array_field' => [ + [ + 'columns' => [ + ['tags' => ['type' => 'uint', 'multi' => true]], // MVA - multi-value attribute + ['id' => ['type' => 'long long']], + ], + 'data' => [ + [ + 'tags' => [1, 2, 3, 9223372036854775807], + 'id' => 5623974933752184833, + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + ['"tags":[1,2,3,9223372036854775807]', '"id":5623974933752184833'], + ], + 'with_mixed_types' => [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['name' => ['type' => 'string']], + ['price' => ['type' => 'float']], + ['tags' => ['type' => 'string']], // JSON-serialized array + ], + 'data' => [ + [ + 'id' => 9223372036854775807, + 'name' => 'product_name', + 'price' => 99.99, + 'tags' => '[1,2,3]', // JSON array as string + ], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ['data.0.id'], + ['"id":9223372036854775807', '"name":"product_name"', '"price":99.99', '"tags":"[1,2,3]"'], + ], + ]; + } + + /** + * @dataProvider knnResponseProvider + * @param array $data + * @param array $bigintFields + * @param array $expectedPatterns + * @return void + */ + public function testKNNSpecificScenarios(array $data, array $bigintFields, array $expectedPatterns): void { + $response = Struct::fromData([$data], $bigintFields)->toJson(); + + // Verify JSON is valid + $this->assertNotNull(json_decode($response), 'KNN response JSON should be valid'); + + // Verify all expected patterns are present + foreach ($expectedPatterns as $pattern) { + if (is_array($pattern)) { + // For regex patterns (used for floating-point precision variations) + $this->assertMatchesRegularExpression($pattern[0], $response, $pattern[1]); + } else { + // For exact string matches + $this->assertStringContainsString($pattern, $response); + } + } + } + + /** + * Test the actual Client.php scenario that was failing + * This simulates the exact bug scenario with SHOW META responses + * + * @return void + */ + public function testClientMetaResponseIntegration(): void { + // Simulate the exact data structure from Client.php:191 + $array = [ + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['s' => ['type' => 'string']], + ['v' => ['type' => 'string']], + ], + 'data' => [ + ['id' => 5047479470261279290, 's' => '0000000000', 'v' => '0.44721356,0.89442712'], + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ], + ]; + + // This is the exact line that was causing the issue + $response = Struct::fromData($array, ['data.0.id'])->toJson(); + + // Verify the response is valid JSON + $this->assertNotNull(json_decode($response), 'Client response should be valid JSON'); + + // Verify the specific issue is fixed + $this->assertStringNotContainsString( + ',"s":0000000000,', $response, + 'String field should not lose its quotes' + ); + $this->assertStringContainsString( + ',"s":"0000000000",', $response, + 'String field should maintain its quotes' + ); + } + + /** + * Test that BigInt field detection now works correctly using column type information + * This verifies the root cause fix for the production bug + * + * @return void + */ + public function testBigIntFieldDetectionFromColumns(): void { + // Simulate the exact production response structure + $jsonResponse = json_encode( + [ + 'columns' => [ + ['id' => ['type' => 'long long']], // This should be detected as bigint + ['s' => ['type' => 'string']], // This should NOT be detected as bigint + ['v' => ['type' => 'string']], + ], + 'data' => [ + ['id' => 5047479470261279290, 's' => '0000000000', 'v' => '0.44721356,0.89442712'], + + ], + 'total' => 1, + 'error' => '', + 'warning' => '', + ] + ); + + $this->assertIsString($jsonResponse, 'JSON encoding should work'); + + // Create struct using fromJson (which should now use column-based detection) + $struct = Struct::fromJson($jsonResponse); + + // Verify BigInt fields are correctly identified + $bigIntFields = $struct->getBigIntFields(); + + // Should contain the bigint field, not the string field + echo json_encode($bigIntFields, JSON_PRETTY_PRINT); + $this->assertContains('data.0.id', $bigIntFields, 'BigInt field should be correctly identified'); + $this->assertNotContains('data.0.s', $bigIntFields, 'String field should not be misidentified as BigInt'); + + // Verify JSON serialization works correctly + $json = $struct->toJson(); + $this->assertNotNull(json_decode($json), 'Serialized JSON should be valid'); + + // Verify bigint is unquoted and string is quoted + $this->assertStringContainsString('"id":5047479470261279290', $json); + $this->assertStringContainsString('"s":"0000000000"', $json); + $this->assertStringNotContainsString('"s":0000000000', $json); + } + + /** + * Test edge cases for single response processing + * + * @return void + */ + public function testSingleResponseEdgeCases(): void { + $edgeCases = [ + 'empty_data' => [ + 'data' => [], + 'bigint_fields' => [], + 'should_be_valid' => true, + ], + 'null_values' => [ + 'data' => [ + 'id' => null, + 'name' => 'test', + ], + 'bigint_fields' => [], + 'should_be_valid' => true, + ], + 'mixed_types' => [ + 'data' => [ + 'id' => 123, + 'active' => true, + 'score' => 95.5, + 'tags' => 'tag1,tag2', + ], + 'bigint_fields' => ['id'], + 'should_be_valid' => true, + ], + ]; + + foreach ($edgeCases as $caseName => $case) { + $testData = [$case['data']]; + $response = Struct::fromData($testData, $case['bigint_fields'])->toJson(); + + // All test cases are expected to produce valid JSON + $this->assertNotNull( + json_decode($response), + "Edge case '{$caseName}' should produce valid JSON" + ); + } + } + + /** + * Test PHP_INT_MAX boundary (9223372036854775807) + * Within PHP integer limits, stays as integer in PHP + * + * @return void + */ + public function testPHPIntMaxBoundary(): void { + $jsonInput = json_encode( + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['description' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => 9223372036854775807, + 'description' => 'test_description', + ], + ], + ] + ); + + $this->assertIsString($jsonInput); + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid + $this->assertNotNull(json_decode($response), 'PHP_INT_MAX should produce valid JSON'); + + // Verify bigint is unquoted + $this->assertStringContainsString('"id":9223372036854775807', $response); + + // Verify string representation stays quoted + $this->assertStringContainsString('"description":"test_description"', $response); + + // Verify bigint field was detected from columns + $this->assertContains('data.0.id', $struct->getBigIntFields()); + } + + /** + * Test PHP_INT_MAX + 1 (9223372036854775808) + * Exceeds PHP_INT_MAX, becomes float in PHP, json_encode uses scientific notation + * This tests that column metadata correctly identifies it as bigint + * + * @return void + */ + public function testPHPIntMaxPlusOne(): void { + // PHP_INT_MAX + 1 becomes float in JSON decode + // json_encode will output: 9.2233720368547758e+18 + $jsonInput = json_encode( + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['description' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => 9223372036854775808, // Beyond PHP_INT_MAX + 'description' => '9223372036854775808', + ], + ], + ] + ); + + $this->assertIsString($jsonInput); + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid (this would fail without column metadata!) + $this->assertNotNull( + json_decode($response), + 'PHP_INT_MAX + 1 should be handled correctly via column metadata' + ); + + // Verify bigint field is detected from columns + $this->assertContains('data.0.id', $struct->getBigIntFields()); + + // Verify string field is NOT detected as bigint + $this->assertNotContains('data.0.description', $struct->getBigIntFields()); + } + + /** + * Test PHP_INT_MIN boundary (-9223372036854775808) + * Negative boundary value, stays as integer in PHP + * Note: -9223372036854775808 becomes scientific notation in some JSON encodes + * + * @return void + */ + public function testPHPIntMinBoundary(): void { + $jsonInput = json_encode( + [ + 'columns' => [ + ['balance' => ['type' => 'long long']], + ['label' => ['type' => 'string']], + ], + 'data' => [ + [ + 'balance' => -9223372036854775807, // Use -1 to avoid scientific notation edge case + 'label' => 'test_label', + ], + ], + ] + ); + + $this->assertIsString($jsonInput); + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid + $this->assertNotNull(json_decode($response), 'PHP_INT_MIN should produce valid JSON'); + + // Verify negative bigint is unquoted + $this->assertStringContainsString('"balance":-9223372036854775807', $response); + + // Verify string stays quoted + $this->assertStringContainsString('"label":"test_label"', $response); + + // Verify bigint field was detected from columns + $this->assertContains('data.0.balance', $struct->getBigIntFields()); + } + + /** + * Test PHP_INT_MIN - 1 (-9223372036854775809) + * Exceeds PHP_INT_MIN, becomes float in PHP, json_encode uses scientific notation + * This tests that column metadata correctly identifies it as bigint + * + * @return void + */ + public function testPHPIntMinMinusOne(): void { + // PHP_INT_MIN - 1 becomes float in JSON decode + // json_encode will output: -9.2233720368547758e+18 + $jsonInput = json_encode( + [ + 'columns' => [ + ['balance' => ['type' => 'long long']], + ['label' => ['type' => 'string']], + ], + 'data' => [ + [ + 'balance' => -9223372036854775809, // Beyond PHP_INT_MIN + 'label' => '-9223372036854775809', + ], + ], + ] + ); + + $this->assertIsString($jsonInput); + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid (this would fail without column metadata!) + $this->assertNotNull( + json_decode($response), + 'PHP_INT_MIN - 1 should be handled correctly via column metadata' + ); + + // Verify bigint field is detected from columns + $this->assertContains('data.0.balance', $struct->getBigIntFields()); + + // Verify string field is NOT detected as bigint + $this->assertNotContains('data.0.label', $struct->getBigIntFields()); + } + + /** + * Test maximum unsigned 64-bit value (2^64 - 1) + * 18446744073709551615 - the largest possible 64-bit unsigned integer + * + * @return void + */ + public function testMax64BitUnsigned(): void { + // Max uint64 = 18446744073709551615 + $maxUint64 = 18446744073709551615; + + $jsonInput = json_encode( + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['code' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => $maxUint64, // Max uint64 + 'code' => '18446744073709551615', + ], + ], + ] + ); + + $this->assertIsString($jsonInput); + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid + $this->assertNotNull(json_decode($response), 'Max uint64 should produce valid JSON'); + + // Verify bigint field is detected from columns + $this->assertContains('data.0.id', $struct->getBigIntFields()); + + // Verify string field stays quoted + $this->assertStringContainsString('"code":"18446744073709551615"', $response); + } + + /** + * Critical test: Numeric string at boundary values should NOT be marked as bigint + * even if they look like large numbers, because they're not in 'long long' columns + * This proves the hybrid approach prevents false positives + * + * @return void + */ + public function testNumericStringNotMisidentifiedAsBigint(): void { + // This tests that even very large numeric strings are kept quoted + // when they're not marked as 'long long' in columns + $jsonInput = json_encode( + [ + 'columns' => [ + ['id' => ['type' => 'long long']], + ['huge_string' => ['type' => 'string']], + ], + 'data' => [ + [ + 'id' => 123, + 'huge_string' => '18446744073709551615', // Looks like bigint but is string! + ], + ], + ] + ); + + $this->assertIsString($jsonInput); + $struct = Struct::fromJson($jsonInput); + $response = $struct->toJson(); + + // Verify JSON is valid + $this->assertNotNull(json_decode($response), 'Should handle large numeric strings'); + + // Verify bigint detection is correct + $this->assertContains('data.0.id', $struct->getBigIntFields()); + $this->assertNotContains('data.0.huge_string', $struct->getBigIntFields()); + + // Verify string stays quoted (this is the critical check) + $this->assertStringContainsString('"huge_string":"18446744073709551615"', $response); + $this->assertStringNotContainsString('"huge_string":18446744073709551615', $response); + } + + /** + * Test isBigIntBoundary() helper using reflection for precise boundary detection + * This tests the mathematical correctness of the new heuristic fallback + * + * @return void + */ + public function testIsBigIntBoundaryDetection(): void { + // Use reflection to access the private method + $method = new ReflectionMethod(Struct::class, 'isBigIntBoundary'); + $method->setAccessible(true); + + // Test cases: [value, expected_result] + $testCases = [ + // Within PHP_INT_MAX (9223372036854775807) + ['1', false], + ['123', false], + ['9223372036854775806', false], // PHP_INT_MAX - 1 + ['9223372036854775807', false], // PHP_INT_MAX (at boundary) + + // Beyond PHP_INT_MAX + ['9223372036854775808', true], // PHP_INT_MAX + 1 + ['18446744073709551615', true], // Max uint64 + + // Large numbers with many digits + ['12345678901234567890', true], // 20 digits + ['123456789012345678901', true], // 21 digits + + // Within PHP_INT_MIN (-9223372036854775808) + ['-1', false], + ['-123', false], + ['-9223372036854775807', false], // PHP_INT_MIN + 1 + ['-9223372036854775808', false], // PHP_INT_MIN (at boundary) + + // Beyond PHP_INT_MIN + ['-9223372036854775809', true], // PHP_INT_MIN - 1 + ['-18446744073709551615', true], // Negative max uint64 + + // Zero-padded numbers (common source of false positives) + ['0000000000', false], // Padded zero + ['00123', false], // Padded small number + ['0000009223372036854775807', false], // Padded PHP_INT_MAX + + // Negative zero-padded + ['-0000000001', false], // Padded negative + ]; + + foreach ($testCases as [$value, $expected]) { + $result = $method->invoke(null, $value); + $this->assertSame( + $expected, + $result, + "isBigIntBoundary('{$value}') should return " . ($expected ? 'true' : 'false') + ); + } + } +}