MySQL: Reworked Varchar/Varbinary column identification (#234)

* MySQL: Reworked Varchar/Varbinary column identification
* Added comments explaining some of the mysql type mappings

Author: Rentacookie

.changeset/famous-spoons-whisper.md

@@ -0,0 +1,5 @@
+---
+'@powersync/service-module-mysql': patch
+---
+
+Fixed mysql varchar column identification for binary encoded varchar columns

modules/module-mysql/dev/docker/mysql/docker-compose-57.yaml

@@ -0,0 +1,17 @@
+services:
+  mysql:
+    image: mysql:5.7
+    environment:
+      MYSQL_ROOT_PASSWORD: root_password
+      MYSQL_DATABASE: mydatabase
+      MYSQL_USER: myuser
+      MYSQL_PASSWORD: mypassword
+    ports:
+      - '3306:3306'
+    volumes:
+      - ./init-scripts/my.cnf:/etc/mysql/my.cnf
+      - ./init-scripts/mysql_57.sql:/docker-entrypoint-initdb.d/init_user.sql
+      - mysql_data_57:/var/lib/mysql
+
+volumes:
+  mysql_data_57:

modules/module-mysql/dev/docker/mysql/init-scripts/mysql_57.sql

@@ -0,0 +1,12 @@
+-- Create a user with necessary privileges
+CREATE USER 'repl_user'@'%' IDENTIFIED BY 'good_password';
+
+-- Grant replication client privilege
+GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD ON *.* TO 'repl_user'@'%';
+GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD ON *.* TO 'myuser'@'%';
+
+-- Grant access to the specific database
+GRANT ALL PRIVILEGES ON mydatabase.* TO 'repl_user'@'%';
+
+-- Apply changes
+FLUSH PRIVILEGES;

modules/module-mysql/src/common/mysql-to-sqlite.ts

@@ -41,26 +41,28 @@ export function toColumnDescriptors(columns: mysql.FieldPacket[] | TableMapEntry
 
 export function toColumnDescriptorFromFieldPacket(column: mysql.FieldPacket): ColumnDescriptor {
   let typeId = column.type!;
-  const BINARY_FLAG = 128;
   const MYSQL_ENUM_FLAG = 256;
   const MYSQL_SET_FLAG = 2048;
+  const MYSQL_BINARY_ENCODING = 'binary';
 
   switch (column.type) {
+    // STRING is overloaded to also include Binary, Enum and Set types
     case mysql.Types.STRING:
-      if (((column.flags as number) & BINARY_FLAG) !== 0) {
+      if (column.encoding === MYSQL_BINARY_ENCODING) {
         typeId = ADDITIONAL_MYSQL_TYPES.BINARY;
       } else if (((column.flags as number) & MYSQL_ENUM_FLAG) !== 0) {
         typeId = mysql.Types.ENUM;
       } else if (((column.flags as number) & MYSQL_SET_FLAG) !== 0) {
         typeId = mysql.Types.SET;
       }
       break;
-
+    // VAR_STRING represents both VARCHAR and VARBINARY types
     case mysql.Types.VAR_STRING:
-      typeId = ((column.flags as number) & BINARY_FLAG) !== 0 ? ADDITIONAL_MYSQL_TYPES.VARBINARY : column.type;
+      typeId = column.encoding === MYSQL_BINARY_ENCODING ? ADDITIONAL_MYSQL_TYPES.VARBINARY : column.type;
       break;
+    // BLOB is also used to represent the TEXT type when the encoding is not binary
     case mysql.Types.BLOB:
-      typeId = ((column.flags as number) & BINARY_FLAG) === 0 ? ADDITIONAL_MYSQL_TYPES.TEXT : column.type;
+      typeId = column.encoding !== MYSQL_BINARY_ENCODING ? ADDITIONAL_MYSQL_TYPES.TEXT : column.type;
       break;
   }
 
@@ -77,13 +79,16 @@ export function toColumnDescriptorFromDefinition(column: ColumnDefinition): Colu
   let typeId = column.type;
 
   switch (column.type) {
+    // STRING is overloaded to also include Binary types, ENUM and SET is already identified upstream in Zongji
     case mysql.Types.STRING:
       typeId = !column.charset ? ADDITIONAL_MYSQL_TYPES.BINARY : column.type;
       break;
+    // VAR_STRING represents both VARCHAR and VARBINARY types
     case mysql.Types.VAR_STRING:
     case mysql.Types.VARCHAR:
       typeId = !column.charset ? ADDITIONAL_MYSQL_TYPES.VARBINARY : column.type;
       break;
+    // BLOB is also used to represent the TEXT type when a charset is specified
     case mysql.Types.BLOB:
       typeId = column.charset ? ADDITIONAL_MYSQL_TYPES.TEXT : column.type;
       break;

modules/module-mysql/test/src/mysql-to-sqlite.test.ts

@@ -40,6 +40,9 @@ describe('MySQL Data Types', () => {
 
     char_col CHAR(10),
     varchar_col VARCHAR(255),
+    varchar_binary_encoding_col VARCHAR(255) CHARACTER SET binary,
+    varchar_with_bin_collation_col VARCHAR(255) COLLATE utf8mb4_bin,
+    
     binary_col BINARY(16),
     varbinary_col VARBINARY(256),
     tinyblob_col TINYBLOB,
@@ -145,6 +148,8 @@ INSERT INTO test_data (
 INSERT INTO test_data (
     char_col,
     varchar_col,
+    varchar_binary_encoding_col,
+    varchar_with_bin_collation_col,
     binary_col,
     varbinary_col,
     tinyblob_col,
@@ -159,7 +164,9 @@ INSERT INTO test_data (
 ) VALUES (
     'CharData',               -- CHAR(10) with padding spaces
     'Variable character data',-- VARCHAR(255)
-    'ShortBin',        -- BINARY(16)
+    'Varchar with binary encoding', -- VARCHAR(255) with binary encoding
+    'Variable character data with bin collation', -- VARCHAR(255) with bin collation
+    'ShortBin',               -- BINARY(16)
     'VariableBinaryData',     -- VARBINARY(256)
     'TinyBlobData',           -- TINYBLOB
     'BlobData',               -- BLOB
@@ -177,6 +184,11 @@ INSERT INTO test_data (
     const expectedResult = {
       char_col: 'CharData',
       varchar_col: 'Variable character data',
+      varchar_binary_encoding_col: new Uint8Array([
+        86, 97, 114, 99, 104, 97, 114, 32, 119, 105, 116, 104, 32, 98, 105, 110, 97, 114, 121, 32, 101, 110, 99, 111,
+        100, 105, 110, 103
+      ]),
+      varchar_with_bin_collation_col: 'Variable character data with bin collation',
       binary_col: new Uint8Array([83, 104, 111, 114, 116, 66, 105, 110, 0, 0, 0, 0, 0, 0, 0, 0]), // Pad with 0
       varbinary_col: new Uint8Array([
         0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61