Skip to content

Commit 4bd8216

Browse files
kerryjiangkerryjiang
committed
improved column metadata loading
1 parent 2a7bb55 commit 4bd8216

32 files changed

+300
-122
lines changed

‎src/SciSharp.MySQL.Replication/ColumnMetadata.cs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
using System.Collections.Generic;
2+
3+
namespace SciSharp.MySQL.Replication
4+
{
5+
/// <summary>
6+
/// Represents metadata information for a database column.
7+
/// </summary>
8+
public class ColumnMetadata
9+
{
10+
/// <summary>
11+
/// Gets or sets the name of the column.
12+
/// </summary>
13+
public string Name { get; set; }
14+
15+
/// <summary>
16+
/// Gets or sets the column type.
17+
/// </summary>
18+
public ColumnType Type { get; set; }
19+
20+
/// <summary>
21+
/// Gets or sets the metadata value associated with the column.
22+
/// </summary>
23+
public short MetadataValue { get; set; }
24+
25+
/// <summary>
26+
/// Gets or sets the column max value length.
27+
/// </summary>
28+
public int MaxLength { get; set; }
29+
30+
/// <summary>
31+
/// Gets or sets the character set ID for the column.
32+
/// </summary>
33+
public int CharsetId { get; set; }
34+
35+
/// <summary>
36+
/// Gets or sets the set of possible values for an ENUM column type.
37+
/// </summary>
38+
public IReadOnlyList<string> EnumValues { get; set; }
39+
40+
/// <summary>
41+
/// Gets or sets the set of possible values for a SET column type.
42+
/// </summary>
43+
public IReadOnlyList<string> SetValues { get; set; }
44+
45+
/// <summary>
46+
/// Gets or sets a value indicating whether the column is visible.
47+
/// </summary>
48+
public bool IsVisible { get; set; }
49+
50+
/// <summary>
51+
/// Gets or sets a value indicating whether the column is unsigned.
52+
/// </summary>
53+
public bool IsUnsigned { get; set; }
54+
55+
/// <summary>
56+
/// Gets or sets the column index in all the numerci columns.
57+
/// </summary>
58+
public int NumericColumnIndex { get; set; }
59+
}
60+
}

‎src/SciSharp.MySQL.Replication/Events/RowsEvent.cs‎

Lines changed: 5 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,6 @@ protected object[] ReadRow(ref SequenceReader<byte> reader, TableMapEvent table,
218218
var cells = new object[columnCount];
219219
var nullColumns = reader.ReadBitArray(columnCount, true);
220220
var columnTypes = table.ColumnTypes;
221-
var columnMetadata = table.ColumnMetadata;
222221

223222
for (int i = 0, numberOfSkippedColumns = 0; i < columnTypes.Length; i++)
224223
{
@@ -234,40 +233,9 @@ protected object[] ReadRow(ref SequenceReader<byte> reader, TableMapEvent table,
234233
continue;
235234

236235
var typeCode = columnTypes[i];
237-
var meta = columnMetadata[i];
238-
var length = 0;
236+
var columnMetadata = table.Metadata.Columns[i];
239237

240-
var columnType = (ColumnType)typeCode;
241-
242-
if (columnType == ColumnType.STRING)
243-
{
244-
if (meta >= 256)
245-
{
246-
int meta0 = meta >> 8, meta1 = meta & 0xFF;
247-
if ((meta0 & 0x30) != 0x30)
248-
{
249-
typeCode = (byte)(meta0 | 0x30);
250-
columnType = (ColumnType)typeCode;
251-
length = meta1 | (((meta0 & 0x30) ^ 0x30) << 4);
252-
}
253-
else
254-
{
255-
// mysql-5.6.24 sql/rpl_utility.h enum_field_types (line 278)
256-
if (meta0 == (int)ColumnType.ENUM || meta0 == (int)ColumnType.SET)
257-
{
258-
typeCode = (byte)meta0;
259-
columnType = (ColumnType)typeCode;
260-
}
261-
length = meta1;
262-
}
263-
}
264-
else
265-
{
266-
length = meta;
267-
}
268-
}
269-
270-
cells[index] = ReadCell(ref reader, columnType, meta, length);
238+
cells[index] = ReadCell(ref reader, (ColumnType)typeCode, columnMetadata);
271239
}
272240

273241
return cells;
@@ -278,17 +246,16 @@ protected object[] ReadRow(ref SequenceReader<byte> reader, TableMapEvent table,
278246
/// </summary>
279247
/// <param name="reader">The sequence reader containing the binary data.</param>
280248
/// <param name="columnType">The type of the column.</param>
281-
/// <param name="meta">The metadata for the column.</param>
282-
/// <param name="length">The length of the column data.</param>
249+
/// <param name="columnMetadata">The metadata for the column.</param>
283250
/// <returns>The parsed cell value.</returns>
284-
private object ReadCell(ref SequenceReader<byte> reader, ColumnType columnType, int meta, int length)
251+
private object ReadCell(ref SequenceReader<byte> reader, ColumnType columnType, ColumnMetadata columnMetadata)
285252
{
286253
var dataType = DataTypes[(int)columnType] as IMySQLDataType;
287254

288255
if (dataType == null)
289256
throw new NotImplementedException();
290257

291-
return dataType.ReadValue(ref reader, meta);
258+
return dataType.ReadValue(ref reader, columnMetadata);
292259
}
293260
}
294261
}

‎src/SciSharp.MySQL.Replication/Events/TableMapEvent.cs‎

Lines changed: 14 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Collections.Generic;
55
using System.Linq;
66
using System.Text;
7+
using SciSharp.MySQL.Replication.Types;
78
using SuperSocket.ProtoBase;
89

910
namespace SciSharp.MySQL.Replication.Events
@@ -99,6 +100,16 @@ protected internal override void DecodeBody(ref SequenceReader<byte> reader, obj
99100

100101
Metadata = ReadTableMetadata(ref reader);
101102

103+
foreach (var columnMetadata in Metadata.Columns)
104+
{
105+
var valueTypeIndex = (int)columnMetadata.Type;
106+
107+
if (valueTypeIndex < DataTypes.Length && DataTypes[valueTypeIndex] is IColumnMetadataLoader columnMetadataLoader)
108+
{
109+
columnMetadataLoader.LoadMetadataValue(columnMetadata);
110+
}
111+
}
112+
102113
if (context is ReplicationState repState)
103114
{
104115
repState.TableMap[TableID] = this;
@@ -160,37 +171,14 @@ private int[] ReadColumnMetadata(ref SequenceReader<byte> reader, byte[] columnT
160171
return columnMetadata;
161172
}
162173

163-
/// <summary>
164-
/// Determines whether the specified column type is a numeric column.
165-
/// </summary>
166-
/// <param name="columnType">The column type to check.</param>
167-
/// <returns>True if the column type is numeric; otherwise, false.</returns>
168-
private bool IsNumberColumn(ColumnType columnType)
169-
{
170-
switch (columnType)
171-
{
172-
case ColumnType.TINY:
173-
case ColumnType.SHORT:
174-
case ColumnType.INT24:
175-
case ColumnType.LONG:
176-
case ColumnType.LONGLONG:
177-
case ColumnType.NEWDECIMAL:
178-
case ColumnType.FLOAT:
179-
case ColumnType.DOUBLE:
180-
return true;
181-
default:
182-
return false;
183-
}
184-
}
185-
186174
/// <summary>
187175
/// Reads the table metadata from the binary representation.
188176
/// </summary>
189177
/// <param name="reader">The sequence reader containing the binary data.</param>
190178
/// <returns>The table metadata.</returns>
191179
private TableMetadata ReadTableMetadata(ref SequenceReader<byte> reader)
192180
{
193-
var numericColumnCount = ColumnTypes.Count(c => IsNumberColumn((ColumnType)c));
181+
var numericColumnCount = ColumnTypes.Count(c => ((ColumnType)c).IsNumberColumn());
194182

195183
var metadata = new TableMetadata();
196184

@@ -213,6 +201,8 @@ private TableMetadata ReadTableMetadata(ref SequenceReader<byte> reader)
213201
}
214202
}
215203

204+
metadata.BuildColumnMetadataList(ColumnTypes.Select(c => (ColumnType)c).ToArray(), ColumnMetadata);
205+
216206
return metadata;
217207
}
218208

‎src/SciSharp.MySQL.Replication/TableMetadata.cs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Collections.Generic;
44
using System.Collections;
55
using System.Text;
6+
using SciSharp.MySQL.Replication.Types;
67

78
namespace SciSharp.MySQL.Replication
89
{
@@ -72,5 +73,47 @@ public class TableMetadata
7273
/// Each bit corresponds to a column, indicating whether it's visible (true) or invisible (false).
7374
/// </summary>
7475
public BitArray ColumnVisibility { get; set; }
76+
77+
/// <summary>
78+
/// Gets the list of column metadata.
79+
/// </summary>
80+
public IReadOnlyList<ColumnMetadata> Columns { get; private set; }
81+
82+
/// <summary>
83+
/// Builds the list of column metadata based on the provided column types.
84+
/// This method is typically called after the table metadata has been fully populated.
85+
/// </summary>
86+
/// <param name="columnTypes">The column types.</param>
87+
/// <param name="columnMetadataValues">The metadata values for each column.</param>
88+
public void BuildColumnMetadataList(IReadOnlyList<ColumnType> columnTypes, IReadOnlyList<int> columnMetadataValues)
89+
{
90+
var columnMetadatas = new List<ColumnMetadata>(ColumnNames.Count);
91+
92+
var numericColumnIndex = -1;
93+
94+
for (int i = 0; i < columnTypes.Count; i++)
95+
{
96+
var columnType = columnTypes[i];
97+
var isNumberColumn = columnType.IsNumberColumn();
98+
numericColumnIndex++;
99+
100+
var columnMetadata = new ColumnMetadata
101+
{
102+
Name = ColumnNames[i],
103+
Type = columnType,
104+
CharsetId = ColumnCharsets != null ? ColumnCharsets[i] : 0,
105+
//If the bit is set (1), the column is UNSIGNED; if not set (0), it's SIGNED (default)
106+
IsUnsigned = isNumberColumn ? Signedness[numericColumnIndex] : false,
107+
EnumValues = EnumStrValues != null ? EnumStrValues[i] : null,
108+
SetValues = SetStrValues != null ? SetStrValues[i] : null,
109+
MetadataValue = (short)columnMetadataValues[i],
110+
NumericColumnIndex = isNumberColumn ? numericColumnIndex : -1
111+
};
112+
113+
columnMetadatas.Add(columnMetadata);
114+
}
115+
116+
Columns = columnMetadatas;
117+
}
75118
}
76119
}

‎src/SciSharp.MySQL.Replication/Types/BitType.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,11 @@ class BitType : IMySQLDataType
1717
/// Reads a BIT value from the binary log.
1818
/// </summary>
1919
/// <param name="reader">The sequence reader containing the bytes to read.</param>
20-
/// <param name="meta">Metadata for the column.</param>
20+
/// <param name="columnMetadata">Metadata for the column.</param>
2121
/// <returns>An object representing the MySQL BIT value.</returns>
22-
public object ReadValue(ref SequenceReader<byte> reader, int meta)
22+
public object ReadValue(ref SequenceReader<byte> reader, ColumnMetadata columnMetadata)
2323
{
24+
int meta = columnMetadata.MetadataValue;
2425
int bitArrayLength = (meta >> 8) * 8 + (meta & 0xFF);
2526
return reader.ReadBitArray(bitArrayLength, false);
2627
}

‎src/SciSharp.MySQL.Replication/Types/BlobType.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@ class BlobType : IMySQLDataType
1717
/// Reads a BLOB value from the binary log.
1818
/// </summary>
1919
/// <param name="reader">The sequence reader containing the bytes to read.</param>
20-
/// <param name="meta">Metadata for the column.</param>
20+
/// <param name="columnMetadata">Metadata for the column.</param>
2121
/// <returns>A byte array representing the MySQL BLOB value.</returns>
22-
public object ReadValue(ref SequenceReader<byte> reader, int meta)
22+
public object ReadValue(ref SequenceReader<byte> reader, ColumnMetadata columnMetadata)
2323
{
24-
int blobLength = reader.ReadInteger(meta);
24+
int blobLength = reader.ReadInteger(columnMetadata.MetadataValue);
2525

2626
try
2727
{

‎src/SciSharp.MySQL.Replication/Types/ColumnTypeExtensions.cs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
using System;
2+
3+
namespace SciSharp.MySQL.Replication.Types
4+
{
5+
/// <summary>
6+
/// Extension methods for the ColumnType enumeration.
7+
/// </summary>
8+
public static class ColumnTypeExtensions
9+
{
10+
/// <summary>
11+
/// Determines if the specified column type is a numeric type.
12+
/// </summary>
13+
/// <param name="columnType">The column type.</param>
14+
internal static bool IsNumberColumn(this ColumnType columnType)
15+
{
16+
switch (columnType)
17+
{
18+
case ColumnType.TINY:
19+
case ColumnType.SHORT:
20+
case ColumnType.INT24:
21+
case ColumnType.LONG:
22+
case ColumnType.LONGLONG:
23+
case ColumnType.NEWDECIMAL:
24+
case ColumnType.FLOAT:
25+
case ColumnType.DOUBLE:
26+
return true;
27+
default:
28+
return false;
29+
}
30+
}
31+
}
32+
}

‎src/SciSharp.MySQL.Replication/Types/DateTimeType.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ class DateTimeType : IMySQLDataType
1717
/// Reads a DATETIME value from the binary log.
1818
/// </summary>
1919
/// <param name="reader">The sequence reader containing the bytes to read.</param>
20-
/// <param name="meta">Metadata for the column.</param>
20+
/// <param name="columnMetadata">Metadata for the column.</param>
2121
/// <returns>A DateTime object representing the MySQL DATETIME value.</returns>
22-
public object ReadValue(ref SequenceReader<byte> reader, int meta)
22+
public object ReadValue(ref SequenceReader<byte> reader, ColumnMetadata columnMetadata)
2323
{
2424
var value = reader.ReadLong(8);
2525

‎src/SciSharp.MySQL.Replication/Types/DateTimeV2Type.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,11 @@ class DateTimeV2Type : TimeBaseType, IMySQLDataType
1717
/// Reads a DATETIME2 value from the binary log.
1818
/// </summary>
1919
/// <param name="reader">The sequence reader containing the bytes to read.</param>
20-
/// <param name="meta">Metadata for the column defining fractional second precision.</param>
20+
/// <param name="columnMetadata">Metadata for the column defining fractional second precision.</param>
2121
/// <returns>A DateTime object representing the MySQL DATETIME2 value.</returns>
22-
public object ReadValue(ref SequenceReader<byte> reader, int meta)
22+
public object ReadValue(ref SequenceReader<byte> reader, ColumnMetadata columnMetadata)
2323
{
24+
int meta = columnMetadata.MetadataValue;
2425
/*
2526
(in big endian)
2627
1 bit sign (1= non-negative, 0= negative)

‎src/SciSharp.MySQL.Replication/Types/DateType.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ class DateType : IMySQLDataType
1717
/// Reads a DATE value from the binary log.
1818
/// </summary>
1919
/// <param name="reader">The sequence reader containing the bytes to read.</param>
20-
/// <param name="meta">Metadata for the column.</param>
20+
/// <param name="columnMetadata">Metadata for the column.</param>
2121
/// <returns>A DateTime object representing the MySQL DATE value.</returns>
22-
public object ReadValue(ref SequenceReader<byte> reader, int meta)
22+
public object ReadValue(ref SequenceReader<byte> reader, ColumnMetadata columnMetadata)
2323
{
2424
// 11111000 00000000 00000000
2525
// 00000100 00000000 00000000

0 commit comments

Comments
 (0)