Skip to content

Commit d7c91ff

Browse files
sodabrewsodabrew
authored
Merge pull request #1040 from felixbuenemann/fix-encoding-list
2 parents 624409d + d0cc2d3 commit d7c91ff

10 files changed

+138
-74
lines changed

‎.travis.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ matrix:
6565
- mysql2gem.example.com
6666
- rvm: 2.4
6767
env: DB=mysql80
68+
dist: xenial
6869
addons:
6970
hosts:
7071
- mysql2gem.example.com

‎.travis_Dockerfile_centos

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ RUN yum -y install \
1717
gcc \
1818
gcc-c++ \
1919
make
20-
RUN gem update --system > /dev/null
21-
RUN gem install bundler
20+
RUN gem install --no-document "rubygems-update:~>2.7" && update_rubygems
21+
RUN gem install --no-document "bundler:~>1.17"
2222

2323
CMD sh .travis_centos.sh

‎.travis_mysql57.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ set -eux
55
apt-get purge -qq '^mysql*' '^libmysql*'
66
rm -fr /etc/mysql
77
rm -fr /var/lib/mysql
8-
apt-key add - < support/5072E1F5.asc
9-
add-apt-repository 'deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-5.7'
8+
apt-key add support/5072E1F5.asc
9+
add-apt-repository 'http://repo.mysql.com/apt/ubuntu mysql-5.7'
1010
apt-get update -qq
1111
apt-get install -qq mysql-server libmysqlclient-dev
1212

‎.travis_mysql80.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ set -eux
55
apt-get purge -qq '^mysql*' '^libmysql*'
66
rm -fr /etc/mysql
77
rm -fr /var/lib/mysql
8-
apt-key add - < support/5072E1F5.asc
9-
add-apt-repository 'deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-8.0'
8+
apt-key add support/5072E1F5.asc
9+
add-apt-repository 'http://repo.mysql.com/apt/ubuntu mysql-8.0'
1010
apt-get update -qq
1111
apt-get install -qq mysql-server libmysqlclient-dev
1212

‎ext/mysql2/mysql_enc_name_to_ruby.h

Lines changed: 60 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ error "gperf generated tables don't work with this execution character set. Plea
3030
#endif
3131

3232
struct mysql2_mysql_enc_name_to_rb_map { const char *name; const char *rb_name; };
33-
/* maximum key range = 66, duplicates = 0 */
33+
/* maximum key range = 71, duplicates = 0 */
3434

3535
#ifdef __GNUC__
3636
__inline
@@ -46,32 +46,32 @@ mysql2_mysql_enc_name_to_rb_hash (str, len)
4646
{
4747
static const unsigned char asso_values[] =
4848
{
49-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
50-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
51-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
52-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
53-
69, 69, 69, 69, 69, 69, 69, 69, 40, 5,
54-
0, 69, 0, 40, 25, 20, 10, 55, 69, 69,
55-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
56-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
57-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
58-
69, 69, 69, 69, 69, 69, 69, 35, 5, 0,
59-
10, 0, 20, 0, 5, 5, 69, 0, 10, 15,
60-
0, 0, 69, 69, 25, 5, 5, 0, 69, 30,
61-
69, 0, 69, 69, 69, 69, 69, 69, 69, 69,
62-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
63-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
64-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
65-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
66-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
67-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
68-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
69-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
70-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
71-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
72-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
73-
69, 69, 69, 69, 69, 69, 69, 69, 69, 69,
74-
69, 69, 69, 69, 69, 69
49+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
50+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
51+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
52+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
53+
74, 74, 74, 74, 74, 74, 74, 74, 15, 5,
54+
0, 74, 5, 25, 40, 10, 20, 50, 74, 74,
55+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
56+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
57+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
58+
74, 74, 74, 74, 74, 74, 74, 40, 5, 0,
59+
15, 10, 0, 0, 0, 5, 74, 0, 25, 5,
60+
0, 5, 74, 74, 20, 5, 5, 0, 74, 45,
61+
74, 0, 74, 74, 74, 74, 74, 74, 74, 74,
62+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
63+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
64+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
65+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
66+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
67+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
68+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
69+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
70+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
71+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
72+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
73+
74, 74, 74, 74, 74, 74, 74, 74, 74, 74,
74+
74, 74, 74, 74, 74, 74
7575
};
7676
return len + asso_values[(unsigned char)str[2]] + asso_values[(unsigned char)str[0]] + asso_values[(unsigned char)str[len - 1]];
7777
}
@@ -89,66 +89,70 @@ mysql2_mysql_enc_name_to_rb (str, len)
8989
{
9090
enum
9191
{
92-
TOTAL_KEYWORDS = 39,
92+
TOTAL_KEYWORDS = 41,
9393
MIN_WORD_LENGTH = 3,
9494
MAX_WORD_LENGTH = 8,
9595
MIN_HASH_VALUE = 3,
96-
MAX_HASH_VALUE = 68
96+
MAX_HASH_VALUE = 73
9797
};
9898

9999
static const struct mysql2_mysql_enc_name_to_rb_map wordlist[] =
100100
{
101101
{""}, {""}, {""},
102102
{"gbk", "GBK"},
103103
{""},
104-
{"greek", "ISO-8859-7"},
104+
{"utf32", "UTF-32"},
105105
{"gb2312", "GB2312"},
106106
{"keybcs2", NULL},
107107
{""},
108108
{"ucs2", "UTF-16BE"},
109109
{"koi8u", "KOI8-R"},
110110
{"binary", "ASCII-8BIT"},
111-
{"eucjpms", "eucJP-ms"},
112-
{""},
111+
{"utf8mb4", "UTF-8"},
112+
{"macroman", "macRoman"},
113113
{"ujis", "eucJP-ms"},
114-
{"cp852", "CP852"},
114+
{"greek", "ISO-8859-7"},
115115
{"cp1251", "Windows-1251"},
116-
{"geostd8", NULL},
116+
{"utf16le", "UTF-16LE"},
117117
{""},
118118
{"sjis", "Shift_JIS"},
119119
{"macce", "macCentEuro"},
120-
{"latin2", "ISO-8859-2"},
120+
{"cp1257", "Windows-1257"},
121+
{"eucjpms", "eucJP-ms"},
122+
{""},
123+
{"utf8", "UTF-8"},
124+
{"cp852", "CP852"},
125+
{"cp1250", "Windows-1250"},
126+
{"gb18030", "GB18030"},
121127
{""},
122-
{"macroman", "macRoman"},
123-
{"dec8", NULL},
124-
{"utf32", "UTF-32"},
125-
{"latin1", "ISO-8859-1"},
126-
{"utf8mb4", "UTF-8"},
127-
{"hp8", NULL},
128128
{"swe7", NULL},
129+
{"koi8r", "KOI8-R"},
130+
{"tis620", "TIS-620"},
131+
{"geostd8", NULL},
132+
{""},
133+
{"big5", "Big5"},
129134
{"euckr", "EUC-KR"},
130-
{"cp1257", "Windows-1257"},
135+
{"latin2", "ISO-8859-2"},
131136
{""}, {""},
132-
{"utf8", "UTF-8"},
133-
{"koi8r", "KOI8-R"},
134-
{"cp1256", "Windows-1256"},
135-
{""}, {""}, {""},
136-
{"cp866", "IBM866"},
137+
{"dec8", NULL},
138+
{"cp850", "CP850"},
139+
{"latin1", "ISO-8859-1"},
140+
{""},
141+
{"hp8", NULL},
142+
{""},
143+
{"utf16", "UTF-16"},
137144
{"latin7", "ISO-8859-13"},
138145
{""}, {""}, {""},
139146
{"ascii", "US-ASCII"},
140-
{"hebrew", "ISO-8859-8"},
141-
{""}, {""},
142-
{"big5", "Big5"},
143-
{"utf16", "UTF-16"},
144-
{"cp1250", "Windows-1250"},
145-
{""}, {""}, {""},
146-
{"cp850", "CP850"},
147-
{"tis620", "TIS-620"},
147+
{"cp1256", "Windows-1256"},
148148
{""}, {""}, {""},
149149
{"cp932", "Windows-31J"},
150+
{"hebrew", "ISO-8859-8"},
151+
{""}, {""}, {""}, {""},
150152
{"latin5", "ISO-8859-9"},
151-
{""}, {""}, {""}, {""}, {""}, {""},
153+
{""}, {""}, {""},
154+
{"cp866", "IBM866"},
155+
{""}, {""}, {""}, {""}, {""}, {""}, {""},
152156
{"armscii8", NULL}
153157
};
154158

‎ext/mysql2/mysql_enc_to_ruby.h

Lines changed: 56 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,13 @@ static const char *mysql2_mysql_enc_to_rb[] = {
5454
"macRoman",
5555
"UTF-16",
5656
"UTF-16",
57-
"",
57+
"UTF-16LE",
5858
"Windows-1256",
5959
"Windows-1257",
6060
"Windows-1257",
6161
"UTF-32",
6262
"UTF-32",
63-
"",
63+
"UTF-16LE",
6464
"ASCII-8BIT",
6565
NULL,
6666
"US-ASCII",
@@ -74,7 +74,7 @@ static const char *mysql2_mysql_enc_to_rb[] = {
7474
NULL,
7575
"KOI8-R",
7676
"KOI8-R",
77-
NULL,
77+
"UTF-8",
7878
"ISO-8859-2",
7979
"ISO-8859-9",
8080
"ISO-8859-13",
@@ -246,14 +246,65 @@ static const char *mysql2_mysql_enc_to_rb[] = {
246246
"UTF-8",
247247
"UTF-8",
248248
"UTF-8",
249+
"GB18030",
250+
"GB18030",
251+
"GB18030",
252+
NULL,
253+
NULL,
249254
NULL,
250255
NULL,
256+
"UTF-8",
257+
"UTF-8",
258+
"UTF-8",
259+
"UTF-8",
260+
"UTF-8",
261+
"UTF-8",
262+
"UTF-8",
263+
"UTF-8",
264+
"UTF-8",
265+
"UTF-8",
266+
"UTF-8",
267+
"UTF-8",
268+
"UTF-8",
269+
"UTF-8",
270+
"UTF-8",
271+
"UTF-8",
272+
"UTF-8",
273+
NULL,
274+
"UTF-8",
275+
"UTF-8",
276+
"UTF-8",
251277
NULL,
278+
"UTF-8",
279+
"UTF-8",
280+
"UTF-8",
281+
"UTF-8",
282+
"UTF-8",
283+
"UTF-8",
284+
"UTF-8",
285+
"UTF-8",
286+
"UTF-8",
287+
"UTF-8",
288+
"UTF-8",
289+
"UTF-8",
290+
"UTF-8",
291+
"UTF-8",
292+
"UTF-8",
293+
"UTF-8",
294+
"UTF-8",
295+
"UTF-8",
252296
NULL,
297+
"UTF-8",
298+
"UTF-8",
299+
"UTF-8",
253300
NULL,
301+
"UTF-8",
254302
NULL,
255303
NULL,
304+
"UTF-8",
305+
"UTF-8",
306+
"UTF-8",
307+
"UTF-8",
308+
"UTF-8",
256309
"UTF-8"
257310
};
258-
259-
#define CHARSETNR_SIZE (sizeof(mysql2_mysql_enc_to_rb)/sizeof(mysql2_mysql_enc_to_rb[0]))

‎ext/mysql2/result.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include <mysql2_ext.h>
22

33
#include "mysql_enc_to_ruby.h"
4+
#define MYSQL2_CHARSETNR_SIZE (sizeof(mysql2_mysql_enc_to_rb)/sizeof(mysql2_mysql_enc_to_rb[0]))
45

56
static rb_encoding *binaryEncoding;
67

@@ -179,7 +180,7 @@ static VALUE mysql2_set_field_string_encoding(VALUE val, MYSQL_FIELD field, rb_e
179180
const char *enc_name;
180181
int enc_index;
181182

182-
enc_name = (field.charsetnr-1 < CHARSETNR_SIZE) ? mysql2_mysql_enc_to_rb[field.charsetnr-1] : NULL;
183+
enc_name = (field.charsetnr-1 < MYSQL2_CHARSETNR_SIZE) ? mysql2_mysql_enc_to_rb[field.charsetnr-1] : NULL;
183184

184185
if (enc_name != NULL) {
185186
/* use the field encoding we were able to match */

‎support/5072E1F5.asc

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-----BEGIN PGP PUBLIC KEY BLOCK-----
2-
Version: GnuPG v1.4.5 (GNU/Linux)
2+
Version: GnuPG v1
33

44
mQGiBD4+owwRBAC14GIfUfCyEDSIePvEW3SAFUdJBtoQHH/nJKZyQT7h9bPlUWC3
55
RODjQReyCITRrdwyrKUGku2FmeVGwn2u2WmDMNABLnpprWPkBdCk96+OmSLN9brZ
@@ -11,9 +11,9 @@ kYpXBACmWpP8NJTkamEnPCia2ZoOHODANwpUkP43I7jsDmgtobZX9qnrAXw+uNDI
1111
QJEXM6FSbi0LLtZciNlYsafwAPEOMDKpMqAK6IyisNtPvaLd8lH0bPAnWqcyefep
1212
rv0sxxqUEMcM3o7wwgfN83POkDasDbs3pjwPhxvhz6//62zQJ7Q2TXlTUUwgUmVs
1313
ZWFzZSBFbmdpbmVlcmluZyA8bXlzcWwtYnVpbGRAb3NzLm9yYWNsZS5jb20+iGwE
14-
ExECACwCGyMCHgECF4ACGQEGCwkIBwMCBhUKCQgCAwUWAgMBAAUCWKcFIAUJHirJ
15-
FAAKCRCMcY07UHLh9VcFAJ46pUyVd8BZ2r5CppMC1tmyQ3ceRgCfVPwuVsiS0VER
16-
5WUqtAQDt+DoetCIaQQTEQIAKQIbIwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAhkB
14+
ExECACwCGyMCHgECF4ACGQEGCwkIBwMCBhUKCQgCAwUWAgMBAAUCXEBY+wUJI87e
15+
5AAKCRCMcY07UHLh9RZPAJ9uvm0zlzfCN+DHxHVaoFLFjdVYTQCfborsC9tmEZYa
16+
whhogjeBkZkorbyIaQQTEQIAKQIbIwYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAhkB
1717
BQJTAdRmBQkaZsvLAAoJEIxxjTtQcuH1X4MAoKNLWAbCBUj96637kv6Xa/fJuX5m
1818
AJwPtmgDfjUe2iuhXdTrFEPT19SB6ohmBBMRAgAmAhsjBgsJCAcDAgQVAggDBBYC
1919
AwECHgECF4AFAk53PioFCRP7AhUACgkQjHGNO1By4fUmzACeJdfqgc9gWTUhgmcM
@@ -428,5 +428,5 @@ GoaU9u41oyZTIiXPiFidJoIZCh7fdurP8pn3X+R5HUNXMr7M+ba8lSNxce/F3kmH
428428
0L7rsKqdh9d/aVxhJINJ+inVDnrXWVoXu9GBjT8Nco1iU9SIVAQYEQIADAUCTnc9
429429
7QUJE/sBuAASB2VHUEcAAQEJEIxxjTtQcuH1FJsAmwWK9vmwRJ/y9gTnJ8PWf0BV
430430
roUTAKClYAhZuX2nUNwH4vlEJQHDqYa5yQ==
431-
=HfUN
431+
=ghXk
432432
-----END PGP PUBLIC KEY BLOCK-----

‎support/mysql_enc_to_ruby.rb

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@
4343
"geostd8" => "NULL",
4444
"cp932" => "Windows-31J",
4545
"eucjpms" => "eucJP-ms",
46+
"utf16le" => "UTF-16LE",
47+
"gb18030" => "GB18030",
4648
}
4749

4850
client = Mysql2::Client.new(username: user, password: pass, host: host, port: port.to_i)
@@ -52,7 +54,10 @@
5254

5355
collations.each do |collation|
5456
mysql_col_idx = collation[2].to_i
55-
rb_enc = mysql_to_rb[collation[1]]
57+
rb_enc = mysql_to_rb.fetch(collation[1]) do |mysql_enc|
58+
$stderr.puts "WARNING: Missing mapping for collation \"#{collation[0]}\" with encoding \"#{mysql_enc}\" and id #{mysql_col_idx}, assuming NULL"
59+
"NULL"
60+
end
5661
encodings[mysql_col_idx - 1] = [mysql_col_idx, rb_enc]
5762
end
5863

‎support/ruby_enc_to_mysql.rb

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@
3838
"geostd8" => nil,
3939
"cp932" => "Windows-31J",
4040
"eucjpms" => "eucJP-ms",
41+
"utf16le" => "UTF-16LE",
42+
"gb18030" => "GB18030",
4143
}
4244

4345
puts <<-HEADER

0 commit comments

Comments
 (0)