Skip to content

Commit 860a51b

Browse files
author
Thomas Preud'homme
committed
[LNT] Python 3 support: fix storage of json data as BLOB
Summary: Machine and Run tables store JSON as binary data in their Parameters column. However, JSON is inserted as text data in several places, causing problem when SQLAlchemy reads it back since it expects binary data but receives text, which Python3 throws an exception for. 2 constructs are responsible for inserting JSON as text: 1) json.dumps() method is used in several places to convert a dictionary to JSON which returns it as text. This commit adds an encoding step to UTF-8 since it is the recommended encoding for JSON. 2) One of the insertion with a JSON Parameters column in lnt_db_create.sql is missing a cast to BLOB. This commit adds the missing cast. Reviewers: cmatthews, hubert.reinterpretcast, kristof.beyls, leandron, PrzemekWirkus Reviewed By: PrzemekWirkus Subscribers: llvm-commits Differential Revision: https://reviews.llvm.org/D68796
1 parent a5e7fc6 commit 860a51b

File tree

7 files changed

+24
-24
lines changed

7 files changed

+24
-24
lines changed

‎lnt/server/db/migrations/upgrade_11_to_12.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ class Machine(object):
2121
name = info.pop('name', None)
2222
if name is not None:
2323
info['hostname'] = name
24-
machine.Parameters = json.dumps(sorted(info.items()))
24+
machine.Parameters = json.dumps(sorted(info.items())).encode("utf-8")
2525

2626
session.commit()
2727
session.close()

‎lnt/server/db/migrations/upgrade_1_to_2.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ def __init__(self, **kwargs):
8484
# Update the run info.
8585
run_info['inferred_run_order'] = run_order
8686
run_info['__report_version__'] = '1'
87-
run.Parameters = json.dumps(sorted(run_info.items()))
87+
run.Parameters = json.dumps(sorted(run_info.items())).encode('utf-8')
8888

8989
if run_order != orig_order:
9090
# Lookup the new run order.

‎lnt/server/db/testsuite.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ class TestSuiteJSONSchema(Base):
106106

107107
def __init__(self, testsuite_name, data):
108108
self.testsuite_name = testsuite_name
109-
self.jsonschema = json.dumps(data, encoding='utf-8', sort_keys=True)
109+
self.jsonschema = json.dumps(data, sort_keys=True).encode('utf-8')
110110

111111

112112
class TestSuite(Base):

‎lnt/server/db/testsuitedb.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ def parameters(self):
143143

144144
@parameters.setter
145145
def parameters(self, data):
146-
self.parameters_data = json.dumps(sorted(data.items()))
146+
self.parameters_data = json.dumps(sorted(data.items())).encode("utf-8")
147147

148148
def get_baseline_run(self, session):
149149
ts = Machine.testsuite
@@ -402,7 +402,7 @@ def parameters(self):
402402

403403
@parameters.setter
404404
def parameters(self, data):
405-
self.parameters_data = json.dumps(sorted(data.items()))
405+
self.parameters_data = json.dumps(sorted(data.items())).encode("utf-8")
406406

407407
def __json__(self, flatten_order=True):
408408
result = {

‎tests/SharedInputs/SmallInstance/data/lnt_db_create.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ CREATE TABLE "NT_Machine" (
153153
);
154154
INSERT INTO "NT_Machine" ("Name", "Parameters", "hardware", "os")
155155
VALUES('localhost__clang_DEV__x86_64',
156-
'[["name", "localhost"], ["uname", "Darwin localhost 11.3.0 Darwin Kernel Version 11.3.0: Thu Jan 12 18:47:41 PST 2012; root:xnu-1699.24.23~1/RELEASE_X86_64 x86_64"]]',
156+
CAST('[["name", "localhost"], ["uname", "Darwin localhost 11.3.0 Darwin Kernel Version 11.3.0: Thu Jan 12 18:47:41 PST 2012; root:xnu-1699.24.23~1/RELEASE_X86_64 x86_64"]]' AS BLOB),
157157
'x86_64','Darwin 11.3.0'); -- ID 1
158158
CREATE TABLE "NT_Test" (
159159
"ID" INTEGER PRIMARY KEY NOT NULL,

‎tests/server/db/Inputs/V4Pages_extra_records.sql

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,27 @@ INSERT INTO "compile_Test" ("Name")
66

77
-- make sure there are 3 machines - to test ?filter-machine-regex= on daily_report page
88
INSERT INTO "NT_Machine" ("Name", "Parameters", "hardware", "os")
9-
VALUES('machine2','[]','AArch64','linux'); -- ID 2
9+
VALUES('machine2',CAST('[]' AS BLOB),'AArch64','linux'); -- ID 2
1010
INSERT INTO "NT_Order" ("NextOrder", "PreviousOrder", "llvm_project_revision")
1111
VALUES(2,NULL,'152290'); -- ID 3
1212
UPDATE "NT_Order" SET "NextOrder" = 3 WHERE "ID" = 2;
1313
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
1414
"EndTime", "SimpleRunID", "Parameters")
1515
VALUES(2,3,'run3.json','2012-04-11 16:28:23.000000',
16-
'2012-04-11 16:28:58.000000',NULL,'[]'); -- ID 3
16+
'2012-04-11 16:28:58.000000',NULL,CAST('[]' AS BLOB)); -- ID 3
1717
INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
1818
"execution_status", "compile_time", "execution_time",
1919
"score", "mem_bytes")
2020
VALUES(3,1,NULL,NULL,0.001,0.0001,NULL,NULL); -- ID 3
2121
INSERT INTO "NT_Machine" ("Name", "Parameters", "hardware", "os")
22-
VALUES('machine3','[]','AArch64','linux'); -- ID 3
22+
VALUES('machine3',CAST('[]' AS BLOB),'AArch64','linux'); -- ID 3
2323
INSERT INTO "NT_Order" ("NextOrder", "PreviousOrder", "llvm_project_revision")
2424
VALUES(3,NULL,'152291'); -- ID 4
2525
UPDATE "NT_Order" SET "PreviousOrder" = 4 WHERE "ID" = 3;
2626
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
2727
"EndTime", "SimpleRunID", "Parameters")
2828
VALUES(3,4,'run4.json','2012-04-11 16:28:24.000000',
29-
'2012-04-11 16:28:59.000000',NULL,'[]'); -- ID 4
29+
'2012-04-11 16:28:59.000000',NULL,CAST('[]' AS BLOB)); -- ID 4
3030
INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
3131
"execution_status", "compile_time", "execution_time",
3232
"score", "mem_bytes")
@@ -40,7 +40,7 @@ INSERT INTO "NT_Order" ("NextOrder", "PreviousOrder", "llvm_project_revision")
4040
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
4141
"EndTime", "SimpleRunID", "Parameters")
4242
VALUES(2,5,'run5.json','2012-05-01 16:28:23.000000',
43-
'2012-05-01 16:28:58.000000',NULL,'[]'); -- ID 5
43+
'2012-05-01 16:28:58.000000',NULL,CAST('[]' AS BLOB)); -- ID 5
4444
INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
4545
"execution_status", "compile_time", "execution_time",
4646
"score", "mem_bytes")
@@ -55,7 +55,7 @@ UPDATE "NT_Order" SET "PreviousOrder" = 6 WHERE "ID" = 5;
5555
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
5656
"EndTime", "SimpleRunID", "Parameters")
5757
VALUES(2,6,'run6.json','2012-05-03 16:28:24.000000',
58-
'2012-05-03 16:28:59.000000',NULL,'[]'); -- ID 6
58+
'2012-05-03 16:28:59.000000',NULL,CAST('[]' AS BLOB)); -- ID 6
5959
INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
6060
"execution_status", "compile_time", "execution_time",
6161
"score", "mem_bytes")
@@ -76,15 +76,15 @@ INSERT INTO "NT_Order" ("NextOrder", "PreviousOrder", "llvm_project_revision")
7676
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
7777
"EndTime", "SimpleRunID", "Parameters")
7878
VALUES(2,6,'run7.json','2012-05-10 16:28:23.000000',
79-
'2012-05-10 16:28:58.000000',NULL,'[]'); -- ID 7
79+
'2012-05-10 16:28:58.000000',NULL,CAST('[]' AS BLOB)); -- ID 7
8080
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
8181
"EndTime", "SimpleRunID", "Parameters")
8282
VALUES(2,7,'run8.json','2012-05-11 16:28:23.000000',
83-
'2012-05-11 16:28:58.000000',NULL,'[]'); -- ID 8
83+
'2012-05-11 16:28:58.000000',NULL,CAST('[]' AS BLOB)); -- ID 8
8484
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
8585
"EndTime", "SimpleRunID", "Parameters")
8686
VALUES(2,8,'run9.json','2012-05-12 16:28:23.000000',
87-
'2012-05-12 16:28:58.000000',NULL,'[]'); -- ID 9
87+
'2012-05-12 16:28:58.000000',NULL,CAST('[]' AS BLOB)); -- ID 9
8888
INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
8989
"execution_status", "compile_time", "execution_time",
9090
"score", "mem_bytes")

‎tests/server/ui/Inputs/V4Pages_extra_records.sql

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,27 @@ INSERT INTO "compile_Test" ("Name")
66

77
-- make sure there are 3 machines - to test ?filter-machine-regex= on daily_report page
88
INSERT INTO "NT_Machine" ("Name", "Parameters", "hardware", "os")
9-
VALUES('machine2','[]','AArch64','linux'); -- ID 2
9+
VALUES('machine2',CAST('[]' AS BLOB),'AArch64','linux'); -- ID 2
1010
INSERT INTO "NT_Order" ("NextOrder", "PreviousOrder", "llvm_project_revision")
1111
VALUES(2,NULL,'152290'); -- ID 3
1212
UPDATE "NT_Order" SET "NextOrder" = 3 WHERE "ID" = 2;
1313
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
1414
"EndTime", "SimpleRunID", "Parameters")
1515
VALUES(2,3,'run3.json','2012-04-11 16:28:23.000000',
16-
'2012-04-11 16:28:58.000000',NULL,'[]'); -- ID 3
16+
'2012-04-11 16:28:58.000000',NULL,CAST('[]' AS BLOB)); -- ID 3
1717
INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
1818
"execution_status", "compile_time", "execution_time",
1919
"score", "mem_bytes")
2020
VALUES(3,1,NULL,NULL,0.001,0.0001,NULL,NULL); -- ID 3
2121
INSERT INTO "NT_Machine" ("Name", "Parameters", "hardware", "os")
22-
VALUES('machine3','[]','AArch64','linux'); -- ID 3
22+
VALUES('machine3',CAST('[]' AS BLOB),'AArch64','linux'); -- ID 3
2323
INSERT INTO "NT_Order" ("NextOrder", "PreviousOrder", "llvm_project_revision")
2424
VALUES(3,NULL,'152291'); -- ID 4
2525
UPDATE "NT_Order" SET "PreviousOrder" = 4 WHERE "ID" = 3;
2626
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
2727
"EndTime", "SimpleRunID", "Parameters")
2828
VALUES(3,4,'run4.json','2012-04-11 16:28:24.000000',
29-
'2012-04-11 16:28:59.000000',NULL,'[]'); -- ID 4
29+
'2012-04-11 16:28:59.000000',NULL,CAST('[]' AS BLOB)); -- ID 4
3030
INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
3131
"execution_status", "compile_time", "execution_time",
3232
"score", "mem_bytes")
@@ -41,7 +41,7 @@ INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
4141
"EndTime", "SimpleRunID", "Parameters")
4242
VALUES(2,5,'run5.json','2012-05-01 16:28:23.000000',
4343
'2012-05-01 16:28:58.000000',NULL,
44-
'[["producer", "http://buildbot.server.url/builders/some-builder/builds/987"]]'); -- ID 5
44+
CAST('[["producer", "http://buildbot.server.url/builders/some-builder/builds/987"]]' AS BLOB)); -- ID 5
4545
INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
4646
"execution_status", "compile_time", "execution_time",
4747
"score", "mem_bytes")
@@ -56,7 +56,7 @@ UPDATE "NT_Order" SET "PreviousOrder" = 6 WHERE "ID" = 5;
5656
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
5757
"EndTime", "SimpleRunID", "Parameters")
5858
VALUES(2,6,'run6.json','2012-05-03 16:28:24.000000',
59-
'2012-05-03 16:28:59.000000',NULL,'[]'); -- ID 6
59+
'2012-05-03 16:28:59.000000',NULL,CAST('[]' AS BLOB)); -- ID 6
6060
INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
6161
"execution_status", "compile_time", "execution_time",
6262
"score", "mem_bytes")
@@ -78,17 +78,17 @@ INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
7878
"EndTime", "SimpleRunID", "Parameters")
7979
VALUES(2,6,'run7.json','2012-05-10 16:28:23.000000',
8080
'2012-05-10 16:28:58.000000',NULL,
81-
'[["producer", "http://my.build.server/buildResult"]]'); -- ID 7
81+
CAST('[["producer", "http://my.build.server/buildResult"]]' AS BLOB)); -- ID 7
8282
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
8383
"EndTime", "SimpleRunID", "Parameters")
8484
VALUES(2,7,'run8.json','2012-05-11 16:28:23.000000',
8585
'2012-05-11 16:28:58.000000',NULL,
86-
'[["producer", "https://my.build.server/buildResult"]]'); -- ID 8
86+
CAST('[["producer", "https://my.build.server/buildResult"]]' AS BLOB)); -- ID 8
8787
INSERT INTO "NT_Run" ("MachineID", "OrderID", "ImportedFrom", "StartTime",
8888
"EndTime", "SimpleRunID", "Parameters")
8989
VALUES(2,8,'run9.json','2012-05-12 16:28:23.000000',
9090
'2012-05-12 16:28:58.000000',NULL,
91-
'[["producer", "https://buildbot.server.url/builders/some-builder/builds/999"]]'); -- ID 9
91+
CAST('[["producer", "https://buildbot.server.url/builders/some-builder/builds/999"]]' AS BLOB)); -- ID 9
9292
INSERT INTO "NT_Sample" ("RunID", "TestID", "compile_status",
9393
"execution_status", "compile_time", "execution_time",
9494
"score", "mem_bytes")

0 commit comments

Comments
 (0)