[LNT] Python 3 support: Parse HTML as text

Summary:
Several tests and the testsuite run import code parse text read from the
data attribute of a Flask response object (werkzeug.wrappers.Response).
However, in Python3 reading from that attribute yields binary data.
The attribute is in fact a property whose getter get_data() as a as_text
parameter that allows to request the result to be a string instead.

This commit adapts all reads from that property to request strings
instead of binary data (the default), except for a few cases in
tests/server/ui/V4Pages.py where the data is fed to ElementTree's
fromstring method in get_xml_tree() which tries to reencode it in ascii
in Python 2 mode if it gets a unicode string.

Reviewers: cmatthews, hubert.reinterpretcast, kristof.beyls, leandron, PrzemekWirkus

Reviewed By: PrzemekWirkus

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D68829

Author: Thomas Preud'homme

lnt/server/ui/api.py

@@ -306,7 +306,7 @@ def post():
         """Add a new run into the lnt database"""
         session = request.session
         db = request.get_db()
-        data = request.data
+        data = request.get_data(as_text=True)
         select_machine = request.values.get('select_machine', 'match')
         merge = request.values.get('merge', None)
         result = lnt.util.ImportData.import_from_string(

tests/server/db/Migrations.py

@@ -28,9 +28,10 @@ def sanity_check_instance(instance_path):
 
     # Visit all the test suites.
     test_suite_link_rex = re.compile("""  <a href="(.*)">(.*)</a><br>""")
-    test_suite_list_start = index.data.index("<h3>Test Suites</h3>")
-    test_suite_list_end = index.data.index("</div>", test_suite_list_start)
-    for ln in index.data[test_suite_list_start:test_suite_list_end].split("\n"):
+    data = index.get_data(as_text=True)
+    test_suite_list_start = data.index("<h3>Test Suites</h3>")
+    test_suite_list_end = data.index("</div>", test_suite_list_start)
+    for ln in data[test_suite_list_start:test_suite_list_end].split("\n"):
         # Ignore non-matching lines.
         print(ln, file=sys.stderr)
         m = test_suite_link_rex.match(ln)

tests/server/ui/V4Pages.py

@@ -78,7 +78,7 @@ def check_code(client, url, expected_code=HTTP_OK, data_to_send=None):
 
 def check_html(client, url, expected_code=HTTP_OK, data_to_send=None):
     resp = check_code(client, url, expected_code, data_to_send)
-    validate_html(resp.data)
+    validate_html(resp.get_data(as_text=True))
     return resp
 
 
@@ -145,7 +145,7 @@ def find_table_with_heading(tree, table_heading):
 
 def check_nr_machines_reported(client, url, expected_nr_machines):
     resp = check_code(client, url)
-    html = resp.data
+    html = resp.get_data(as_text=sys.version_info[0] >= 3)
     tree = get_xml_tree(html)
     # look for the table containing the machines on the page.
     # do this by looking for the title containing "Reported Machine Order"
@@ -166,7 +166,7 @@ def convert_html_to_text(element):
 
 def get_table_by_header(client, url, table_header):
     resp = check_code(client, url)
-    html = resp.data
+    html = resp.get_data(as_text=sys.version_info[0] >= 3)
     tree = get_xml_tree(html)
     table = find_table_with_heading(tree, table_header)
     assert table is not None, \
@@ -227,7 +227,7 @@ def check_body_nr_tests_table(client, url, expected_content):
 def check_producer_label(client, url, label):
     table_header = "Produced by"
     resp = check_code(client, url)
-    tree = get_xml_tree(resp.data)
+    tree = get_xml_tree(resp.get_data(as_text=sys.version_info[0] >= 3))
     table = find_table_by_thead_content(tree, table_header)
     check_row_is_in_table(table, label)
 
@@ -333,7 +333,7 @@ def main():
                     follow_redirects=True)
     assert r.status_code == HTTP_OK
     # Should see baseline displayed in page body.
-    assert "Baseline - foo_baseline" in r.data
+    assert "Baseline - foo_baseline" in r.get_data(as_text=True)
 
     # Now demote it.
     data2 = dict(name="foo_baseline",
@@ -344,7 +344,7 @@ def main():
     r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True)
     assert r.status_code == HTTP_OK
     # Baseline should no longer be shown in page baseline.
-    assert "Baseline - foo_baseline" not in r.data
+    assert "Baseline - foo_baseline" not in r.get_data(as_text=True)
 
     # Leave a baseline in place for the rest of the tests.
     client.post('/v4/nts/order/3', data=form_data)
@@ -638,10 +638,11 @@ def main():
     check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
     app.testing = False
     error_page = check_html(client, '/explode', expected_code=500)
-    assert re.search("division (or modulo )?by zero", error_page.data)
+    assert re.search("division (or modulo )?by zero",
+                     error_page.get_data(as_text=True))
 
     error_page = check_html(client, '/gone', expected_code=404)
-    assert "test" in error_page.data
+    assert "test" in error_page.get_data(as_text=True)
 
     check_html(client, '/sleep?timeout=0', expected_code=200)
 
@@ -650,9 +651,9 @@ def main():
     check_html(client, '/rules')
     check_html(client, '/log')
     resp = check_code(client, '/__health')
-    assert resp.data == "Ok"
+    assert resp.get_data(as_text=True) == "Ok"
     resp = check_code(client, '/ping')
-    assert resp.data == "pong"
+    assert resp.get_data(as_text=True) == "pong"
 
     # Check we can convert a sample into a graph page.
     graph_to_sample = check_code(client, '/db_default/v4/nts/graph_for_sample/10/compile_time?foo=bar',

tests/server/ui/test_api_modify.py

@@ -148,7 +148,7 @@ def test_02_delete_machine(self):
         resp = client.delete('api/db_default/v4/nts/machines/2',
                              headers={'AuthToken': 'test_token'})
         self.assertEqual(resp.status_code, 200)
-        self.assertEqual(resp.get_data(),
+        self.assertEqual(resp.get_data(as_text=True),
                          '''Deleting runs 3 5 6 7 8 9 (6/6)
 Deleted machine machine2:2
 ''')

tests/server/ui/test_matrix_page.py

@@ -36,25 +36,26 @@ def test_config_errors(self):
         client = self.client
         reply = check_code(client, '/v4/nts/matrix',
                            expected_code=HTTP_NOT_FOUND)
-        self.assertIn("Request requires some data arguments.", reply.data)
+        self.assertIn("Request requires some data arguments.",
+                      reply.get_data(as_text=True))
 
         reply = check_code(client, '/v4/nts/matrix?plot.0=1.1.1',
                            expected_code=HTTP_NOT_FOUND)
-        self.assertIn("No data found.", reply.data)
+        self.assertIn("No data found.", reply.get_data(as_text=True))
 
         reply = check_code(client, '/v4/nts/matrix?plot.0=a.2.0',
                            expected_code=HTTP_BAD_REQUEST)
-        self.assertIn("malformed", reply.data)
+        self.assertIn("malformed", reply.get_data(as_text=True))
 
         reply = check_code(client, '/v4/nts/matrix?plot.0=999.0.0',
                            expected_code=HTTP_NOT_FOUND)
-        self.assertIn("Invalid machine", reply.data)
+        self.assertIn("Invalid machine", reply.get_data(as_text=True))
         reply = check_code(client, '/v4/nts/matrix?plot.0=1.999.0',
                            expected_code=HTTP_NOT_FOUND)
-        self.assertIn("Invalid test", reply.data)
+        self.assertIn("Invalid test", reply.get_data(as_text=True))
         reply = check_code(client, '/v4/nts/matrix?plot.0=1.1.999',
                            expected_code=HTTP_NOT_FOUND)
-        self.assertIn("Invalid field", reply.data)
+        self.assertIn("Invalid field", reply.get_data(as_text=True))
 
     def test_matrix_view(self):
         """Does the page load with the data as expected.
@@ -66,15 +67,16 @@ def test_matrix_view(self):
                          description="foo_description",
                          prmote=True)
         rc = client.post('/v4/nts/order/6', data=form_data)
-        self.assertEquals(rc.status_code, HTTP_REDIRECT)
+        self.assertEqual(rc.status_code, HTTP_REDIRECT)
         check_code(client, '/v4/nts/set_baseline/1',
                    expected_code=HTTP_REDIRECT)
 
         reply = check_html(client, '/v4/nts/matrix?plot.0=2.6.2')
         # Make sure the data is in the page.
-        self.assertIn("test6", reply.data)
-        self.assertIn("1.0000", reply.data)
-        self.assertIn("1.2000", reply.data)
+        data = reply.get_data(as_text=True)
+        self.assertIn("test6", data)
+        self.assertIn("1.0000", data)
+        self.assertIn("1.2000", data)
 
         reply = check_html(client, '/v4/nts/matrix?plot.0=2.6.2&limit=1')