Merge pull request #65 from juanjux/feature/deny_nonUTF8

Check for non UTF8 contents + test. Factorize content loading.

Author: juanjux

bblfsh/client.py

@@ -12,6 +12,10 @@
 sys.path.insert(0, os.path.dirname(__file__))
 
 
+class NonUTF8ContentException(Exception):
+    pass
+
+
 class BblfshClient(object):
     """
     Babelfish gRPC client. Currently it is only capable of fetching UASTs.
@@ -28,6 +32,21 @@ def __init__(self, endpoint):
         self._channel = grpc.insecure_channel(endpoint)
         self._stub = ProtocolServiceStub(self._channel)
 
+    @staticmethod
+    def _check_utf8(text):
+        try:
+            text.decode("utf-8")
+        except UnicodeDecodeError:
+            raise NonUTF8ContentException("Content must be UTF-8, ASCII or Base64 encoded")
+
+    @staticmethod
+    def _get_contents(contents, filename):
+        if contents is None:
+            with open(filename, "rb") as fin:
+                contents = fin.read()
+        BblfshClient._check_utf8(contents)
+        return contents
+
     def parse(self, filename, language=None, contents=None, timeout=None):
         """
         Queries the Babelfish server and receives the UAST response for the specified
@@ -48,9 +67,7 @@ def parse(self, filename, language=None, contents=None, timeout=None):
         :return: UAST object.
         """
 
-        if contents is None:
-            with open(filename, "rb") as fin:
-                contents = fin.read()
+        contents = self._get_contents(contents, filename)
         request = ParseRequest(filename=os.path.basename(filename),
                                content=contents,
                                language=self._scramble_language(language))
@@ -76,9 +93,7 @@ def native_parse(self, filename, language=None, contents=None, timeout=None):
         :return: Native AST object.
         """
 
-        if contents is None:
-            with open(filename, "rb") as fin:
-                contents = fin.read()
+        contents = self._get_contents(contents, filename)
         request = NativeParseRequest(filename=os.path.basename(filename),
                                content=contents,
                                language=self._scramble_language(language))

bblfsh/test.py

@@ -5,6 +5,7 @@
 
 from bblfsh import BblfshClient, filter, role_id, role_name, Node, ParseResponse
 from bblfsh.launcher import ensure_bblfsh_is_running
+from bblfsh.client import NonUTF8ContentException
 
 
 class BblfshTests(unittest.TestCase):
@@ -35,6 +36,10 @@ def testNativeParse(self):
         reply = self.client.native_parse(__file__)
         assert(reply.ast)
 
+    def testNonUTF8ParseError(self):
+        self.assertRaises(NonUTF8ContentException,
+                          self.client.parse, "", "Python", b"a = '\x80abc'")
+
     def testUASTDefaultLanguage(self):
         self._validate_resp(self.client.parse(__file__))
 
@@ -117,14 +122,6 @@ def testFilterBadQuery(self):
         node = Node()
         self.assertRaises(RuntimeError, filter, node, "//*roleModule")
 
-    def testIssue60(self):
-        fixtures_dir = os.path.join(
-                        os.path.dirname(os.path.realpath(__file__)),
-                        "fixtures")
-        rep = self.client.parse(os.path.join(fixtures_dir, "issue60.py"))
-        assert(rep.uast)
-        self.assertFalse(any(filter(rep.uast, "//@roleLiteral")))
-
     def testRoleIdName(sedlf):
         assert(role_id(role_name(1)) == 1)
         assert(role_name(role_id("IDENTIFIER")) == "IDENTIFIER")