Fixes for the latest version bblfshd and python driver + extra tests (encode and get_all/load)

Signed-off-by: kuba-- <[email protected]>

Author: kuba--

.gitignore

@@ -1,4 +1,5 @@
 .idea
+.vscode
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.travis.yml

@@ -1,8 +1,6 @@
 language: python
 sudo: true
 dist: xenial
-env: 
-- BBLFSHD_VERSION=v2.12.1 BBLFSH_PYTHON_VERSION=v2.9.0
 services:
   - docker
 cache:
@@ -12,8 +10,9 @@ python:
   - "3.6"
   - "3.7"
 install:
-  - docker run --privileged -d -p 9432:9432 --name bblfshd bblfsh/bblfshd:$BBLFSHD_VERSION
-  - docker exec bblfshd bblfshctl driver install bblfsh/python-driver:$BBLFSH_PYTHON_VERSION
+  - docker run --privileged -d -p 9432:9432 --name bblfshd bblfsh/bblfshd
+  - docker exec bblfshd bblfshctl driver install bblfsh/python-driver
+  - docker exec bblfshd bblfshctl driver install bblfsh/cpp-driver
   - wget https://github.com/bblfsh/python-client/releases/download/v2.2.1/protobuf-python_3.4.1-1_amd64.deb
   - sudo dpkg -i protobuf-python_3.4.1-1_amd64.deb
   - if [[ ! -z "$TRAVIS_TAG" ]]; then sed -i -e "s/^VERSION\s*=\s*\"[^\"]\+\"/VERSION = \"${TRAVIS_TAG#v}\"/g" setup.py; fi

README.md

@@ -41,9 +41,17 @@ If you don't have a bblfsh server running you can execute it using the following
 
 ```sh
 docker run --privileged --rm -it -p 9432:9432 -v bblfsh_cache:/var/lib/bblfshd --name bblfshd bblfsh/bblfshd
-docker exec -it bblfshd bblfshctl driver install python bblfsh/python-driver:latest
 ```
 
+To parse Python files you will also need a python driver installed on bblfsh server:
+
+```sh
+docker exec -it bblfshd bblfshctl driver install python bblfsh/python-driver
+```
+
+List of all available drivers you can find at the [official documentation](https://doc.bblf.sh/languages.html).
+
+
 Please, read the [getting started](https://doc.bblf.sh/using-babelfish/getting-started.html) guide to learn more about how to use and deploy a bblfshd.
 
 ### Parsing a file
@@ -79,7 +87,7 @@ for node in it:
     doSomething(node.get())
 ```
 
-XPath queries can return different types (`dict`, `int`, `float`, `bool` or `str`), 
+XPath queries can return different types (`dict`, `int`, `float`, `bool` or `str`),
 calling `get()` with an item will return the right type, but if you must ensure
 that you are getting the expected type (to avoid errors in the queries) there
 are alternative typed versions:
@@ -92,7 +100,7 @@ z = next(ctx.filter("count(//*)")).get_int() # or get_float()
 
 ### Iteration
 
-You can also iterate using iteration orders different than the 
+You can also iterate using iteration orders different than the
 default pre-order using the `iterate` method on `parse` result or node objects:
 
 ```python
@@ -104,14 +112,14 @@ for node in it:
 # Over filter results (which by default are already iterators with PRE_ORDER):
 ctx = client.parse("file.py")
 it = ctx.filter("//python:Call").iterate(bblfsh.TreeOrder.LEVEL_ORDER)
-for node in it: 
+for node in it:
     print(node)
 
 # Over individual node objects to change the iteration order of
 # a specific subtree:
 ast = ctx.root
 it = ast.iterate(bblfsh.TreeOrder.POSITION_ORDER)
-for node in it: 
+for node in it:
     print(node)
 ```
 

bblfsh/fixtures/test.c

@@ -0,0 +1,10 @@
+unsigned long long fib(int n);
+
+int main() {
+    fib(12);
+    return 0;
+}
+
+unsigned long long fib(int n) {
+    return (n <= 1) ? 1ULL : fib(n-2) + fib(n-1);
+}

bblfsh/test.py

@@ -10,12 +10,12 @@
 from bblfsh.client import NonUTF8ContentException
 from bblfsh.node import NodeTypedGetException
 from bblfsh.result_context import (Node, NodeIterator, ResultContext)
-from bblfsh.pyuast import uast
-
+from bblfsh.pyuast import uast, decode
 
 class BblfshTests(unittest.TestCase):
     BBLFSH_SERVER_EXISTED = None
-    fixtures_file = "fixtures/test.py"
+    fixtures_pyfile = "fixtures/test.py"
+    fixtures_cfile = "fixtures/test.c"
 
     @classmethod
     def setUpClass(cls: t.Any) -> None:
@@ -32,7 +32,7 @@ def setUp(self) -> None:
         self.client = BblfshClient("localhost:9432")
 
     def _parse_fixture(self) -> ResultContext:
-        ctx = self.client.parse(self.fixtures_file)
+        ctx = self.client.parse(self.fixtures_pyfile)
         self._validate_ctx(ctx)
         return ctx
 
@@ -44,7 +44,7 @@ def testVersion(self) -> None:
         self.assertTrue(version.build)
 
     def testNativeParse(self) -> None:
-        ctx = self.client.parse(self.fixtures_file, mode=Modes.NATIVE)
+        ctx = self.client.parse(self.fixtures_pyfile, mode=Modes.NATIVE)
         self._validate_ctx(ctx)
         self.assertIsNotNone(ctx)
 
@@ -67,12 +67,24 @@ def testUASTDefaultLanguage(self) -> None:
         self.assertEqual(ctx.language, "python")
 
     def testUASTWithLanguage(self) -> None:
-        ctx = self.client.parse(self.fixtures_file, language="Python")
+        ctx = self.client.parse(self.fixtures_pyfile, language="Python")
         self._validate_ctx(ctx)
         self.assertEqual(ctx.language, "python")
 
+    def testUASTWithLanguageAlias(self) -> None:
+        ctx = self.client.parse(self.fixtures_cfile)
+        self._validate_ctx(ctx)
+        self.assertEqual(ctx.language, "c")
+
+        it = ctx.filter("//uast:FunctionGroup/Nodes/uast:Alias/Name/uast:Identifier/Name")
+        self.assertIsInstance(it, NodeIterator)
+
+        self.assertEqual(next(it).get(), "main")
+        self.assertEqual(next(it).get(), "fib")
+
+
     def testUASTFileContents(self) -> None:
-        with open(self.fixtures_file, "r") as fin:
+        with open(self.fixtures_pyfile, "r") as fin:
             contents = fin.read()
 
         ctx = self.client.parse("file.py", contents=contents)
@@ -83,14 +95,15 @@ def assert_strnode(n: Node, expected: str) -> None:
             self.assertIsInstance(n.get_str(), str)
             self.assertEqual(n.get_str(), expected)
 
-        it = ctx.filter("//uast:RuntimeImport/Path/uast:Alias/Name/uast:Identifier/Name")
+        it = ctx.filter("//uast:RuntimeImport/Path/uast:Identifier/Name")
         self.assertIsInstance(it, NodeIterator)
 
         assert_strnode(next(it), "os")
         assert_strnode(next(it), "resource")
         assert_strnode(next(it), "unittest")
         assert_strnode(next(it), "docker")
         assert_strnode(next(it), "bblfsh")
+        assert_strnode(next(it), "bblfsh")
         self.assertRaises(StopIteration, next, it)
 
     def testBrokenFilter(self) -> None:
@@ -285,15 +298,17 @@ def testFilterInsideIter(self) -> None:
 
     def testItersMixingIterations(self) -> None:
         ctx = self._parse_fixture()
+
         it = ctx.iterate(TreeOrder.PRE_ORDER)
-        next(it); next(it); next(it)
+        next(it); next(it); next(it); next(it)
+
 
-        n = next(it)
         it2 = it.iterate(TreeOrder.PRE_ORDER)
         next(it2)
+
         a = next(it).get()
         b = next(it2).get()
-        self.assertListEqual(a, b)
+        self.assertEqual(a, b)
 
     def testManyFilters(self) -> None:
         ctx = self._parse_fixture()
@@ -310,7 +325,7 @@ def testManyFilters(self) -> None:
     def testManyParses(self) -> None:
         before = resource.getrusage(resource.RUSAGE_SELF)
         for _ in range(100):
-            self.client.parse(self.fixtures_file)
+            self.client.parse(self.fixtures_pyfile)
 
         after = resource.getrusage(resource.RUSAGE_SELF)
 
@@ -320,7 +335,7 @@ def testManyParses(self) -> None:
     def testManyParsesAndFilters(self) -> None:
         before = resource.getrusage(resource.RUSAGE_SELF)
         for _ in range(100):
-            ctx = self.client.parse(self.fixtures_file)
+            ctx = self.client.parse(self.fixtures_pyfile)
             ctx.filter("//*[@role='Identifier']")
 
         after = resource.getrusage(resource.RUSAGE_SELF)
@@ -336,6 +351,48 @@ def testSupportedLanguages(self) -> None:
                 self.assertTrue(hasattr(l, key))
                 self.assertIsNotNone(getattr(l, key))
 
+    def testEncode(self) -> None:
+        ctx = self._parse_fixture()
+        self.assertEqual(ctx.ctx.encode(None, 0), ctx._response.uast)
+
+    def testEncodeWithEmptyContext(self) -> None:
+        ctx = ResultContext()
+        obj = {"k1": "v1", "k2": "v2"}
+        fmt = 1 # YAML
+
+        data = ctx.ctx.encode(obj, fmt)
+        self.assertDictEqual(obj, decode(data, format=fmt).load())
+
+    def testGetAll(self) -> None:
+        ctx = self._parse_fixture()
+
+        expected = ["os", "resource", "unittest", "docker", "bblfsh"]
+        actual = []
+        for k in ctx.get_all()["body"]:
+            if "@type" in k and k["@type"] == "uast:RuntimeImport" and "Path" in k:
+                path = k["Path"]
+                if "Name" in path:
+                    actual.append(k["Path"]["Name"])
+
+        self.assertListEqual(expected, actual)
+
+    def testLoad(self) -> None:
+        ctx = self._parse_fixture()
+
+        it = ctx.iterate(TreeOrder.PRE_ORDER)
+        next(it); next(it); next(it); next(it)
+
+        it2 = it.iterate(TreeOrder.PRE_ORDER)
+        n = next(it2)
+        node_ext = n.node_ext
+
+        obj = node_ext.load()
+        typ = obj["@type"]
+        self.assertEqual("uast:RuntimeImport", typ)
+
+        path = obj["Path"]
+        self.assertEqual("uast:Identifier", path["@type"])
+        self.assertEqual("os", path["Name"])
 
 if __name__ == "__main__":
     unittest.main()

bblfsh/test_compat.py

@@ -1,17 +1,15 @@
-import os
 import resource
 import unittest
 
-
 import docker
 
+from bblfsh.client import NonUTF8ContentException
+from bblfsh.compat import CompatBblfshClient as BblfshClient
 from bblfsh.compat import (
     filter as xpath_filter, role_id, iterator, role_name, Node, TreeOrder, filter_bool,
     filter_number, CompatNodeIterator
 )
-from bblfsh.compat import CompatBblfshClient as BblfshClient
 from bblfsh.launcher import ensure_bblfsh_is_running
-from bblfsh.client import NonUTF8ContentException
 
 
 class BblfshTests(unittest.TestCase):
@@ -252,7 +250,7 @@ def testManyFilters(self):
         root.properties['k2'] = 'v1'
 
         before = resource.getrusage(resource.RUSAGE_SELF)
-        for i in range(1000):
+        for _ in range(1000):
             xpath_filter(root, "//*[@roleIdentifier]")
 
         after = resource.getrusage(resource.RUSAGE_SELF)