From d3baafb7413693144041dc69306abe42c6eb6ca8 Mon Sep 17 00:00:00 2001
From: Jeffrey04 <mycyberpet@yahoo.com>
Date: Sat, 16 Jul 2022 08:06:17 +0100
Subject: [PATCH 1/6] 2291 proof-of-concept to turn toy example into stub

---
 utils/completion_convert.py | 39 +++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 utils/completion_convert.py

diff --git a/utils/completion_convert.py b/utils/completion_convert.py
new file mode 100644
index 000000000..5ccc1c4a3
--- /dev/null
+++ b/utils/completion_convert.py
@@ -0,0 +1,39 @@
+"""
+GitHub Issue #2291
+Convert python stub file into Qscintilla autocompletion files
+"""
+
+import importlib.util
+import sys
+from inspect import getmembers, isclass, isfunction, signature
+
+spec = importlib.util.spec_from_file_location("module", sys.argv[1])
+module = importlib.util.module_from_spec(spec)
+# sys.modules["testcode"] = module
+spec.loader.exec_module(module)
+
+
+def format_docstring(x):
+    return (
+        " ".join(sentence.strip() for sentence in x.__doc__.split("\n"))
+        if x.__doc__
+        else ""
+    )
+
+
+def main():
+    print(
+        list(
+            f"{name}{str(signature(fn))} {format_docstring(fn)}".strip()
+            for name, fn in getmembers(module, isfunction)
+        )
+        + list(
+            f"{class_name}.{class_method}{str(signature(fn))} {format_docstring(fn)}".strip()
+            for class_name, _class in getmembers(module, isclass)
+            for class_method, fn in getmembers(_class, isfunction)
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

From 86c45432f84b0c34cc4dd07eab62e3899b8750b3 Mon Sep 17 00:00:00 2001
From: Jeffrey04 <mycyberpet@yahoo.com>
Date: Sat, 16 Jul 2022 09:23:00 +0100
Subject: [PATCH 2/6] mu #2291 code now works with arbitary module

---
 utils/completion_convert.py | 64 ++++++++++++++++++++++++-------------
 1 file changed, 41 insertions(+), 23 deletions(-)

diff --git a/utils/completion_convert.py b/utils/completion_convert.py
index 5ccc1c4a3..fdd1a60e7 100644
--- a/utils/completion_convert.py
+++ b/utils/completion_convert.py
@@ -3,36 +3,54 @@
 Convert python stub file into Qscintilla autocompletion files
 """
 
-import importlib.util
+import ast
 import sys
-from inspect import getmembers, isclass, isfunction, signature
 
-spec = importlib.util.spec_from_file_location("module", sys.argv[1])
-module = importlib.util.module_from_spec(spec)
-# sys.modules["testcode"] = module
-spec.loader.exec_module(module)
 
+def format_arguments(arguments):
+    result = []
 
-def format_docstring(x):
-    return (
-        " ".join(sentence.strip() for sentence in x.__doc__.split("\n"))
-        if x.__doc__
-        else ""
-    )
+    for arg in arguments.args:
+        result.append(f"{arg.arg}")
+
+    if arguments.vararg:
+        result.append(f"*{arguments.vararg.arg}")
+
+    if arguments.kwarg:
+        result.append(f"**{arguments.kwarg.arg}")
+
+    return f"({', '.join(result)})"
 
 
 def main():
-    print(
-        list(
-            f"{name}{str(signature(fn))} {format_docstring(fn)}".strip()
-            for name, fn in getmembers(module, isfunction)
-        )
-        + list(
-            f"{class_name}.{class_method}{str(signature(fn))} {format_docstring(fn)}".strip()
-            for class_name, _class in getmembers(module, isclass)
-            for class_method, fn in getmembers(_class, isfunction)
-        )
-    )
+    with open(sys.argv[1], "r") as f:
+        tree = ast.parse(f.read())
+
+        for node in ast.iter_child_nodes(tree):
+            if isinstance(node, ast.FunctionDef):
+                docstring = ast.get_docstring(node, clean=True)
+                print(
+                    "{}{} {}".format(
+                        node.name,
+                        format_arguments(node.args),
+                        " ".join(docstring.split("\n")) if docstring else "",
+                    )
+                )
+            elif isinstance(node, ast.ClassDef):
+                for node_sub in node.body:
+                    if isinstance(node_sub, ast.FunctionDef) and (
+                        not node_sub.name.startswith("__")
+                        and not node_sub.name.endswith("__")
+                    ):
+                        docstring = ast.get_docstring(node_sub, clean=True)
+                        print(
+                            "{}.{}{} {}".format(
+                                node.name,
+                                node_sub.name,
+                                format_arguments(node_sub.args),
+                                " ".join(docstring.split("\n")) if docstring else "",
+                            )
+                        )
 
 
 if __name__ == "__main__":

From db4091dba2cb5ef597a1a6e2934d9c2ed04595b6 Mon Sep 17 00:00:00 2001
From: Jeffrey04 <mycyberpet@yahoo.com>
Date: Sat, 16 Jul 2022 10:12:30 +0100
Subject: [PATCH 3/6] mu #2291 writing generated code into a script

---
 utils/completion_convert.py | 64 ++++++++++++++++++++++---------------
 1 file changed, 38 insertions(+), 26 deletions(-)

diff --git a/utils/completion_convert.py b/utils/completion_convert.py
index fdd1a60e7..f290eb3b5 100644
--- a/utils/completion_convert.py
+++ b/utils/completion_convert.py
@@ -23,34 +23,46 @@ def format_arguments(arguments):
 
 
 def main():
-    with open(sys.argv[1], "r") as f:
-        tree = ast.parse(f.read())
-
-        for node in ast.iter_child_nodes(tree):
-            if isinstance(node, ast.FunctionDef):
-                docstring = ast.get_docstring(node, clean=True)
-                print(
-                    "{}{} {}".format(
-                        node.name,
-                        format_arguments(node.args),
-                        " ".join(docstring.split("\n")) if docstring else "",
+    with open("test.py", "w") as fout:
+        result = []
+
+        with open(sys.argv[1], "r") as fin:
+            tree = ast.parse(fin.read())
+
+            for node in ast.iter_child_nodes(tree):
+                if isinstance(node, ast.FunctionDef):
+                    docstring = ast.get_docstring(node, clean=True)
+                    result.append(
+                        "{}{} {}".format(
+                            node.name,
+                            format_arguments(node.args),
+                            " ".join(docstring.split("\n")) if docstring else "",
+                        )
                     )
-                )
-            elif isinstance(node, ast.ClassDef):
-                for node_sub in node.body:
-                    if isinstance(node_sub, ast.FunctionDef) and (
-                        not node_sub.name.startswith("__")
-                        and not node_sub.name.endswith("__")
-                    ):
-                        docstring = ast.get_docstring(node_sub, clean=True)
-                        print(
-                            "{}.{}{} {}".format(
-                                node.name,
-                                node_sub.name,
-                                format_arguments(node_sub.args),
-                                " ".join(docstring.split("\n")) if docstring else "",
+                elif isinstance(node, ast.ClassDef):
+                    for node_sub in node.body:
+                        if isinstance(node_sub, ast.FunctionDef) and (
+                            not node_sub.name.startswith("__")
+                            and not node_sub.name.endswith("__")
+                        ):
+                            docstring = ast.get_docstring(node_sub, clean=True)
+                            result.append(
+                                "{}.{}{} {}".format(
+                                    node.name,
+                                    node_sub.name,
+                                    format_arguments(node_sub.args),
+                                    " ".join(docstring.split("\n"))
+                                    if docstring
+                                    else "",
+                                )
                             )
-                        )
+
+        # FIXME fix this hack with ast.unparse when version is bumped to >= 3.9
+        fout.write(
+            "MICROBIT_APIS = [{}]".format(
+                ", ".join(fr'_("{doc.strip()}")' for doc in result)
+            )
+        )
 
 
 if __name__ == "__main__":

From 3bf381b0966529ae77ecdaf54ad3dc69ed594600 Mon Sep 17 00:00:00 2001
From: Jeffrey04 <mycyberpet@yahoo.com>
Date: Sat, 16 Jul 2022 11:20:04 +0100
Subject: [PATCH 4/6] mu #2291 now traverse the directory for stub file

---
 utils/completion_convert.py | 86 +++++++++++++++++++++++--------------
 1 file changed, 53 insertions(+), 33 deletions(-)

diff --git a/utils/completion_convert.py b/utils/completion_convert.py
index f290eb3b5..c61375eac 100644
--- a/utils/completion_convert.py
+++ b/utils/completion_convert.py
@@ -3,15 +3,17 @@
 Convert python stub file into Qscintilla autocompletion files
 """
 
+import argparse
 import ast
-import sys
+import pathlib
+from glob import glob
+from os.path import isdir, sep
 
 
-def format_arguments(arguments):
+def format_signature(arguments):
     result = []
 
-    for arg in arguments.args:
-        result.append(f"{arg.arg}")
+    result.extend(f"{arg.arg}" for arg in arguments.args)
 
     if arguments.vararg:
         result.append(f"*{arguments.vararg.arg}")
@@ -22,43 +24,57 @@ def format_arguments(arguments):
     return f"({', '.join(result)})"
 
 
-def main():
-    with open("test.py", "w") as fout:
-        result = []
+def format_function(node):
+    docstring = ast.get_docstring(node, clean=True)
+    return "{}{} {}".format(
+        node.name,
+        format_signature(node.args),
+        " ".join(doc.strip() for doc in docstring.split("\n")) if docstring else "",
+    )
 
-        with open(sys.argv[1], "r") as fin:
+
+def format_methods(node_class):
+    result = []
+
+    for node_sub in node_class.body:
+        if isinstance(node_sub, ast.FunctionDef) and (
+            not node_sub.name.startswith("__") and not node_sub.name.endswith("__")
+        ):
+            docstring = ast.get_docstring(node_sub, clean=True)
+            result.append(
+                "{}.{}{} {}".format(
+                    node_class.name,
+                    node_sub.name,
+                    format_signature(node_sub.args),
+                    " ".join(doc.strip() for doc in docstring.split("\n"))
+                    if docstring
+                    else "",
+                )
+            )
+
+    return result
+
+
+def main(arguments):
+    assert isdir(arguments.stub_dir)
+
+    result = []
+
+    for stub in glob(f"{arguments.stub_dir}{sep}*.py"):
+        with open(stub, "r") as fin:
             tree = ast.parse(fin.read())
 
             for node in ast.iter_child_nodes(tree):
                 if isinstance(node, ast.FunctionDef):
-                    docstring = ast.get_docstring(node, clean=True)
-                    result.append(
-                        "{}{} {}".format(
-                            node.name,
-                            format_arguments(node.args),
-                            " ".join(docstring.split("\n")) if docstring else "",
-                        )
-                    )
+                    result.append(format_function(node))
+
                 elif isinstance(node, ast.ClassDef):
-                    for node_sub in node.body:
-                        if isinstance(node_sub, ast.FunctionDef) and (
-                            not node_sub.name.startswith("__")
-                            and not node_sub.name.endswith("__")
-                        ):
-                            docstring = ast.get_docstring(node_sub, clean=True)
-                            result.append(
-                                "{}.{}{} {}".format(
-                                    node.name,
-                                    node_sub.name,
-                                    format_arguments(node_sub.args),
-                                    " ".join(docstring.split("\n"))
-                                    if docstring
-                                    else "",
-                                )
-                            )
+                    result.extend(format_methods(node))
 
+    with arguments.output as fout:
         # FIXME fix this hack with ast.unparse when version is bumped to >= 3.9
         fout.write(
+            # FIXME what should be the variable name? make this another argument to the script?
             "MICROBIT_APIS = [{}]".format(
                 ", ".join(fr'_("{doc.strip()}")' for doc in result)
             )
@@ -66,4 +82,8 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    parser = argparse.ArgumentParser()
+    parser.add_argument("stub_dir", type=pathlib.Path)
+    parser.add_argument("output", type=argparse.FileType("w"))
+
+    main(parser.parse_args())

From 588064775e3912dcdee9424001a0d2937d798ed9 Mon Sep 17 00:00:00 2001
From: Jeffrey04 <mycyberpet@yahoo.com>
Date: Sat, 16 Jul 2022 11:38:01 +0100
Subject: [PATCH 5/6] mu #2291 refactored and cleaned up the script

---
 utils/completion_convert.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/utils/completion_convert.py b/utils/completion_convert.py
index c61375eac..5e9073252 100644
--- a/utils/completion_convert.py
+++ b/utils/completion_convert.py
@@ -10,6 +10,17 @@
 from os.path import isdir, sep
 
 
+def check_function_is_magic(fn):
+    return fn.name.startswith("__") and fn.name.endswith("__")
+
+
+def format_docstring(node):
+    docstring = ast.get_docstring(node, clean=True)
+    return (
+        " ".join(doc.strip() for doc in docstring.split("\n")) if docstring else "",
+    )
+
+
 def format_signature(arguments):
     result = []
 
@@ -25,11 +36,8 @@ def format_signature(arguments):
 
 
 def format_function(node):
-    docstring = ast.get_docstring(node, clean=True)
     return "{}{} {}".format(
-        node.name,
-        format_signature(node.args),
-        " ".join(doc.strip() for doc in docstring.split("\n")) if docstring else "",
+        node.name, format_signature(node.args), format_docstring(node)
     )
 
 
@@ -37,18 +45,15 @@ def format_methods(node_class):
     result = []
 
     for node_sub in node_class.body:
-        if isinstance(node_sub, ast.FunctionDef) and (
-            not node_sub.name.startswith("__") and not node_sub.name.endswith("__")
+        if isinstance(node_sub, ast.FunctionDef) and not check_function_is_magic(
+            node_sub
         ):
-            docstring = ast.get_docstring(node_sub, clean=True)
             result.append(
                 "{}.{}{} {}".format(
                     node_class.name,
                     node_sub.name,
                     format_signature(node_sub.args),
-                    " ".join(doc.strip() for doc in docstring.split("\n"))
-                    if docstring
-                    else "",
+                    format_docstring(node_sub),
                 )
             )
 

From ed15bf1936a41b877410a514a4636f9e88f46f62 Mon Sep 17 00:00:00 2001
From: Jeffrey04 <mycyberpet@yahoo.com>
Date: Sat, 16 Jul 2022 11:47:21 +0100
Subject: [PATCH 6/6] mu #2291 add a newline after signature, as instructed in
 the issue

---
 utils/completion_convert.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/utils/completion_convert.py b/utils/completion_convert.py
index 5e9073252..c5e7c5be2 100644
--- a/utils/completion_convert.py
+++ b/utils/completion_convert.py
@@ -16,9 +16,7 @@ def check_function_is_magic(fn):
 
 def format_docstring(node):
     docstring = ast.get_docstring(node, clean=True)
-    return (
-        " ".join(doc.strip() for doc in docstring.split("\n")) if docstring else "",
-    )
+    return " ".join(doc.strip() for doc in docstring.split("\n")) if docstring else ""
 
 
 def format_signature(arguments):
@@ -36,7 +34,7 @@ def format_signature(arguments):
 
 
 def format_function(node):
-    return "{}{} {}".format(
+    return r"{}{} \n{}".format(
         node.name, format_signature(node.args), format_docstring(node)
     )
 
@@ -49,7 +47,7 @@ def format_methods(node_class):
             node_sub
         ):
             result.append(
-                "{}.{}{} {}".format(
+                r"{}.{}{} \n{}".format(
                     node_class.name,
                     node_sub.name,
                     format_signature(node_sub.args),