pybricksMicropython/lib: replace parser

This replaces the Python parsing library. The python-ast package has
really poor performance (many seconds to parse a 500 line file). The
new package can parse a file about 100 times faster.

Author: dlech

CHANGELOG.md

@@ -4,6 +4,11 @@
 
 ## [Unreleased]
 
+### Fixed
+- Fixed UI temporary freeze when downloading and running programs ([support#750]).
+
+[support#750]: https://github.com/pybricks/support/issues/750
+
 ## [2.0.0-beta.6] - 2022-10-21
 
 ### Added

config/webpack.config.js

@@ -802,5 +802,6 @@ module.exports = function (webpackEnv) {
     // Turn off performance processing because we utilize
     // our own hints via the FileSizeReporter
     performance: false,
+    ignoreWarnings: [/Failed to parse source map/],
   };
 };

package.json

@@ -18,6 +18,7 @@
         "@pybricks/jedi": "^1.2.0",
         "@pybricks/mpy-cross-v5": "^2.0.0",
         "@pybricks/mpy-cross-v6": "^2.0.0",
+        "@qoretechnologies/python-parser": "^0.4.10",
         "@reduxjs/toolkit": "^1.8.6",
         "@shopify/react-i18n": "^7.2.4",
         "@svgr/webpack": "^6.5.0",
@@ -38,7 +39,6 @@
         "@types/web-locks-api": "^0.0.2",
         "@types/wicg-file-system-access": "^2020.9.5",
         "@types/zen-push": "^0.1.1",
-        "assert": "^2.0.0",
         "babel-jest": "^29.2.1",
         "babel-loader": "^8.2.3",
         "babel-plugin-macros": "^3.0.1",
@@ -86,7 +86,6 @@
         "prompts": "^2.4.2",
         "prop-types": "^15.8.1",
         "pyodide": "0.21.3",
-        "python-ast": "^0.1.0",
         "react": "^16.13.1",
         "react-app-polyfill": "^3.0.0",
         "react-aria": "^3.20.0",
@@ -118,7 +117,6 @@
         "typescript": "~4.8.4",
         "usehooks-ts": "^2.9.1",
         "user-agent-data-types": "^0.3.0",
-        "util": "^0.12.5",
         "web-vitals": "^3.0.4",
         "webpack": "^5.74.0",
         "webpack-dev-server": "^4.11.1",
@@ -169,7 +167,6 @@
     },
     "packageManager": "[email protected]",
     "resolutions": {
-        "antlr4ts@^0.5.0-alpha.3": "0.5.0-alpha.4",
         "[email protected]": "patch:mq-polyfill@npm:1.1.8#.yarn/patches/mq-polyfill-npm-1.1.8-62fe162439.patch",
         "react-error-overlay": "6.0.9",
         "react-dev-utils@^12.0.1": "patch:react-dev-utils@npm:12.0.1#.yarn/patches/react-dev-utils-npm-12.0.1-83ba06e3ee.patch",

src/pybricksMicropython/lib.test.ts

@@ -96,6 +96,17 @@ from .r import x
 from ..r import x
 from ...r import x
 from ....r import x
+
+# import q
+# from q import q
+"""
+import q
+from q import q
+"""
+'''
+import q
+from q import q
+'''
 `;
 
     const modules = findImportedModules(script);

src/pybricksMicropython/lib.ts

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) 2022 The Pybricks Authors
 
-import { parse, walk } from 'python-ast';
+import { parse, walk } from '@qoretechnologies/python-parser';
 import type { FileContents, FileStorageDb } from '../fileStorage';
 
 /** The Python file extension ('.py') */
@@ -82,47 +82,17 @@ export function findImportedModules(py: string): ReadonlySet<string> {
     const tree = parse(py);
 
     // find all import statements in the syntax tree and collect imported modules
-    walk(
-        {
-            enterImport_stmt: (ctx) => {
-                // import statements have two forms:
-                // import_stmt: import_name | import_from;
-
-                // import_name: 'import' dotted_as_names
-                const name = ctx.import_name();
-
-                if (name) {
-                    // dotted_as_names: dotted_as_name (',' dotted_as_name)*;
-                    // dotted_as_name: dotted_name ('as' NAME)?;
-                    for (const dottedAsName of name
-                        .dotted_as_names()
-                        .dotted_as_name()) {
-                        // dotted_name: NAME ('.' NAME)*;
-                        modules.add(dottedAsName.dotted_name().text);
-                    }
+    walk(tree, {
+        onEnterNode(node, _ancestors) {
+            if (node.type === 'import') {
+                for (const name of node.names) {
+                    modules.add(name.path);
                 }
-
-                // import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) 'import' ('*' | '(' import_as_names ')' | import_as_names ));
-                const from = ctx.import_from();
-
-                if (from) {
-                    // the leading dots aren't included in dotted_name, so
-                    // we need to collect them separately
-                    const leadingDots = from
-                        .DOT()
-                        .concat(from.ELLIPSIS())
-                        .map((n) => n.symbol.text)
-                        .join('');
-
-                    // dotted_name: NAME ('.' NAME)*;
-                    const dottedName = from.dotted_name()?.text ?? '';
-
-                    modules.add(leadingDots + dottedName);
-                }
-            },
+            } else if (node.type === 'from') {
+                modules.add(node.base);
+            }
         },
-        tree,
-    );
+    });
 
     return modules;
 }

yarn.lock

@@ -2412,6 +2412,7 @@ __metadata:
     "@pybricks/jedi": ^1.2.0
     "@pybricks/mpy-cross-v5": ^2.0.0
     "@pybricks/mpy-cross-v6": ^2.0.0
+    "@qoretechnologies/python-parser": ^0.4.10
     "@reduxjs/toolkit": ^1.8.6
     "@shopify/react-i18n": ^7.2.4
     "@svgr/webpack": ^6.5.0
@@ -2434,7 +2435,6 @@ __metadata:
     "@types/zen-push": ^0.1.1
     "@typescript-eslint/eslint-plugin": ^5.40.1
     "@typescript-eslint/parser": ^5.40.1
-    assert: ^2.0.0
     babel-jest: ^29.2.1
     babel-loader: ^8.2.3
     babel-plugin-macros: ^3.0.1
@@ -2493,7 +2493,6 @@ __metadata:
     prompts: ^2.4.2
     prop-types: ^15.8.1
     pyodide: 0.21.3
-    python-ast: ^0.1.0
     react: ^16.13.1
     react-app-polyfill: ^3.0.0
     react-aria: ^3.20.0
@@ -2525,7 +2524,6 @@ __metadata:
     typescript: ~4.8.4
     usehooks-ts: ^2.9.1
     user-agent-data-types: ^0.3.0
-    util: ^0.12.5
     web-vitals: ^3.0.4
     webpack: ^5.74.0
     webpack-dev-server: ^4.11.1
@@ -2537,6 +2535,15 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@qoretechnologies/python-parser@npm:^0.4.10":
+  version: 0.4.10
+  resolution: "@qoretechnologies/python-parser@npm:0.4.10"
+  dependencies:
+    lodash: ^4.17.15
+  checksum: ab0d91719a8301b64f9c7fb1a7711721d52d78325abbf0c7c75940a48fd4a3d2b07456c95f8507af19faa333bf87116dd62f18e6c8e76486a57673e3224ac3ee
+  languageName: node
+  linkType: hard
+
 "@react-aria/breadcrumbs@npm:^3.3.2":
   version: 3.3.2
   resolution: "@react-aria/breadcrumbs@npm:3.3.2"
@@ -5425,13 +5432,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"antlr4ts@npm:0.5.0-alpha.4":
-  version: 0.5.0-alpha.4
-  resolution: "antlr4ts@npm:0.5.0-alpha.4"
-  checksum: 37948499d59477f5b5a8ea71dfb8b5330e71d5a7cee60f57351dd744219b8619fa6aac1a5b6ec1a9991846e8ddc9ca47680eb166c59b44333369b3115e7aa358
-  languageName: node
-  linkType: hard
-
 "anymatch@npm:^3.0.3, anymatch@npm:~3.1.2":
   version: 3.1.2
   resolution: "anymatch@npm:3.1.2"
@@ -5580,18 +5580,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"assert@npm:^2.0.0":
-  version: 2.0.0
-  resolution: "assert@npm:2.0.0"
-  dependencies:
-    es6-object-assign: ^1.1.0
-    is-nan: ^1.2.1
-    object-is: ^1.0.1
-    util: ^0.12.0
-  checksum: bb91f181a86d10588ee16c5e09c280f9811373974c29974cbe401987ea34e966699d7989a812b0e19377b511ea0bc627f5905647ce569311824848ede382cae8
-  languageName: node
-  linkType: hard
-
 "ast-types-flow@npm:^0.0.7":
   version: 0.0.7
   resolution: "ast-types-flow@npm:0.0.7"
@@ -5645,13 +5633,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"available-typed-arrays@npm:^1.0.5":
-  version: 1.0.5
-  resolution: "available-typed-arrays@npm:1.0.5"
-  checksum: 20eb47b3cefd7db027b9bbb993c658abd36d4edd3fe1060e83699a03ee275b0c9b216cc076ff3f2db29073225fb70e7613987af14269ac1fe2a19803ccc97f1a
-  languageName: node
-  linkType: hard
-
 "axe-core@npm:^4.3.5":
   version: 4.4.1
   resolution: "axe-core@npm:4.4.1"
@@ -7500,7 +7481,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"es-abstract@npm:^1.19.0, es-abstract@npm:^1.19.1, es-abstract@npm:^1.19.2, es-abstract@npm:^1.19.5, es-abstract@npm:^1.20.0":
+"es-abstract@npm:^1.19.0, es-abstract@npm:^1.19.1, es-abstract@npm:^1.19.2, es-abstract@npm:^1.19.5":
   version: 1.20.4
   resolution: "es-abstract@npm:1.20.4"
   dependencies:
@@ -7559,13 +7540,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"es6-object-assign@npm:^1.1.0":
-  version: 1.1.0
-  resolution: "es6-object-assign@npm:1.1.0"
-  checksum: 8d4fdf63484d78b5c64cacc2c2e1165bc7b6a64b739d2a9db6a4dc8641d99cc9efb433cdd4dc3d3d6b00bfa6ce959694e4665e3255190339945c5f33b692b5d8
-  languageName: node
-  linkType: hard
-
 "escalade@npm:^3.1.1":
   version: 3.1.1
   resolution: "escalade@npm:3.1.1"
@@ -8371,15 +8345,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"for-each@npm:^0.3.3":
-  version: 0.3.3
-  resolution: "for-each@npm:0.3.3"
-  dependencies:
-    is-callable: ^1.1.3
-  checksum: 6c48ff2bc63362319c65e2edca4a8e1e3483a2fabc72fbe7feaf8c73db94fc7861bd53bc02c8a66a0c1dd709da6b04eec42e0abdd6b40ce47305ae92a25e5d28
-  languageName: node
-  linkType: hard
-
 "fork-ts-checker-webpack-plugin@npm:^6.5.0":
   version: 6.5.2
   resolution: "fork-ts-checker-webpack-plugin@npm:6.5.2"
@@ -9291,7 +9256,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"is-callable@npm:^1.1.3, is-callable@npm:^1.1.4, is-callable@npm:^1.2.7":
+"is-callable@npm:^1.1.4, is-callable@npm:^1.2.7":
   version: 1.2.7
   resolution: "is-callable@npm:1.2.7"
   checksum: 61fd57d03b0d984e2ed3720fb1c7a897827ea174bd44402878e059542ea8c4aeedee0ea0985998aa5cc2736b2fa6e271c08587addb5b3959ac52cf665173d1ac
@@ -9346,15 +9311,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"is-generator-function@npm:^1.0.7":
-  version: 1.0.10
-  resolution: "is-generator-function@npm:1.0.10"
-  dependencies:
-    has-tostringtag: ^1.0.0
-  checksum: d54644e7dbaccef15ceb1e5d91d680eb5068c9ee9f9eb0a9e04173eb5542c9b51b5ab52c5537f5703e48d5fddfd376817c1ca07a84a407b7115b769d4bdde72b
-  languageName: node
-  linkType: hard
-
 "is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3, is-glob@npm:~4.0.1":
   version: 4.0.3
   resolution: "is-glob@npm:4.0.3"
@@ -9392,16 +9348,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"is-nan@npm:^1.2.1":
-  version: 1.3.2
-  resolution: "is-nan@npm:1.3.2"
-  dependencies:
-    call-bind: ^1.0.0
-    define-properties: ^1.1.3
-  checksum: 5dfadcef6ad12d3029d43643d9800adbba21cf3ce2ec849f734b0e14ee8da4070d82b15fdb35138716d02587c6578225b9a22779cab34888a139cc43e4e3610a
-  languageName: node
-  linkType: hard
-
 "is-negative-zero@npm:^2.0.2":
   version: 2.0.2
   resolution: "is-negative-zero@npm:2.0.2"
@@ -9518,19 +9464,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"is-typed-array@npm:^1.1.3, is-typed-array@npm:^1.1.9":
-  version: 1.1.9
-  resolution: "is-typed-array@npm:1.1.9"
-  dependencies:
-    available-typed-arrays: ^1.0.5
-    call-bind: ^1.0.2
-    es-abstract: ^1.20.0
-    for-each: ^0.3.3
-    has-tostringtag: ^1.0.0
-  checksum: 11910f1e58755fef43bf0074e52fa5b932bf101ec65d613e0a83d40e8e4c6e3f2ee142d624ebc7624c091d3bbe921131f8db7d36ecbbb71909f2fe310c1faa65
-  languageName: node
-  linkType: hard
-
 "is-weakref@npm:^1.0.2":
   version: 1.0.2
   resolution: "is-weakref@npm:1.0.2"
@@ -12645,15 +12578,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"python-ast@npm:^0.1.0":
-  version: 0.1.0
-  resolution: "python-ast@npm:0.1.0"
-  dependencies:
-    antlr4ts: ^0.5.0-alpha.3
-  checksum: 1fc453e89f932a1e7f6f57a0f9dc304252093e8aa6c934a426ef30c7e2c9fd0c05aad168fa3ad053ba3bb88362a2fb5097c6e9d360351539f5027ff2af0e5eab
-  languageName: node
-  linkType: hard
-
 "qs@npm:6.10.3":
   version: 6.10.3
   resolution: "qs@npm:6.10.3"
@@ -15032,19 +14956,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"util@npm:^0.12.0, util@npm:^0.12.5":
-  version: 0.12.5
-  resolution: "util@npm:0.12.5"
-  dependencies:
-    inherits: ^2.0.3
-    is-arguments: ^1.0.4
-    is-generator-function: ^1.0.7
-    is-typed-array: ^1.1.3
-    which-typed-array: ^1.1.2
-  checksum: 705e51f0de5b446f4edec10739752ac25856541e0254ea1e7e45e5b9f9b0cb105bc4bd415736a6210edc68245a7f903bf085ffb08dd7deb8a0e847f60538a38a
-  languageName: node
-  linkType: hard
-
 "utila@npm:~0.4":
   version: 0.4.0
   resolution: "utila@npm:0.4.0"
@@ -15407,20 +15318,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"which-typed-array@npm:^1.1.2":
-  version: 1.1.8
-  resolution: "which-typed-array@npm:1.1.8"
-  dependencies:
-    available-typed-arrays: ^1.0.5
-    call-bind: ^1.0.2
-    es-abstract: ^1.20.0
-    for-each: ^0.3.3
-    has-tostringtag: ^1.0.0
-    is-typed-array: ^1.1.9
-  checksum: bedf4d30a738e848404fe67fe0ace33433a7298cf3f5a4d4b2c624ba99c4d25f06a7fd6f3566c3d16af5f8a54f0c6293cbfded5b1208ce11812753990223b45a
-  languageName: node
-  linkType: hard
-
 "which@npm:^1.3.1":
   version: 1.3.1
   resolution: "which@npm:1.3.1"