feat(powerbi): replace Lark M-Query parser with Microsoft powerquery-parser

.gitattributes

@@ -3,3 +3,4 @@
 *.tsx text eol=lf
 gradlew text eol=lf
 metadata-utils/src/test/resources/filterQuery/* text eol=lf
+

docs/how/updating-datahub.md

@@ -33,8 +33,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ### Breaking Changes
 
-- #16341 (Ingestion) SQL parsing: View query IDs are now generated using a SHA-256 hash instead of URL-encoding the view URN. This affects all connectors that use view lineage tracking (Snowflake, Oracle, BigQuery, Postgres, MySQL, Hive, Trino, ClickHouse, DB2, Dremio, SQL Server, and others). Previously, query entities had URNs like `urn:li:query:view_urn%3Ali%3Adataset%3A%28...%29`; they now use `urn:li:query:view_<sha256hash>`. After upgrading, the old URL-encoded query entities will become stale and orphaned. To clean them up, enable stateful ingestion with stale entity removal in your recipe and re-run ingestion.
-
+- #16685:(Ingestion) PowerBI M-Query lineage extraction has been rewritten using Microsoft's official `@microsoft/powerquery-parser`. As part of this change, the `native_query_parsing: false` configuration flag now suppresses only expressions containing `Value.NativeQuery`. Previously it suppressed all M-Query lineage extraction. Users who set this flag to block all lineage extraction will now see lineage produced for non-NativeQuery sources (Snowflake, PostgreSQL, MSSQL, etc.). To restore the suppress-all behaviour, add a `table_pattern.deny` rule in your recipe.
+- #16341:(Ingestion) SQL parsing: View query IDs are now generated using a SHA-256 hash instead of URL-encoding the view URN. This affects all connectors that use view lineage tracking (Snowflake, Oracle, BigQuery, Postgres, MySQL, Hive, Trino, ClickHouse, DB2, Dremio, SQL Server, and others). Previously, query entities had URNs like `urn:li:query:view_urn%3Ali%3Adataset%3A%28...%29`; they now use `urn:li:query:view_<sha256hash>`. After upgrading, the old URL-encoded query entities will become stale and orphaned. To clean them up, enable stateful ingestion with stale entity removal in your recipe and re-run ingestion.
 - #16396: Oracle connector: When connecting via `service_name` to a multitenant Oracle database, the database name used in URNs will now reflect the Pluggable Database (PDB) name instead of the Container Database (CDB) name. In Oracle Multitenant architecture, a CDB is the top-level container (e.g. `cdb`) and a PDB is an individual tenant database within it (e.g. `mypdb`); `service_name` typically routes to the PDB, so the PDB name is the correct identifier for your datasets. This affects both dataset URNs (when `add_database_name_to_urn: true`) and database/schema container URNs (always, since containers always include the database name). If your existing metadata was ingested with the old CDB-based URNs, re-ingesting will create new entities under the corrected URNs. To preserve the old URN shape and avoid re-creating entities, set `urn_db_name` explicitly in your recipe to match your previous CDB name.
 - #16628 (Ingestion) Fabric OneLake source: Workspace containers now use the `fabric` platform instead of `fabric-onelake`. This changes workspace container URNs and the `dataPlatformInstance.platform` emitted for workspace entities. Lakehouse, warehouse, schema, and dataset entities remain on `fabric-onelake`.
 - **Retention service disabled: only current version retained.** When the retention service is not enabled (not configured or unavailable), the write path now retains only the current version (version 0) and does not create version-history rows. Previously, version history was still written when retention was disabled. **Impact:** Deployments that run without retention enabled will no longer accumulate aspect version history; only the latest aspect value is stored. **Migration:** Enable and configure the retention service (e.g. ingest retention policies from `boot/retention.yaml`) if you need version history for any entity/aspect.

metadata-ingestion/constraints.txt

@@ -782,8 +782,6 @@ langcodes==3.5.1
     # via spacy
 langdetect==1.0.9
     # via unstructured
-lark==1.1.4
-    # via acryl-datahub
 leb128==1.0.9
     # via asynch
 linear-tsv==1.1.0
@@ -841,6 +839,8 @@ mdurl==0.1.2
     # via markdown-it-py
 memray==1.19.1
     # via acryl-datahub
+mini-racer==0.14.1
+    # via acryl-datahub
 mistune==3.2.0
     # via
     #   acryl-great-expectations
@@ -1370,7 +1370,6 @@ referencing==0.37.0
     #   jupyter-events
 regex==2026.2.28
     # via
-    #   lark
     #   nltk
     #   tiktoken
 requests==2.32.5

metadata-ingestion/pyproject.toml

@@ -943,7 +943,7 @@ postgres = [
 ]
 
 powerbi = [
-    "lark[regex]==1.1.4",
+    "mini-racer==0.14.1",
     "more-itertools<11.0.0",
     "msal>=1.31.1,<2.0.0",
     "patchy==2.8.0",
@@ -1419,10 +1419,10 @@ all = [
     "jsonpath-ng==1.7.0",
     "jupyter_server>=2.14.1,<3.0.0",
     "kerberos>=1.3.0,<2.0.0",
-    "lark[regex]==1.1.4",
     "litellm==1.80.5",
     "lkml>=1.3.4,<2.0.0",
     "looker-sdk>=23.0.0,<26.0.0",
+    "mini-racer==0.14.1",
     "mlflow-skinny>=2.3.0,<2.21.0",
     "more-itertools>=8.12.0,<11.0.0",
     "moto[s3]>=5.0.0,<6.0.0",
@@ -1570,10 +1570,10 @@ dev = [
     "jsonschema<5.0.0",
     "jupyter_server>=2.14.1,<3.0.0",
     "kerberos>=1.3.0,<2.0.0",
-    "lark[regex]==1.1.4",
     "litellm==1.80.5",
     "lkml>=1.3.4,<2.0.0",
     "looker-sdk>=23.0.0,<26.0.0",
+    "mini-racer==0.14.1",
     "mixpanel>=4.9.0,<6.0.0",
     "mlflow-skinny>=2.3.0,<2.21.0",
     "more-itertools>=8.12.0,<11.0.0",
@@ -1761,10 +1761,10 @@ docs = [
     "jsonschema<5.0.0",
     "jupyter_server>=2.14.1,<3.0.0",
     "kerberos>=1.3.0,<2.0.0",
-    "lark[regex]==1.1.4",
     "litellm==1.80.5",
     "lkml>=1.3.4,<2.0.0",
     "looker-sdk>=23.0.0,<26.0.0",
+    "mini-racer==0.14.1",
     "mixpanel>=4.9.0,<6.0.0",
     "mlflow-skinny>=2.3.0,<2.21.0",
     "more-itertools>=8.12.0,<11.0.0",
@@ -2147,7 +2147,7 @@ datahub = ["py.typed"]
 "datahub.cli.gql" = ["*.gql"]
 "datahub.cli.resources" = ["*.md"]
 "datahub.ingestion.autogenerated" = ["*.json"]
-"datahub.ingestion.source.powerbi" = ["powerbi-lexical-grammar.rule"]
+"datahub.ingestion.source.powerbi.m_query.mquery_bridge" = ["bundle.js.gz"]
 "datahub.metadata" = ["schema.avsc"]
 "datahub.metadata.schemas" = ["*.avsc"]
 

metadata-ingestion/setup.py

@@ -744,7 +744,7 @@
     "nifi": {"requests<3.0.0", "packaging<26.0.0", "requests-gssapi<2.0.0"},
     "powerbi": (
         microsoft_common
-        | {"lark[regex]==1.1.4", "sqlparse<1.0.0", "more-itertools<11.0.0"}
+        | {"sqlparse<1.0.0", "more-itertools<11.0.0", "mini-racer==0.14.1"}
         | sqlglot_lib
         | threading_timeout_common
     ),
@@ -1204,7 +1204,7 @@
         "datahub": ["py.typed"],
         "datahub.metadata": ["schema.avsc"],
         "datahub.metadata.schemas": ["*.avsc"],
-        "datahub.ingestion.source.powerbi": ["powerbi-lexical-grammar.rule"],
+        "datahub.ingestion.source.powerbi.m_query.mquery_bridge": ["bundle.js.gz"],
         "datahub.ingestion.autogenerated": ["*.json"],
         "datahub.cli.gql": ["*.gql"],
         "datahub.cli.resources": ["*.md"],

metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py

@@ -247,8 +247,14 @@ class PowerBiDashboardSourceReport(StaleEntityRemovalSourceReport):
     m_query_parse_attempts: int = 0
     m_query_parse_successes: int = 0
     m_query_parse_timeouts: int = 0
+    m_query_native_query_skipped: int = 0
+    # Expressions that reached the parser but are not M-Query at all
+    # (e.g. DAX computed-table expressions, empty strings, label rows).
+    # These fail with MQueryParseError but are expected and logged at INFO.
+    m_query_non_mquery_expressions: int = 0
     m_query_parse_validation_errors: int = 0
     m_query_parse_unexpected_character_errors: int = 0
+    # Genuine M-Query expressions that the parser could not handle.
     m_query_parse_unknown_errors: int = 0
     m_query_resolver_errors: int = 0
     m_query_resolver_no_lineage: int = 0

metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/_bridge.py

@@ -0,0 +1,156 @@
+import gzip
+import json
+import logging
+import threading
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+NodeIdMap = dict[int, dict]
+
+_BUNDLE_PATH = Path(__file__).parent / "mquery_bridge" / "bundle.js.gz"
+
+
+class MQueryBridgeError(RuntimeError):
+    """V8 context error or malformed response from the M-Query bridge."""
+
+    pass
+
+
+class MQueryParseError(RuntimeError):
+    """Parser returned a structured parse error for a specific expression."""
+
+    def __init__(self, message: str, expression: str = "") -> None:
+        super().__init__(message)
+        self.expression = expression
+
+
+class MQueryBridge:
+    def __init__(self) -> None:
+        if not _BUNDLE_PATH.exists():
+            raise ImportError(
+                f"M-Query bridge bundle not found at {_BUNDLE_PATH}. "
+                "Re-installing acryl-datahub[powerbi] may fix this."
+            )
+        try:
+            from py_mini_racer import MiniRacer
+        except ImportError as e:
+            raise ImportError(
+                "PowerBI M-Query parsing requires 'mini-racer'. "
+                "Install it with: pip install 'acryl-datahub[powerbi]'"
+            ) from e
+
+        # Decompress bundle.js.gz in memory — fast (~50ms for 500KB) and happens once per process.
+        try:
+            bundle_js = gzip.decompress(_BUNDLE_PATH.read_bytes()).decode("utf-8")
+        except (gzip.BadGzipFile, OSError, EOFError) as e:
+            raise ImportError(
+                f"M-Query bridge bundle at {_BUNDLE_PATH} appears to be corrupt: {e}. "
+                "Re-installing acryl-datahub[powerbi] may fix this."
+            ) from e
+        self._ctx = MiniRacer()
+        self._ctx.eval(bundle_js)
+
+    def parse(self, expression: str) -> NodeIdMap:
+        """
+        Parse an M-Query expression and return a flat node map.
+
+        Each key is a node ID (int); each value is a node dict with at least
+        ``kind`` (NodeKind string) and ``id``. Child nodes are embedded inline,
+        not as ID references, so you can walk them directly or look up any
+        node by ID via the returned map.
+
+        Example — the LetExpression root for ``let x = 1 in x`` is at the
+        root of the returned dict and looks roughly like::
+
+            {1: {"kind": "LetExpression", "id": 1, "variableList": {...}, ...},
+             2: {"kind": "ArrayWrapper", "id": 2, ...},
+             ...}
+
+        Not thread-safe — callers must be single-threaded.
+
+        Raises:
+            MQueryParseError: parser returned a structured error for this expression.
+            MQueryBridgeError: V8 context error or malformed response.
+        """
+        # JSPromise is available: __init__ already guaranteed py_mini_racer is installed.
+        from py_mini_racer import JSPromise
+
+        try:
+            # parseExpression is async, so ctx.call() returns an unresolved plain dict.
+            # Use ctx.eval() instead, which returns a JSPromise; call .get() to await it.
+            result = self._ctx.eval(f"parseExpression({json.dumps(expression)})")
+            if not isinstance(result, JSPromise):
+                raise MQueryBridgeError(
+                    f"M-Query bridge: expected JSPromise from parseExpression, got {type(result).__name__}"
+                )
+            raw = result.get()
+        except MQueryBridgeError:
+            raise
+        except Exception as e:
+            # Catches all py_mini_racer errors (JSEvalException, JSTimeoutException, etc.)
+            # MiniRacerBaseException is not exported from the top-level namespace in mini-racer.
+            raise MQueryBridgeError(f"M-Query bridge V8 error: {e}") from e
+
+        if not isinstance(raw, str):
+            raise MQueryBridgeError(
+                f"M-Query bridge returned non-string result: {type(raw).__name__}"
+            )
+
+        try:
+            result = json.loads(raw)
+        except (json.JSONDecodeError, TypeError) as e:
+            raise MQueryBridgeError(
+                f"M-Query bridge returned malformed JSON: {e}. Raw: {raw!r}"
+            ) from e
+
+        if not result.get("ok"):
+            raise MQueryParseError(
+                result.get("error", "unknown error"), expression=expression
+            )
+
+        node_id_map = result.get("nodeIdMap")
+        if node_id_map is None:
+            raise MQueryBridgeError(
+                "M-Query bridge returned ok=true but 'nodeIdMap' is missing from response"
+            )
+
+        return {int(node_id): node for node_id, node in node_id_map}
+
+
+_bridge_instance: Optional[MQueryBridge] = None
+_bridge_lock = threading.Lock()
+
+
+def get_bridge() -> MQueryBridge:
+    """Return the process-wide MQueryBridge, creating it on first call."""
+    global _bridge_instance
+    if _bridge_instance is None:
+        with _bridge_lock:
+            if _bridge_instance is None:
+                _bridge_instance = MQueryBridge()
+    return _bridge_instance
+
+
+def _clear_bridge() -> None:
+    """Drop the singleton so the next call to get_bridge() starts fresh.
+
+    Called after a V8 crash to avoid reusing a broken context, and in tests
+    to ensure each test module gets an isolated bridge.
+    """
+    global _bridge_instance
+    with _bridge_lock:
+        if _bridge_instance is not None:
+            # Explicitly close the V8 context before dropping the reference.
+            # If we just set _bridge_instance = None here, Python's GC decides
+            # when to finalize the MiniRacer object. If that happens while other
+            # threads are active (e.g. in a later, unrelated test), MiniRacer's
+            # __del__ -> close() path segfaults. Closing synchronously here, while
+            # the lock is held and no concurrent parse() calls are in flight,
+            # shuts down V8 cleanly.
+            try:
+                _bridge_instance._ctx.close()
+            except Exception:
+                pass
+        _bridge_instance = None