diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 5b646cba9d1..8f0d8f6e571 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -892,6 +892,7 @@ jobs: # Install test requirements pip install -r backends/nxp/requirements-tests-pypi.txt pip install -r backends/nxp/requirements-tests-eiq.txt + PYTHON_EXECUTABLE=python bash examples/nxp/setup.sh # Run pytest PYTHON_EXECUTABLE=python bash backends/nxp/run_unittests.sh diff --git a/backends/nxp/tests/exported_program_vizualize.py b/backends/nxp/tests/exported_program_vizualize.py deleted file mode 100644 index 0f4b8db697c..00000000000 --- a/backends/nxp/tests/exported_program_vizualize.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2024 NXP -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import random - -from gvgen import GvGen -from torch.export import ExportedProgram - - -def exported_program_to_dot( # noqa C901 - exported_program: ExportedProgram, dot_file_name="graph.dot", show_tags=True -): - """ - Generate dot file for tagged exported program. - - :param exported_program: Exported program with optional meta values: 'delegation_tag' and 'cluster'. - :param dot_file_name: Produced .dot file name. - :param show_tags: If True, nodes will be shown as a subcomponent of tag nodes. - """ - graph = GvGen() - - def name_color(string): # pseudo-randomization function - h = hash(string) # hash string and int together - if h < 0: # ensure positive number - h = h * -1 - random.seed(h) # set the seed to use for randomization - r = int(random.random() * 255) - g = int(random.random() * 255) - b = int(random.random() * 255) - return "#%02x%02x%02x" % (r, g, b) - - graph_items = {} - delegation_tags = {} - - # Find tags (parent objects) - for node in exported_program.graph.nodes: - if "delegation_tag" in node.meta and show_tags: - tag = node.meta["delegation_tag"] - if tag not in delegation_tags: - item = graph.newItem(tag) - delegation_tags[tag] = item - - for node in exported_program.graph.nodes: - if "delegation_tag" in node.meta and show_tags: - # Delegated node -> add color - tag = node.meta["delegation_tag"] - item = graph.newItem(node.name, delegation_tags[tag]) - - graph.propertyAppend(item, "fillcolor", name_color(tag)) - graph.propertyAppend(item, "style", "filled") - else: - item = graph.newItem(node.name) - - label = graph.propertyGet(item, "label") - if "cluster" in node.meta: - graph.propertyAppend( - item, "label", label + "\n QDQ Cluster: " + node.meta["cluster"] - ) - - # Change shape of node for (de)quantize and rest of nodes - if any(q in label for q in ["_quantize_per_tensor_", "_quantize_per_channel_"]): - graph.propertyAppend(item, "shape", "invhouse") - elif any( - dq in label - for dq in ["_dequantize_per_tensor_", "_dequantize_per_channel_"] - ): - graph.propertyAppend(item, "shape", "house") - else: - graph.propertyAppend(item, "shape", "box") - - graph_items[node.name] = item - - # Add connections between nodes - for node in exported_program.graph.nodes: - for user in node.users: - link = graph.newLink(graph_items[node.name], graph_items[user.name]) - - label = "" - if "val" in node.meta: - tensor = node.meta["val"] - if isinstance(tensor, tuple): - tensor = tensor[0] # Fake tensor - label = f" ({list(tensor.shape)} | {tensor.dtype})" - - graph.propertyAppend(link, "label", label) - - with open(dot_file_name, "w") as f: - graph.dot(f) diff --git a/devtools/visualization/model_explorer_styles/cluster_highlight_style.json b/devtools/visualization/model_explorer_styles/cluster_highlight_style.json new file mode 100644 index 00000000000..cced07d6a55 --- /dev/null +++ b/devtools/visualization/model_explorer_styles/cluster_highlight_style.json @@ -0,0 +1,236 @@ + [ + { + "queries": [ + { + "type": "node_type", + "nodeType": "op_nodes" + }, + { + "type": "regex", + "queryRegex": "quantize", + "matchTypes": [ + "title" + ] + } + ], + "nodeType": "op_nodes", + "styles": { + "node_bg_color": { + "id": "node_bg_color", + "value": "#dce9e9" + } + }, + "version": "v2" + }, + { + "queries": [ + { + "type": "node_type", + "nodeType": "op_nodes" + }, + { + "type": "regex", + "queryRegex": "aten.", + "matchTypes": [ + "title" + ] + } + ], + "nodeType": "op_nodes", + "styles": { + "node_bg_color": { + "id": "node_bg_color", + "value": "#b4e3f5" + } + }, + "version": "v2" + }, + { + "queries": [ + { + "type": "node_type", + "nodeType": "layer_nodes" + }, + { + "type": "regex", + "queryRegex": "cluster", + "matchTypes": [ + "title" + ] + } + ], + "nodeType": "op_nodes", + "styles": { + "node_bg_color": { + "id": "node_bg_color", + "value": "#d0eae9" + }, + "node_border_color": { + "id": "node_border_color", + "value": "#ffffff" + } + }, + "version": "v2" + }, + { + "queries": [ + { + "type": "node_type", + "nodeType": "layer_nodes" + }, + { + "type": "regex", + "queryRegex": "partition 0", + "matchTypes": [ + "title" + ] + } + ], + "nodeType": "op_nodes", + "styles": { + "node_bg_color": { + "id": "node_bg_color", + "value": "#fff1d5" + } + }, + "version": "v2" + }, + { + "queries": [ + { + "type": "node_type", + "nodeType": "layer_nodes" + }, + { + "type": "regex", + "queryRegex": "partition 1", + "matchTypes": [ + "title" + ] + } + ], + "nodeType": "op_nodes", + "styles": { + "node_bg_color": { + "id": "node_bg_color", + "value": "#fdffcc" + } + }, + "version": "v2" + }, + { + "queries": [ + { + "type": "node_type", + "nodeType": "layer_nodes" + }, + { + "type": "regex", + "queryRegex": "partition 2", + "matchTypes": [ + "title" + ] + } + ], + "nodeType": "op_nodes", + "styles": { + "node_bg_color": { + "id": "node_bg_color", + "value": "#ccffcc" + } + }, + "version": "v2" + }, + { + "queries": [ + { + "type": "node_type", + "nodeType": "layer_nodes" + }, + { + "type": "regex", + "queryRegex": "partition 3", + "matchTypes": [ + "title" + ] + } + ], + "nodeType": "op_nodes", + "styles": { + "node_bg_color": { + "id": "node_bg_color", + "value": "#ccffff" + } + }, + "version": "v2" + }, + { + "queries": [ + { + "type": "node_type", + "nodeType": "layer_nodes" + }, + { + "type": "regex", + "queryRegex": "partition 4", + "matchTypes": [ + "title" + ] + } + ], + "nodeType": "op_nodes", + "styles": { + "node_bg_color": { + "id": "node_bg_color", + "value": "#ffc6e2" + } + }, + "version": "v2" + }, + { + "queries": [ + { + "type": "node_type", + "nodeType": "layer_nodes" + }, + { + "type": "regex", + "queryRegex": "partition 5", + "matchTypes": [ + "title" + ] + } + ], + "nodeType": "op_nodes", + "styles": { + "node_bg_color": { + "id": "node_bg_color", + "value": "#ffcaff" + } + }, + "version": "v2" + }, + { + "queries": [ + { + "type": "node_type", + "nodeType": "layer_nodes" + }, + { + "type": "regex", + "queryRegex": "partition 6", + "matchTypes": [ + "title" + ] + } + ], + "nodeType": "op_nodes", + "styles": { + "node_bg_color": { + "id": "node_bg_color", + "value": "#d7d7ff" + } + }, + "version": "v2" + } +] \ No newline at end of file diff --git a/devtools/visualization/visualization_utils.py b/devtools/visualization/visualization_utils.py index b21a953f4d2..6dd0c327048 100644 --- a/devtools/visualization/visualization_utils.py +++ b/devtools/visualization/visualization_utils.py @@ -1,21 +1,29 @@ # Copyright 2025 Arm Limited and/or its affiliates. +# Copyright 2025 NXP # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. - +import json import subprocess import time from typing import Any, Callable, Type -from executorch.exir import EdgeProgramManager, ExecutorchProgramManager -from executorch.exir.program._program import _update_exported_program_graph_module +from executorch.exir import EdgeProgramManager, ExecutorchProgramManager # type: ignore +from executorch.exir.program._program import ( # type: ignore + _update_exported_program_graph_module, +) + from torch._export.verifier import Verifier -from torch.export.exported_program import ExportedProgram -from torch.fx import GraphModule +from torch.export.exported_program import ExportedProgram # type: ignore +from torch.fx import GraphModule, Node # type: ignore try: from model_explorer import config, consts, visualize_from_config # type: ignore + from model_explorer.config import ModelExplorerConfig # type: ignore + from model_explorer.pytorch_exported_program_adater_impl import ( # type: ignore + PytorchExportedProgramAdapterImpl, + ) except ImportError: print( "Error: 'model_explorer' is not installed. Install using devtools/install_requirement.sh" @@ -139,6 +147,134 @@ def visualize_model_explorer( ) +def _save_model_as_json(cur_config: ModelExplorerConfig, file_name: str): + """Save the graphs stored in the `cur_config` in JSON format, which can be loaded by the Model Explorer GUI. + + :param cur_config: ModelExplorerConfig containing the graph for visualization. + :param file_name: Name of the JSON file for storage. + """ + # Extract the graphs from the config file. + graphs_list = json.loads(cur_config.get_transferrable_data()["graphs_list"]) + graphs = json.loads(graphs_list[0])["graphs"] + + # The returned dictionary is missing the `collectionLabel` entry. Add it manually. + for graph in graphs: + graph["collectionLabel"] = "Executorch" + + # Create the JSON according to the structure required by the Model Explorer GUI. + json_data = { + "label": "Executorch", + "graphs": graphs, + "graphsWithLevel": [ + {"graph": graph, "level": level} for level, graph in enumerate(graphs) + ], + } + + # Store the JSON. + with open(file_name, "w") as f: + json.dump(json_data, f) + + +def visualize_with_clusters( + exported_program: ExportedProgram, + json_file_name: str | None = None, + reuse_server: bool = False, + get_node_partition_name: Callable[[Node], str | None] = lambda node: node.meta.get( + "delegation_tag", None + ), + get_node_qdq_cluster_name: Callable[ + [Node], str | None + ] = lambda node: node.meta.get("cluster", None), + **kwargs, +): + """Visualize exported programs using the Model Explorer. The QDQ clusters and individual partitions are highlighted. + + To install the Model Explorer, run `devtools/install_requirements.sh`. + To display a stored json file, first launch the Model Explorer server by running `model-explorer`, and then + use the GUI to open the json. + + NOTE: FireFox seems to have issues rendering the graphs. Other browsers seem to work well. + + :param exported_program: Program to visualize. + :param json_file_name: If not None, a JSON of the visualization will be stored in the provided file. The JSON can + then be opened in the Model Explorer GUI later. + If None, a Model Explorer instance will be launched with the model visualization. + :param reuse_server: If True, an existing instance of the Model Explorer server will be used (if one exists). + Otherwise, a new instance at a separate port will start. + :param get_node_partition_name: Function which takes a `Node` and returns a string with the name of the partition + the `Node` belongs to, or `None` if it has no partition. + :param get_node_qdq_cluster_name: Function which takes a `Node` and returns a string with the name of the QDQ + cluster the `Node` belongs to, or `None` if it has no cluster. + :param kwargs: Additional kwargs for the `visualize_from_config()` function. + """ + + cur_config = config() + + # Create a Model Explorer graph from the `exported_program`. + adapter = PytorchExportedProgramAdapterImpl( + exported_program, consts.DEFAULT_SETTINGS + ) + graphs = adapter.convert() + + nodes = list(exported_program.graph.nodes) + explorer_nodes = graphs["graphs"][0].nodes + + # Highlight QDQ clusters and individual partitions. + known_partition_names = [] + for explorer_node, node in zip(explorer_nodes, nodes, strict=True): + # Generate the `namespace` for the node, which will determine node grouping in the visualizer. + # The character "/" is used as a divider when a node has multiple namespaces. + namespace = "" + + if (partition_name := get_node_partition_name(node)) is not None: + # If the nodes are tagged by the partitioner, highlight the tagged groups. + + # Create a custom naming for the partitions ("partition " where `i` = 0, 1, 2, ...). + if partition_name not in known_partition_names: + known_partition_names.append(partition_name) + partition_id = known_partition_names.index(partition_name) + + safe_partition_name = partition_name.replace( + "/", ":" + ) # Avoid using unwanted "/". + namespace += f"partition {partition_id} ({safe_partition_name})" + + if (cluster_name := get_node_qdq_cluster_name(node)) is not None: + # Highlight the QDQ cluster. + + # Add a separator, in case the namespace already contains the `partition`. + if len(namespace) != 0: + namespace += "/" + + # Create a custom naming for the clusters ("cluster ()"). + safe_cluster_name = cluster_name.replace( + "/", ":" + ) # Avoid using unwanted "/". + namespace += f"cluster ({safe_cluster_name})" + + explorer_node.namespace = namespace + + # Store the modified graph in the config. + graphs_index = len(cur_config.graphs_list) + cur_config.graphs_list.append(graphs) + name = "Executorch" + model_source: config.ModelSource = {"url": f"graphs://{name}/{graphs_index}"} + cur_config.model_sources.append(model_source) + + if json_file_name is not None: + # Just save the visualization. + _save_model_as_json(cur_config, json_file_name) + + else: + # Start the ModelExplorer server, and visualize the graph in the browser. + if reuse_server: + cur_config.set_reuse_server() + visualize_from_config( + cur_config, + **kwargs, + ) + + def visualize_graph( graph_module: GraphModule, exported_program: ExportedProgram | EdgeProgramManager | ExecutorchProgramManager, diff --git a/docs/source/_static/img/visualization/1.png b/docs/source/_static/img/visualization/1.png new file mode 100644 index 00000000000..9d76c793492 Binary files /dev/null and b/docs/source/_static/img/visualization/1.png differ diff --git a/docs/source/_static/img/visualization/2.png b/docs/source/_static/img/visualization/2.png new file mode 100644 index 00000000000..0efe1fe8555 Binary files /dev/null and b/docs/source/_static/img/visualization/2.png differ diff --git a/docs/source/_static/img/visualization/3.png b/docs/source/_static/img/visualization/3.png new file mode 100644 index 00000000000..18d45bc4412 Binary files /dev/null and b/docs/source/_static/img/visualization/3.png differ diff --git a/docs/source/_static/img/visualization/4.png b/docs/source/_static/img/visualization/4.png new file mode 100644 index 00000000000..9e20a92d962 Binary files /dev/null and b/docs/source/_static/img/visualization/4.png differ diff --git a/docs/source/_static/img/visualization/5.png b/docs/source/_static/img/visualization/5.png new file mode 100644 index 00000000000..08becaee177 Binary files /dev/null and b/docs/source/_static/img/visualization/5.png differ diff --git a/docs/source/_static/img/visualization/6.png b/docs/source/_static/img/visualization/6.png new file mode 100644 index 00000000000..342b47bc415 Binary files /dev/null and b/docs/source/_static/img/visualization/6.png differ diff --git a/docs/source/_static/img/visualization/visualize_with_clusters_example.png b/docs/source/_static/img/visualization/visualize_with_clusters_example.png new file mode 100644 index 00000000000..938ae24ae48 Binary files /dev/null and b/docs/source/_static/img/visualization/visualize_with_clusters_example.png differ diff --git a/docs/source/devtools-overview.md b/docs/source/devtools-overview.md index 8e13e67f1a1..ac797252daf 100644 --- a/docs/source/devtools-overview.md +++ b/docs/source/devtools-overview.md @@ -17,7 +17,7 @@ The ExecuTorch Developer Tools support the following features: - **Debugging** - Intermediate outputs and output quality analysis - **Numerical Discrepancy Detection** - Operator-level numerical discrepancy detection between AOT and runtime intermediate outputs to streamline numerical debugging and validation. - **Memory Allocation Insights** - Visualize how memory is planned, where all the live tensors are at any point in time -- **Visualization** - Coming soon +- **Visualization** - Visualize the model as a computational graph (see more [here](visualize.md)) ## Fundamental components of the Developer Tools diff --git a/docs/source/tools-section.md b/docs/source/tools-section.md index 461a1f6849a..c54b4933c44 100644 --- a/docs/source/tools-section.md +++ b/docs/source/tools-section.md @@ -13,6 +13,7 @@ In this section, explore ExecuTorch's comprehensive developer tools for profilin - {doc}`model-inspector` — Model Inspector - {doc}`memory-planning-inspection` — Memory Planning Inspection - {doc}`devtools-tutorial` — Development Utilities +- {doc}`visualization` — Model Visualization ```{toctree} :hidden: @@ -28,3 +29,4 @@ model-debugging model-inspector memory-planning-inspection devtools-tutorial +visualization diff --git a/docs/source/visualize.md b/docs/source/visualize.md new file mode 100644 index 00000000000..fdd868df4f0 --- /dev/null +++ b/docs/source/visualize.md @@ -0,0 +1,144 @@ +# Visualize a Model using ModelExplorer + +The [visualization_utils.py](../../devtools/visualization/visualization_utils.py) contains functions for +visualizing ExecuTorch models as computational graphs using the `ModelExplorer` utility. + +## Installation + +To install the `ModelExplorer` and its dependencies, run: + +``` +./devtools/install_requirements.sh +``` + +## Visualize a model + +The function `visualize()` takes an `ExportedProgram` and launches a `ModelExplorer` server instance. A browser tab will +open, containing the visualization. + +The operations in the graph will be grouped together into collapsable nodes, based on which `nn.Module` instances they +originate from (see **Figure 1**). These nodes can be expanded by clicking the button in their top +left corner, as shown +in **Figure 2**. The model can contain an entire hierarchy of collapsable nodes, reflecting its +original _PyTorch_ +implementation (see **Figure 3**). + +
+ +
Figure 1: Model visualization collapsed into a single node representing the original module.
+
+ +
+ +
Figure 2: Button to expand a node.
+
+ +
+ +
Figure 3: Hierarchy of expandable nodes.
+
+ +The **Model Explorer GUI** provides a button in the top left corner of the screen (see **Figure 4 +**), +which expands all the nested expandable nodes. The result will display all the low-level operations, surrounded by +rectangles which indicate their membership to specific `nn.Module` instances. + +
+ +
Figure 4: Expand all nodes.
+
+ + +Sometimes, it is not ideal to view the model like this. Focusing on visualizing the origin of the final nodes can make +it harder to see the flow of data in the graph. For this purpose, a button in the top left corner can flatten all the +layers (expandable nodes), effectively hiding the original `nn.Module` instances and just displaying the model as a +computational graph (see **Figure 5**). + +
+ +
Figure 5: Flatten the model to a simple computational graph.
+
+ +--- + +# Visualize a Model with Highlighted QDQ Clusters and Partitions + +The [visualization_utils.py](../../devtools/visualization/visualization_utils.py) contains the function +`visualize_with_clusters()` which takes an `ExportedProgram` and visualizes it using the `ModelExplorer` utility. +It groups QDQ clusters and individual partitions together to improve readability. Example usage is available +in [examples/nxp/aot_neutron_compile.py](../../examples/nxp/aot_neutron_compile.py). + +An example of the visualization is shown in **Figure 6.** +
+ +
Figure 6: Example of the QDQ cluster and partition highlighting visualization.
+
+ +## Usage + +There are two main use cases for the visualization: + +### 1. Launching the `ModelExplorer` and Visualizing the Model Immediately + +Call: + +```python +visualize_with_clusters(exported_program) +``` + +This starts a `ModelExplorer` server and opens a browser tab with the visualization. + +By default, each call starts a new server instance and opens a new browser tab. +To reuse an existing server, set the `reuse_server` parameter to `True`. + +Starting the server is **blocking**, so the rest of your script will not run. + +### 2. Storing a Serialized Graph and Visualizing Later (Non-blocking) + +To save the visualization to a JSON file, call: + +```python +visualize_with_clusters(exported_program, "my_model.json") +``` + +This just saves the visualization in the file, and it does **not** start the `ModelExplorer` server. You can then open +the file in the `ModelExplorer` GUI at any point. To launch the server, run: + +```bash + model-explorer [model-file-json] +``` + +If the `model-file-json` is provided, the `ModelExplorer` will open the model visualization. Otherwise, the +`ModelBuilder` GUI home page will appear. In that case, click **Select from your computer**, choose the JSON file, +and then click **View selected models** to display the graph. + +--- + +## Styling the Graph + +`visualize_with_clusters()` supports custom grouping of nodes into QDQ clusters and partitions. + +You can pass the following optional parameters: + +- `get_node_partition_name` +- `get_node_qdq_cluster_name` + +These are functions that take a node and return a string identifying the partition or cluster it belongs to. +Nodes with the same partition/cluster string will be grouped together and labeled accordingly in the visualization. + +### Load a predefined style for QDQ cluster and partition highlighting. + +A color style for the QDQ cluster and partition highlighting is already provided +in [devtools/visualization/model_explorer_styles/cluster_highlight_style.json](../../devtools/visualization/model_explorer_styles/cluster_highlight_style.json). +To load it follow these steps: + +1. Click the **palette icon** in the top-right corner of the `ModelExplorer` interface. +2. Click **Import rules**. +3. Select + the [cluster_highlight_style.json](../../devtools/visualization/model_explorer_styles/cluster_highlight_style.json) + file to apply predefined styles that highlight each partition in a different color. + +
+ +
Figure 7: Add custom color styling to the graph.
+
diff --git a/examples/nxp/aot_neutron_compile.py b/examples/nxp/aot_neutron_compile.py index cb23f99a54d..0708434551f 100644 --- a/examples/nxp/aot_neutron_compile.py +++ b/examples/nxp/aot_neutron_compile.py @@ -21,6 +21,9 @@ from executorch.backends.nxp.neutron_partitioner import NeutronPartitioner from executorch.backends.nxp.nxp_backend import generate_neutron_compile_spec from executorch.backends.nxp.quantizer.neutron_quantizer import NeutronQuantizer +from executorch.devtools.visualization.visualization_utils import ( + visualize_with_clusters, +) from executorch.examples.models import MODEL_NAME_TO_MODEL from executorch.examples.models.model_factory import EagerModelFactory from executorch.exir import ( @@ -210,6 +213,12 @@ def _get_batch_size(data): nargs="*", help="List of operators not to delegate. E.g., --operators_not_to_delegate aten::convolution aten::mm", ) + parser.add_argument( + "--visualize", + choices=["show", "store"], + help="Visualize the lowered program. `show` launches a browser tab with the visualization. `store` stores the " + "visualization in a json file for later inspection. See `docs/source/visualize-with-clusters.md` for details.", + ) args = parser.parse_args() @@ -237,7 +246,7 @@ def _get_batch_size(data): module = post_training_quantize(module, calibration_inputs) if args.so_library is not None: - logging.debug(f"Loading libraries: {args.so_library} and {args.portable_lib}") + logging.debug(f"Loading libraries: {args.so_library}") torch.ops.load_library(args.so_library) if args.test: @@ -284,7 +293,7 @@ def _get_batch_size(data): raise RuntimeError( e.args[0] + ".\nThis likely due to an external so library not being loaded. Supply a path to it with the " - "--portable_lib flag." + "--so_library flag." ).with_traceback(e.__traceback__) from None else: raise e @@ -301,3 +310,13 @@ def executorch_program_to_str(ep, verbose=False): "_nxp_delegate" if args.delegate is True else "" ) save_pte_program(exec_prog, model_name) + + # 7. Optionally visualize the model. + if args.visualize == "show": + visualize_with_clusters(exec_prog.exported_program()) + elif args.visualize == "store": + file_name = f"{args.model_name}-visualization.json" + logging.info( + f"Saved the graph visualization in `{file_name}`. It can be opened using the ModelExplorer." + ) + visualize_with_clusters(exec_prog.exported_program(), file_name) diff --git a/examples/nxp/setup.sh b/examples/nxp/setup.sh index 5e85ed4edc5..b1ded16e1cf 100755 --- a/examples/nxp/setup.sh +++ b/examples/nxp/setup.sh @@ -8,3 +8,9 @@ set -u # Install neutron-converter pip install --index-url https://eiq.nxp.com/repository neutron_converter_SDK_25_09 + +# Get the directory of the current script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Install the required visualization dependencies. +"${SCRIPT_DIR}/../../devtools/install_requirements.sh"