FEATURE: Use MAVFTP to speed up download of parameters values

Also downloads and updates parameter default values

Author: amilcarlucas

MethodicConfigurator/annotate_params.py

@@ -198,7 +198,7 @@ def missionplanner_sort(item: str) -> Tuple[str, ...]:
         return tuple(parts)
 
     @staticmethod
-    def format_params(param_dict: Dict[str, 'Par'], file_format: str = "missionplanner") -> List[str]:
+    def format_params(param_dict: Dict[str, 'Par'], file_format: str = "missionplanner") -> List[str]:  # pylint: disable=too-many-branches
         """
         Formats the parameters in the provided dictionary into a list of strings.
 
@@ -225,17 +225,23 @@ def format_params(param_dict: Dict[str, 'Par'], file_format: str = "missionplann
         formatted_params = []
         if file_format == "missionplanner":
             for key, parameter in param_dict.items():
-                if parameter.comment:
-                    formatted_params.append(f"{key},{format(parameter.value, '.6f').rstrip('0').rstrip('.')}"
-                                            f"  # {parameter.comment}")
+                if isinstance(parameter, dict):
+                    if parameter.comment:
+                        formatted_params.append(f"{key},{format(parameter.value, '.6f').rstrip('0').rstrip('.')}"
+                                                f"  # {parameter.comment}")
+                    else:
+                        formatted_params.append(f"{key},{format(parameter.value, '.6f').rstrip('0').rstrip('.')}")
                 else:
-                    formatted_params.append(f"{key},{format(parameter.value, '.6f').rstrip('0').rstrip('.')}")
+                    formatted_params.append(f"{key},{format(parameter, '.6f').rstrip('0').rstrip('.')}")
         elif file_format == "mavproxy":
             for key, parameter in param_dict.items():
-                if parameter.comment:
-                    formatted_params.append(f"{key:<16} {parameter.value:<8.6f}  # {parameter.comment}")
+                if isinstance(parameter, dict):
+                    if parameter.comment:
+                        formatted_params.append(f"{key:<16} {parameter.value:<8.6f}  # {parameter.comment}")
+                    else:
+                        formatted_params.append(f"{key:<16} {parameter.value:<8.6f}")
                 else:
-                    formatted_params.append(f"{key:<16} {parameter.value:<8.6f}")
+                    formatted_params.append(f"{key:<16} {parameter:<8.6f}")
         return formatted_params
 
     @staticmethod
@@ -690,9 +696,7 @@ def get_xml_url(vehicle_type: str, firmware_version: str) -> str:
 def parse_parameter_metadata(xml_url: str, xml_dir: str, xml_file: str,
                         vehicle_type: str, max_line_length: int) -> Dict[str, Any]:
     xml_root = get_xml_data(xml_url, xml_dir, xml_file)
-    param_default_dict = load_default_param_file(xml_dir)
-    doc_dict = create_doc_dict(xml_root, vehicle_type, max_line_length)
-    return doc_dict, param_default_dict
+    return create_doc_dict(xml_root, vehicle_type, max_line_length)
 
 
 def main():
@@ -701,8 +705,9 @@ def main():
         xml_url = get_xml_url(args.vehicle_type, args.firmware_version)
         xml_dir = get_xml_dir(args.target)
 
-        [doc_dict, param_default_dict] = parse_parameter_metadata(xml_url, xml_dir, PARAM_DEFINITION_XML_FILE,
-                                                                  args.vehicle_type, args.max_line_length)
+        doc_dict = parse_parameter_metadata(xml_url, xml_dir, PARAM_DEFINITION_XML_FILE,
+                                            args.vehicle_type, args.max_line_length)
+        param_default_dict = load_default_param_file(xml_dir)
         update_parameter_documentation(doc_dict, args.target, args.sort, param_default_dict,
                                        args.delete_documentation_annotations)
         if args.verbose:
@@ -711,9 +716,10 @@ def main():
         # Annotate lua MAGfit XML documentation into the respective parameter file
         xml_file = LUA_PARAM_DEFINITION_XML_FILE
         if os_path.isfile(os_path.join(os_path.dirname(args.target), xml_file)):
-            [doc_dict, param_default_dict] = parse_parameter_metadata(xml_url, xml_dir, xml_file,
-                                                                      args.vehicle_type, args.max_line_length)
+            doc_dict = parse_parameter_metadata(xml_url, xml_dir, xml_file,
+                                                args.vehicle_type, args.max_line_length)
             target = os_path.join(os_path.dirname(args.target), "24_inflight_magnetometer_fit_setup.param")
+            param_default_dict = load_default_param_file(xml_dir)
             update_parameter_documentation(doc_dict, target, args.sort, param_default_dict,
                                            args.delete_documentation_annotations)
         else:

MethodicConfigurator/ardupilot_methodic_configurator.py

@@ -116,8 +116,10 @@ def main():
     # Connect to the flight controller and read the parameters
     flight_controller, vehicle_type = connect_to_fc_and_read_parameters(args)
 
+    param_default_values = {}
     if flight_controller.master is not None or args.device == 'test':
-        FlightControllerInfoWindow(flight_controller)
+        fciw = FlightControllerInfoWindow(flight_controller)
+        param_default_values = fciw.get_param_default_values()
 
     try:
         local_filesystem = LocalFilesystem(args.vehicle_dir, vehicle_type, flight_controller.info.flight_sw_version,
@@ -126,6 +128,9 @@ def main():
         show_error_message("Fatal error reading parameter files", f"{exp}")
         raise
 
+    if param_default_values:
+        local_filesystem.write_param_default_values(param_default_values)
+
     # Get the list of intermediate parameter files files that will be processed sequentially
     files = list(local_filesystem.file_parameters.keys()) if local_filesystem.file_parameters else []
 

MethodicConfigurator/backend_filesystem.py

@@ -33,6 +33,7 @@
 from MethodicConfigurator.annotate_params import PARAM_DEFINITION_XML_FILE, Par
 from MethodicConfigurator.annotate_params import get_xml_url
 from MethodicConfigurator.annotate_params import get_xml_dir
+from MethodicConfigurator.annotate_params import load_default_param_file
 from MethodicConfigurator.annotate_params import parse_parameter_metadata
 from MethodicConfigurator.annotate_params import format_columns
 from MethodicConfigurator.annotate_params import split_into_lines
@@ -88,12 +89,12 @@ def __init__(self, vehicle_dir: str, vehicle_type: str, fw_version: str, allow_e
         self.vehicle_type = vehicle_type
         self.fw_version = fw_version
         self.allow_editing_template_files = allow_editing_template_files
+        self.param_default_dict = {}
         if vehicle_dir is not None:
             self.re_init(vehicle_dir, vehicle_type)
 
     def re_init(self, vehicle_dir: str, vehicle_type: str):
         self.vehicle_dir = vehicle_dir
-        self.param_default_dict = {}
         self.doc_dict = {}
 
         if not self.load_vehicle_components_json_data(vehicle_dir):
@@ -122,15 +123,16 @@ def re_init(self, vehicle_dir: str, vehicle_type: str):
         # Read ArduPilot parameter documentation
         xml_url = get_xml_url(vehicle_type, self.fw_version)
         xml_dir = get_xml_dir(vehicle_dir)
-        [self.doc_dict, self.param_default_dict] = parse_parameter_metadata(xml_url, xml_dir, PARAM_DEFINITION_XML_FILE,
-                                                                            vehicle_type, TOOLTIP_MAX_LENGTH)
+        self.doc_dict = parse_parameter_metadata(xml_url, xml_dir, PARAM_DEFINITION_XML_FILE,
+                                                 vehicle_type, TOOLTIP_MAX_LENGTH)
+        self.param_default_dict = load_default_param_file(xml_dir)
 
         # Extend parameter documentation metadata if <parameter_file>.pdef.xml exists
         for filename in self.file_parameters:
             pdef_xml_file = filename.replace(".param", ".pdef.xml")
             if os_path.exists(os_path.join(xml_dir, pdef_xml_file)):
-                [doc_dict, _param_default_dict] = parse_parameter_metadata("", xml_dir, pdef_xml_file,
-                                                                           vehicle_type, TOOLTIP_MAX_LENGTH)
+                doc_dict = parse_parameter_metadata("", xml_dir, pdef_xml_file,
+                                                    vehicle_type, TOOLTIP_MAX_LENGTH)
                 self.doc_dict.update(doc_dict)
 
         self.__extend_and_reformat_parameter_documentation_metadata()
@@ -530,6 +532,17 @@ def copy_fc_params_values_to_template_created_vehicle_files(self, fc_parameters:
             Par.export_to_param(Par.format_params(param_dict), os_path.join(self.vehicle_dir, param_filename))
         return ''
 
+    def write_param_default_values(self, param_default_values: Dict[str, float]):
+        param_default_values = dict(sorted(param_default_values.items()))
+        if self.param_default_dict != param_default_values:
+            self.param_default_dict = param_default_values
+            return True
+        return False
+
+    def write_param_default_values_to_file(self, param_default_values: Dict[str, float], filename: str='00_default.param'):
+        if self.write_param_default_values(param_default_values):
+            Par.export_to_param(Par.format_params(param_default_values), os_path.join(self.vehicle_dir, filename))
+
     @staticmethod
     def add_argparse_arguments(parser):
         parser.add_argument('-t', '--vehicle-type',

MethodicConfigurator/backend_flightcontroller.py

@@ -13,12 +13,13 @@
 from logging import warning as logging_warning
 from logging import error as logging_error
 
-# import sys
+from sys import exit as sys_exit
 from time import sleep as time_sleep
 from os import path as os_path
 from os import name as os_name
 from os import readlink as os_readlink
 from typing import Dict
+from typing import Tuple
 
 import serial.tools.list_ports
 import serial.tools.list_ports_common
@@ -28,6 +29,8 @@
 from MethodicConfigurator.annotate_params import Par
 
 from MethodicConfigurator.backend_flightcontroller_info import BackendFlightcontrollerInfo
+from MethodicConfigurator.backend_mavftp import MAVFTP
+from MethodicConfigurator.param_ftp import ftp_param_decode
 
 from MethodicConfigurator.argparse_check_range import CheckRange
 
@@ -268,12 +271,13 @@ def __process_autopilot_version(self, m):
         self.info.set_vendor_id_and_product_id(m.vendor_id, m.product_id)
         return ""
 
-    def download_params(self, progress_callback=None) -> Dict[str, float]:
+    def download_params(self, progress_callback=None) -> Tuple[Dict[str, float], Dict[str, float]]:
         """
         Requests all flight controller parameters from a MAVLink connection.
 
         Returns:
             Dict[str, float]: A dictionary of flight controller parameters.
+            Dict[str, float]: A dictionary of flight controller default parameters.
         """
         # FIXME this entire if statement is for testing only, remove it later pylint: disable=fixme
         if self.master is None and self.comport is not None and self.comport.device == 'test':
@@ -287,17 +291,13 @@ def download_params(self, progress_callback=None) -> Dict[str, float]:
 
         # Check if MAVFTP is supported
         if self.info.is_mavftp_supported:
-            logging_info("MAVFTP is supported by the %s flight controller, but not yet from this SW",
+            logging_info("MAVFTP is supported by the %s flight controller",
                          self.comport.device)
 
-            # FIXME remove this call and uncomment below once it works pylint: disable=fixme
-            return self.__download_params_via_mavlink(progress_callback)
-
-            # parameters, _defaults = self.download_params_via_mavftp(progress_callback)
-            # return parameters
+            return self.download_params_via_mavftp(progress_callback)
 
         logging_info("MAVFTP is not supported by the %s flight controller, fallback to MAVLink", self.comport.device)
-        return self.__download_params_via_mavlink(progress_callback)
+        return self.__download_params_via_mavlink(progress_callback), {}
 
     def __download_params_via_mavlink(self, progress_callback=None) -> Dict[str, float]:
         logging_debug("Will fetch all parameters from the %s flight controller", self.comport.device)
@@ -332,6 +332,51 @@ def __download_params_via_mavlink(self, progress_callback=None) -> Dict[str, flo
                 break
         return parameters
 
+    def download_params_via_mavftp(self, progress_callback=None) -> Tuple[Dict[str, float], Dict[str, float]]:
+        # FIXME global variables should be avoided but I found no other practical way get the FTP result pylint: disable=fixme
+        global ftp_should_run  # pylint: disable=global-variable-undefined
+        global pdict  # pylint: disable=global-variable-undefined
+        global defdict  # pylint: disable=global-variable-undefined
+        ftp_should_run = True
+        pdict = {}
+        defdict = {}
+
+        mavftp = MAVFTP(self.master,
+                        target_system=self.master.target_system,
+                        target_component=self.master.target_component)
+
+        def callback(fh):
+            '''called on ftp completion'''
+            global ftp_should_run  # pylint: disable=global-variable-undefined
+            global pdict  # pylint: disable=global-variable-not-assigned
+            global defdict  # pylint: disable=global-variable-not-assigned
+            data = fh.read()
+            logging_info("'MAVFTP get' parameter values and defaults done, got %u bytes", len(data))
+            pdata = ftp_param_decode(data)
+            if pdata is None:
+                logging_error("param decode failed")
+                sys_exit(1)
+
+            for (name, value, _ptype) in pdata.params:
+                pdict[name.decode('utf-8')] = value
+            logging_info("got %u parameter values", len(pdict))
+            if pdata.defaults:
+                for (name, value, _ptype) in pdata.defaults:
+                    defdict[name.decode('utf-8')] = value
+                logging_info("got %u parameter default values", len(defdict))
+            ftp_should_run = False
+            progress_callback(len(data), len(data))
+
+        mavftp.cmd_get(['@PARAM/param.pck?withdefaults=1'], callback=callback, callback_progress=progress_callback)
+
+        while ftp_should_run:
+            m = self.master.recv_match(type=['FILE_TRANSFER_PROTOCOL'], timeout=0.1)
+            if m is not None:
+                mavftp.mavlink_packet(m)
+            mavftp.idle_task()
+
+        return pdict, defdict
+
     def set_param(self, param_name: str, param_value: float):
         """
         Set a parameter on the flight controller.