From 94570e7facdfd83df267069442d1ba96b6f73634 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 5 Sep 2025 19:55:43 +0200 Subject: [PATCH 01/13] Refine DSP component protection logic Updated DSP component protection logic to allow removal if 'dsp' is explicitly ignored. --- builder/frameworks/component_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 90a0e5433..1bad67501 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -927,8 +927,8 @@ def _remove_ignored_lib_includes(self) -> None: self.logger.log_change(f"Protected BT library: {lib_name}") continue - # Hard protection for DSP components - if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r']: + # Protection for DSP components, remove DSP components only if 'dsp' is explicitly in lib_ignore + if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r'] and 'dsp' not in [entry.lower() for entry in self.ignored_libs]: self.logger.log_change(f"Protected DSP component: {lib_name}") continue From 2d8c9f630921be021a2cf2cec65db92199638d8d Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 5 Sep 2025 19:58:01 +0200 Subject: [PATCH 02/13] Add 'dsp' to platformio.ini to lib_ignore --- examples/arduino-rmt-blink/platformio.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/arduino-rmt-blink/platformio.ini b/examples/arduino-rmt-blink/platformio.ini index 62842d7da..d1330643c 100644 --- a/examples/arduino-rmt-blink/platformio.ini +++ b/examples/arduino-rmt-blink/platformio.ini @@ -11,6 +11,7 @@ lib_ignore = ble Zigbee Matter OpenThread + dsp build_flags = -DBUILTIN_RGBLED_PIN=18 -DNR_OF_LEDS=1 @@ -27,6 +28,7 @@ lib_ignore = ble Zigbee Matter OpenThread + dsp build_flags = -DBUILTIN_RGBLED_PIN=48 -DNR_OF_LEDS=1 @@ -43,6 +45,7 @@ lib_ignore = ble Zigbee Matter OpenThread + dsp build_flags = -DBUILTIN_RGBLED_PIN=8 -DNR_OF_LEDS=1 @@ -59,6 +62,7 @@ lib_ignore = ble Zigbee Matter OpenThread + dsp build_flags = -DBUILTIN_RGBLED_PIN=27 -DNR_OF_LEDS=1 @@ -75,5 +79,6 @@ lib_ignore = ble Zigbee Matter OpenThread + dsp build_flags = -DBUILTIN_RGBLED_PIN=8 -DNR_OF_LEDS=1 From f1d03b80bf437b18be767ce64e414f2e89322873 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 6 Sep 2025 21:51:51 +0200 Subject: [PATCH 03/13] Fix direct mapping and enhance DSP protection logic --- builder/frameworks/component_manager.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 1bad67501..bd42e62a1 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -885,7 +885,9 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: direct_mapping = { 'ble': 'bt', 'bluetooth': 'bt', - 'bluetoothserial': 'bt' + 'bluetoothserial': 'bt', + 'dsp': 'esp_dsp', + 'esp-dsp': 'esp_dsp' } if cleaned_name in direct_mapping: @@ -927,10 +929,11 @@ def _remove_ignored_lib_includes(self) -> None: self.logger.log_change(f"Protected BT library: {lib_name}") continue - # Protection for DSP components, remove DSP components only if 'dsp' is explicitly in lib_ignore - if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r'] and 'dsp' not in [entry.lower() for entry in self.ignored_libs]: - self.logger.log_change(f"Protected DSP component: {lib_name}") - continue + for lib_name in self.ignored_libs: + if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r']: + if 'dsp' not in [entry.lower() for entry in original_lib_ignore]: + self.logger.log_change(f"Protected DSP component: {lib_name}") + continue # Multiple patterns to catch different include formats patterns = [ From cad8e74fd8b51f6059b6d8ea894fc97204753902 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 6 Sep 2025 22:03:36 +0200 Subject: [PATCH 04/13] Implement _get_original_lib_ignore_entries method Add method to retrieve original lib_ignore entries. --- builder/frameworks/component_manager.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index bd42e62a1..a1f16a994 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -837,7 +837,17 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: # Fallback: Use directory name as include path return dir_name_lower - + + def _get_original_lib_ignore_entries(self) -> List[str]: + """Get original lib_ignore entries without conversion.""" + try: + lib_ignore = self.config.env.GetProjectOption("lib_ignore", []) + if isinstance(lib_ignore, str): + lib_ignore = [lib_ignore] + return [str(entry).strip().lower() for entry in lib_ignore if str(entry).strip()] + except Exception: + return [] + def _convert_lib_name_to_include(self, lib_name: str) -> str: """ Convert library name to potential include directory name. From ce0527926ee160cd20e99984d9e9e809abfbb5fc Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 6 Sep 2025 22:17:26 +0200 Subject: [PATCH 05/13] Fix indentation in _get_original_lib_ignore_entries method --- builder/frameworks/component_manager.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index a1f16a994..338de7a1f 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -839,14 +839,14 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: return dir_name_lower def _get_original_lib_ignore_entries(self) -> List[str]: - """Get original lib_ignore entries without conversion.""" - try: - lib_ignore = self.config.env.GetProjectOption("lib_ignore", []) - if isinstance(lib_ignore, str): - lib_ignore = [lib_ignore] - return [str(entry).strip().lower() for entry in lib_ignore if str(entry).strip()] - except Exception: - return [] + """Get original lib_ignore entries without conversion.""" + try: + lib_ignore = self.config.env.GetProjectOption("lib_ignore", []) + if isinstance(lib_ignore, str): + lib_ignore = [lib_ignore] + return [str(entry).strip().lower() for entry in lib_ignore if str(entry).strip()] + except Exception: + return [] def _convert_lib_name_to_include(self, lib_name: str) -> str: """ From 4a6f6c864ea74e6d186bf859ef94f5820da5436f Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 6 Sep 2025 22:45:55 +0200 Subject: [PATCH 06/13] Refactor _remove_ignored_lib_includes method Refactor _remove_ignored_lib_includes method for clarity and consistency. Improve logging and maintain structure in the component manager. --- builder/frameworks/component_manager.py | 81 ++++++++++++++----------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 338de7a1f..b92d05051 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -908,43 +908,46 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: def _remove_ignored_lib_includes(self) -> None: """ Remove include entries for ignored libraries from pioarduino-build.py. - + Processes the Arduino build script to remove CPPPATH entries for all ignored libraries. Implements protection for BT/BLE and DSP components when dependencies are detected. Uses multiple regex patterns to catch different include path formats. """ build_py_path = str(Path(self.config.arduino_libs_mcu) / "pioarduino-build.py") - + if not os.path.exists(build_py_path): self.logger.log_change("Build file not found") return - + # Check if BT/BLE dependencies exist in lib_deps bt_ble_protected = self._has_bt_ble_dependencies() if bt_ble_protected: self.logger.log_change("BT/BLE protection enabled") - + + # Get original lib_ignore entries for DSP protection check + original_lib_ignore = self._get_original_lib_ignore_entries() + try: with open(build_py_path, 'r', encoding='utf-8') as f: content = f.read() - + original_content = content total_removed = 0 - + # Remove CPPPATH entries for each ignored library for lib_name in self.ignored_libs: # Skip BT-related libraries if BT/BLE dependencies are present if bt_ble_protected and self._is_bt_related_library(lib_name): self.logger.log_change(f"Protected BT library: {lib_name}") continue - - for lib_name in self.ignored_libs: - if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r']: - if 'dsp' not in [entry.lower() for entry in original_lib_ignore]: - self.logger.log_change(f"Protected DSP component: {lib_name}") - continue - + + # Protection for DSP components - only remove if 'dsp' is explicitly in lib_ignore + if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r', 'espressif__esp-dsp']: + if 'dsp' not in original_lib_ignore: + self.logger.log_change(f"Protected DSP component: {lib_name}") + continue + # Multiple patterns to catch different include formats patterns = [ rf'.*join\([^,]*,\s*"include",\s*"{re.escape(lib_name)}"[^)]*\),?\n', @@ -959,71 +962,77 @@ def _remove_ignored_lib_includes(self) -> None: rf'.*Path\([^)]*\)\s*/\s*"include"\s*/\s*"{re.escape(lib_name)}"[^,\n]*,?\n', rf'.*Path\([^)]*{re.escape(lib_name)}[^)]*\)\s*/\s*"include"[^,\n]*,?\n' ] - + removed_count = 0 for pattern in patterns: matches = re.findall(pattern, content) if matches: content = re.sub(pattern, '', content) removed_count += len(matches) - + if removed_count > 0: self.logger.log_change(f"Ignored library: {lib_name} ({removed_count} entries)") total_removed += removed_count - + # Clean up empty lines and trailing commas content = re.sub(r'\n\s*\n', '\n', content) content = re.sub(r',\s*\n\s*\]', '\n]', content) - + # Validate and write changes if self._validate_changes(original_content, content) and content != original_content: with open(build_py_path, 'w', encoding='utf-8') as f: f.write(content) self.logger.log_change(f"Updated build file ({total_removed} total removals)") - + except (IOError, OSError) as e: self.logger.log_change(f"Error processing libraries: {str(e)}") except Exception as e: self.logger.log_change(f"Unexpected error processing libraries: {str(e)}") - + def _validate_changes(self, original_content: str, new_content: str) -> bool: """ Validate that the changes are reasonable and safe. - + Performs sanity checks on the modified content to ensure that the changes don't remove too much content or create invalid modifications that could break the build process. - + Args: original_content: Original file content before modifications - new_content: Modified file content after processing - + new_content: Modified file content after changes + Returns: - True if changes are within acceptable limits and safe to apply + True if changes are valid and safe to apply """ - original_lines = len(original_content.splitlines()) - new_lines = len(new_content.splitlines()) - removed_lines = original_lines - new_lines - - # Don't allow removing more than 50% of the file or negative changes - return not (removed_lines > original_lines * 0.5 or removed_lines < 0) + # Check if too much content was removed (more than 50% indicates potential error) + if len(new_content) < len(original_content) * 0.5: + self.logger.log_change("Warning: Too much content removed, skipping changes") + return False + + # Check for basic Python syntax structure preservation + if 'CPPPATH' not in new_content or 'env.Append' not in new_content: + self.logger.log_change("Warning: Critical build structure missing, skipping changes") + return False + return True + def _backup_pioarduino_build_py(self) -> None: """ Create backup of the original pioarduino-build.py file. - - Creates a backup copy of the Arduino build script before making - modifications. Only operates when Arduino framework is active - and uses MCU-specific backup naming to avoid conflicts. + + Creates a backup of the Arduino framework's build script before + making modifications. Only operates when Arduino framework is active + and creates MCU-specific backup names to avoid conflicts. """ if "arduino" not in self.config.env.subst("$PIOFRAMEWORK"): return - + if not self.config.arduino_libs_mcu: return + build_py_path = str(Path(self.config.arduino_libs_mcu) / "pioarduino-build.py") backup_path = str(Path(self.config.arduino_libs_mcu) / f"pioarduino-build.py.{self.config.mcu}") - + if os.path.exists(build_py_path) and not os.path.exists(backup_path): shutil.copy2(build_py_path, backup_path) From 733578c3e0aa4f8ce01f3d643a509951d9f14eda Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sun, 7 Sep 2025 00:20:17 +0200 Subject: [PATCH 07/13] Disable DSP component protection checks Comment out DSP component protection logic. --- builder/frameworks/component_manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index b92d05051..9bee3a1e6 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -942,11 +942,11 @@ def _remove_ignored_lib_includes(self) -> None: self.logger.log_change(f"Protected BT library: {lib_name}") continue - # Protection for DSP components - only remove if 'dsp' is explicitly in lib_ignore - if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r', 'espressif__esp-dsp']: - if 'dsp' not in original_lib_ignore: - self.logger.log_change(f"Protected DSP component: {lib_name}") - continue +# # Protection for DSP components - only remove if 'dsp' is explicitly in lib_ignore +# if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r', 'espressif__esp-dsp']: +# if 'dsp' not in original_lib_ignore: +# self.logger.log_change(f"Protected DSP component: {lib_name}") +# continue # Multiple patterns to catch different include formats patterns = [ From c75c5f1555e1fbc7103a847a33cd27b6eb5fe9b1 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sun, 7 Sep 2025 14:19:55 +0200 Subject: [PATCH 08/13] Add DSP related mappings to component manager --- builder/frameworks/component_manager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 9bee3a1e6..6f3c3b12c 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -824,7 +824,11 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: 'tcpip': 'lwip', 'usb': 'arduino_tinyusb', 'tinyusb': 'arduino_tinyusb', - 'arduino_tinyusb': 'arduino_tinyusb' + 'dsp': 'espressif__esp-dsp', + 'esp_dsp': 'espressif__esp-dsp', + 'dsps': 'espressif__esp-dsp', + 'fft2r': 'espressif__esp-dsp', + 'dsps_fft2r': 'espressif__esp-dsp' } # Check extended mapping first From 0883ef9d961fa54600f93ebc1afebab2b7675bec Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sun, 7 Sep 2025 14:45:51 +0200 Subject: [PATCH 09/13] Enhance library mapping and ignore handling --- builder/frameworks/component_manager.py | 99 +++++++++++-------------- 1 file changed, 43 insertions(+), 56 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 6f3c3b12c..66e242d67 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -770,6 +770,8 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: # Arduino Core specific mappings (safe mappings that don't conflict with critical components) 'esp32blearduino': 'bt', 'esp32_ble_arduino': 'bt', + 'simpleble': 'bt', + 'esp-nimble-cpp': 'bt', 'esp32': 'esp32', 'wire': 'driver', 'spi': 'driver', @@ -841,17 +843,7 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: # Fallback: Use directory name as include path return dir_name_lower - - def _get_original_lib_ignore_entries(self) -> List[str]: - """Get original lib_ignore entries without conversion.""" - try: - lib_ignore = self.config.env.GetProjectOption("lib_ignore", []) - if isinstance(lib_ignore, str): - lib_ignore = [lib_ignore] - return [str(entry).strip().lower() for entry in lib_ignore if str(entry).strip()] - except Exception: - return [] - + def _convert_lib_name_to_include(self, lib_name: str) -> str: """ Convert library name to potential include directory name. @@ -900,8 +892,13 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: 'ble': 'bt', 'bluetooth': 'bt', 'bluetoothserial': 'bt', - 'dsp': 'esp_dsp', - 'esp-dsp': 'esp_dsp' + 'simpleble': 'bt', + 'esp-nimble-cpp': 'bt', + 'dsp': 'espressif__esp-dsp', + 'esp_dsp': 'espressif__esp-dsp', + 'dsps': 'espressif__esp-dsp', + 'fft2r': 'espressif__esp-dsp', + 'dsps_fft2r': 'espressif__esp-dsp' } if cleaned_name in direct_mapping: @@ -912,46 +909,42 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: def _remove_ignored_lib_includes(self) -> None: """ Remove include entries for ignored libraries from pioarduino-build.py. - + Processes the Arduino build script to remove CPPPATH entries for all ignored libraries. Implements protection for BT/BLE and DSP components when dependencies are detected. Uses multiple regex patterns to catch different include path formats. """ build_py_path = str(Path(self.config.arduino_libs_mcu) / "pioarduino-build.py") - + if not os.path.exists(build_py_path): self.logger.log_change("Build file not found") return - + # Check if BT/BLE dependencies exist in lib_deps bt_ble_protected = self._has_bt_ble_dependencies() if bt_ble_protected: self.logger.log_change("BT/BLE protection enabled") - - # Get original lib_ignore entries for DSP protection check - original_lib_ignore = self._get_original_lib_ignore_entries() - + try: with open(build_py_path, 'r', encoding='utf-8') as f: content = f.read() - + original_content = content total_removed = 0 - + # Remove CPPPATH entries for each ignored library for lib_name in self.ignored_libs: # Skip BT-related libraries if BT/BLE dependencies are present if bt_ble_protected and self._is_bt_related_library(lib_name): self.logger.log_change(f"Protected BT library: {lib_name}") continue - -# # Protection for DSP components - only remove if 'dsp' is explicitly in lib_ignore -# if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r', 'espressif__esp-dsp']: -# if 'dsp' not in original_lib_ignore: -# self.logger.log_change(f"Protected DSP component: {lib_name}") -# continue - + +# # Hard protection for DSP components +# if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r']: +# self.logger.log_change(f"Protected DSP component: {lib_name}") +# continue + # Multiple patterns to catch different include formats patterns = [ rf'.*join\([^,]*,\s*"include",\s*"{re.escape(lib_name)}"[^)]*\),?\n', @@ -966,28 +959,28 @@ def _remove_ignored_lib_includes(self) -> None: rf'.*Path\([^)]*\)\s*/\s*"include"\s*/\s*"{re.escape(lib_name)}"[^,\n]*,?\n', rf'.*Path\([^)]*{re.escape(lib_name)}[^)]*\)\s*/\s*"include"[^,\n]*,?\n' ] - + removed_count = 0 for pattern in patterns: matches = re.findall(pattern, content) if matches: content = re.sub(pattern, '', content) removed_count += len(matches) - + if removed_count > 0: self.logger.log_change(f"Ignored library: {lib_name} ({removed_count} entries)") total_removed += removed_count - + # Clean up empty lines and trailing commas content = re.sub(r'\n\s*\n', '\n', content) content = re.sub(r',\s*\n\s*\]', '\n]', content) - + # Validate and write changes if self._validate_changes(original_content, content) and content != original_content: with open(build_py_path, 'w', encoding='utf-8') as f: f.write(content) self.logger.log_change(f"Updated build file ({total_removed} total removals)") - + except (IOError, OSError) as e: self.logger.log_change(f"Error processing libraries: {str(e)}") except Exception as e: @@ -996,47 +989,41 @@ def _remove_ignored_lib_includes(self) -> None: def _validate_changes(self, original_content: str, new_content: str) -> bool: """ Validate that the changes are reasonable and safe. - + Performs sanity checks on the modified content to ensure that the changes don't remove too much content or create invalid modifications that could break the build process. - + Args: original_content: Original file content before modifications - new_content: Modified file content after changes - + new_content: Modified file content after processing + Returns: - True if changes are valid and safe to apply + True if changes are within acceptable limits and safe to apply """ - # Check if too much content was removed (more than 50% indicates potential error) - if len(new_content) < len(original_content) * 0.5: - self.logger.log_change("Warning: Too much content removed, skipping changes") - return False - - # Check for basic Python syntax structure preservation - if 'CPPPATH' not in new_content or 'env.Append' not in new_content: - self.logger.log_change("Warning: Critical build structure missing, skipping changes") - return False - - return True + original_lines = len(original_content.splitlines()) + new_lines = len(new_content.splitlines()) + removed_lines = original_lines - new_lines + + # Don't allow removing more than 50% of the file or negative changes + return not (removed_lines > original_lines * 0.5 or removed_lines < 0) def _backup_pioarduino_build_py(self) -> None: """ Create backup of the original pioarduino-build.py file. - - Creates a backup of the Arduino framework's build script before - making modifications. Only operates when Arduino framework is active - and creates MCU-specific backup names to avoid conflicts. + + Creates a backup copy of the Arduino build script before making + modifications. Only operates when Arduino framework is active + and uses MCU-specific backup naming to avoid conflicts. """ if "arduino" not in self.config.env.subst("$PIOFRAMEWORK"): return if not self.config.arduino_libs_mcu: return - build_py_path = str(Path(self.config.arduino_libs_mcu) / "pioarduino-build.py") backup_path = str(Path(self.config.arduino_libs_mcu) / f"pioarduino-build.py.{self.config.mcu}") - + if os.path.exists(build_py_path) and not os.path.exists(backup_path): shutil.copy2(build_py_path, backup_path) From 7d3a60f2c0ac41866725748bcbfdb15a28d4c25e Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sun, 7 Sep 2025 15:26:59 +0200 Subject: [PATCH 10/13] clean up --- builder/frameworks/component_manager.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 66e242d67..41e437686 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -771,7 +771,8 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: 'esp32blearduino': 'bt', 'esp32_ble_arduino': 'bt', 'simpleble': 'bt', - 'esp-nimble-cpp': 'bt', + 'esp_nimble_cpp': 'bt', + 'nimble_arduino': 'bt', 'esp32': 'esp32', 'wire': 'driver', 'spi': 'driver', @@ -830,7 +831,8 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: 'esp_dsp': 'espressif__esp-dsp', 'dsps': 'espressif__esp-dsp', 'fft2r': 'espressif__esp-dsp', - 'dsps_fft2r': 'espressif__esp-dsp' + 'dsps_fft2r': 'espressif__esp-dsp', + 'esp-dsp': 'espressif__esp-dsp' } # Check extended mapping first @@ -898,7 +900,8 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: 'esp_dsp': 'espressif__esp-dsp', 'dsps': 'espressif__esp-dsp', 'fft2r': 'espressif__esp-dsp', - 'dsps_fft2r': 'espressif__esp-dsp' + 'dsps_fft2r': 'espressif__esp-dsp', + 'esp-dsp': 'espressif__esp-dsp' } if cleaned_name in direct_mapping: @@ -911,8 +914,8 @@ def _remove_ignored_lib_includes(self) -> None: Remove include entries for ignored libraries from pioarduino-build.py. Processes the Arduino build script to remove CPPPATH entries for - all ignored libraries. Implements protection for BT/BLE and DSP - components when dependencies are detected. Uses multiple regex + all ignored libraries. Implements protection for BT/BLE components + when dependencies are detected. Uses multiple regex patterns to catch different include path formats. """ build_py_path = str(Path(self.config.arduino_libs_mcu) / "pioarduino-build.py") @@ -940,11 +943,6 @@ def _remove_ignored_lib_includes(self) -> None: self.logger.log_change(f"Protected BT library: {lib_name}") continue -# # Hard protection for DSP components -# if lib_name.lower() in ['dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r']: -# self.logger.log_change(f"Protected DSP component: {lib_name}") -# continue - # Multiple patterns to catch different include formats patterns = [ rf'.*join\([^,]*,\s*"include",\s*"{re.escape(lib_name)}"[^)]*\),?\n', @@ -1019,8 +1017,6 @@ def _backup_pioarduino_build_py(self) -> None: if "arduino" not in self.config.env.subst("$PIOFRAMEWORK"): return - if not self.config.arduino_libs_mcu: - return build_py_path = str(Path(self.config.arduino_libs_mcu) / "pioarduino-build.py") backup_path = str(Path(self.config.arduino_libs_mcu) / f"pioarduino-build.py.{self.config.mcu}") From cf0b5d41879991faae694a4f9a10c116cd9327c7 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sun, 7 Sep 2025 15:39:03 +0200 Subject: [PATCH 11/13] Add 'dsp' to lib_ignore in platformio.ini --- examples/arduino-blink/platformio.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/arduino-blink/platformio.ini b/examples/arduino-blink/platformio.ini index ff1bbec71..7b68aeadf 100644 --- a/examples/arduino-blink/platformio.ini +++ b/examples/arduino-blink/platformio.ini @@ -14,6 +14,7 @@ board = esp32-solo1 build_flags = -DLED_BUILTIN=2 lib_ignore = wifi ble + dsp spiffs NetworkClientSecure Matter @@ -44,6 +45,7 @@ board = esp32-c2-devkitm-1 monitor_speed = 115200 lib_ignore = wifi ble + dsp spiffs NetworkClientSecure Matter @@ -75,6 +77,7 @@ board = arduino_nano_esp32 monitor_speed = 115200 lib_ignore = wifi ble + dsp spiffs NetworkClientSecure Matter @@ -105,6 +108,7 @@ framework = arduino board = esp32s3_120_16_8-qio_opi lib_ignore = wifi ble + dsp spiffs NetworkClientSecure Matter @@ -148,6 +152,7 @@ board = esp32-c6-devkitc-1 monitor_speed = 115200 lib_ignore = wifi ble + dsp spiffs NetworkClientSecure Matter @@ -177,6 +182,7 @@ framework = arduino board = esp32-h2-devkitm-1 monitor_speed = 115200 lib_ignore = ble + dsp spiffs NetworkClientSecure Matter @@ -207,6 +213,7 @@ board = esp32-p4 build_flags = -DLED_BUILTIN=2 lib_ignore = wifi ble + dsp spiffs NetworkClientSecure Matter From a56a4c1aa342cbce0634133f4f272c7022baac75 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sun, 7 Sep 2025 17:18:51 +0200 Subject: [PATCH 12/13] speed optimize --- builder/frameworks/component_manager.py | 795 +++++++++++++----------- 1 file changed, 445 insertions(+), 350 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 41e437686..83ef42c98 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -14,27 +14,27 @@ import yaml from yaml import SafeLoader from pathlib import Path -from typing import Set, Optional, Dict, Any, List, Tuple +from typing import Set, Optional, Dict, Any, List, Tuple, Pattern class ComponentManagerConfig: """ Handles configuration and environment setup for component management. - + This class centralizes all configuration-related operations and provides a unified interface for accessing PlatformIO environment settings, - board configurations, and framework paths. + board configurations, and framework paths with optimized caching. """ - + def __init__(self, env): """ Initialize the configuration manager with PlatformIO environment. - + Extracts and stores essential configuration parameters from the PlatformIO environment including platform details, board configuration, MCU type, and framework paths. This initialization ensures all dependent classes have consistent access to configuration data. - + Args: env: PlatformIO environment object containing project configuration, board settings, and platform information @@ -47,65 +47,86 @@ def __init__(self, env): self.mcu = self.board.get("build.mcu", "esp32").lower() # Get project source directory path self.project_src_dir = env.subst("$PROJECT_SRC_DIR") - # Get Arduino framework installation directory - self.arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") - # Get Arduino libraries installation directory - ald = self.platform.get_package_dir("framework-arduinoespressif32-libs") - # Get MCU-specific Arduino libraries directory - self.arduino_libs_mcu = ( - str(Path(ald) / self.mcu) if ald else "" - ) + + # Cache expensive operations using lazy loading + self._arduino_framework_dir = None + self._arduino_libs_mcu = None + + @property + def arduino_framework_dir(self): + """ + Lazy-loaded property for Arduino framework directory. + + Returns: + Path to Arduino ESP32 framework installation directory + """ + if self._arduino_framework_dir is None: + self._arduino_framework_dir = self.platform.get_package_dir("framework-arduinoespressif32") + return self._arduino_framework_dir + + @property + def arduino_libs_mcu(self): + """ + Lazy-loaded property for MCU-specific Arduino libraries directory. + + Returns: + Path to MCU-specific Arduino libraries directory + """ + if self._arduino_libs_mcu is None: + ald = self.platform.get_package_dir("framework-arduinoespressif32-libs") + self._arduino_libs_mcu = str(Path(ald) / self.mcu) if ald else "" + return self._arduino_libs_mcu class ComponentLogger: """ Simple logging functionality for component operations. - + Provides centralized logging for all component management operations, tracking changes made during the build process and offering summary reporting capabilities. """ - + def __init__(self): """ Initialize the logger with empty change tracking. - + Sets up internal data structures for tracking component changes and modifications made during the build process. """ # List to store all change messages for summary reporting self.component_changes: List[str] = [] - + def log_change(self, message: str) -> None: """ Log a change message with immediate console output. - + Records the change message internally for summary reporting and immediately prints it to the console with a component manager prefix for real-time feedback during build operations. - + Args: message: Descriptive message about the change or operation performed """ self.component_changes.append(message) print(f"[ComponentManager] {message}") - + def get_changes_summary(self) -> List[str]: """ Get a copy of all changes made during the session. - + Returns a defensive copy of the change log to prevent external modification while allowing access to the complete change history. - + Returns: List of change messages in chronological order """ return self.component_changes.copy() - + def print_changes_summary(self) -> None: """ Print a formatted summary of all changes made. - + Outputs a nicely formatted summary of all component changes if any were made, or a simple message indicating no changes occurred. Useful for end-of-build reporting and debugging. @@ -122,20 +143,20 @@ def print_changes_summary(self) -> None: class ComponentHandler: """ Handles IDF component addition and removal operations. - + Manages the core functionality for adding and removing ESP-IDF components from Arduino framework projects, including YAML file manipulation, component validation, and cleanup operations. """ - + def __init__(self, config: ComponentManagerConfig, logger: ComponentLogger): """ Initialize the component handler with configuration and logging. - + Sets up the component handler with necessary dependencies for configuration access and change logging. Initializes tracking for removed components to enable proper cleanup operations. - + Args: config: Configuration manager instance providing access to paths and settings logger: Logger instance for recording component operations @@ -144,15 +165,15 @@ def __init__(self, config: ComponentManagerConfig, logger: ComponentLogger): self.logger = logger # Track removed components for cleanup operations self.removed_components: Set[str] = set() - + def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: """ Handle adding and removing IDF components based on project configuration. - + Main entry point for component management operations. Processes both component additions and removals based on project configuration options, manages backup creation, and handles cleanup of removed components. - + Args: add_components: Whether to process component additions from custom_component_add remove_components: Whether to process component removals from custom_component_remove @@ -161,32 +182,32 @@ def handle_component_settings(self, add_components: bool = False, remove_compone if remove_components and not self.removed_components or add_components: self._backup_pioarduino_build_py() self.logger.log_change("Created backup of build file") - + # Check if env and GetProjectOption are available if hasattr(self.config, 'env') and hasattr(self.config.env, 'GetProjectOption'): component_yml_path = self._get_or_create_component_yml() component_data = self._load_component_yml(component_yml_path) - + if remove_components: self._process_component_removals(component_data) - + if add_components: self._process_component_additions(component_data) - + self._save_component_yml(component_yml_path, component_data) - + # Clean up removed components if self.removed_components: self._cleanup_removed_components() - + def _process_component_removals(self, component_data: Dict[str, Any]) -> None: """ Process component removal requests from project configuration. - + Reads the custom_component_remove option from platformio.ini and processes each component for removal from the dependency list. Handles errors gracefully and logs all operations. - + Args: component_data: Component configuration data dictionary containing dependencies """ @@ -198,15 +219,15 @@ def _process_component_removals(self, component_data: Dict[str, Any]) -> None: self._remove_components(component_data, components_to_remove) except Exception as e: self.logger.log_change(f"Error removing components: {str(e)}") - + def _process_component_additions(self, component_data: Dict[str, Any]) -> None: """ Process component addition requests from project configuration. - + Reads the custom_component_add option from platformio.ini and processes each component for addition to the dependency list. Handles errors gracefully and logs all operations. - + Args: component_data: Component configuration data dictionary containing dependencies """ @@ -218,16 +239,16 @@ def _process_component_additions(self, component_data: Dict[str, Any]) -> None: self._add_components(component_data, components_to_add) except Exception as e: self.logger.log_change(f"Error adding components: {str(e)}") - + def _get_or_create_component_yml(self) -> str: """ Get path to idf_component.yml, creating it if necessary. - + Searches for existing idf_component.yml files in the Arduino framework directory first, then in the project source directory. If no file exists, creates a new one in the project source directory with default content. - + Returns: Absolute path to the component YAML file """ @@ -237,40 +258,40 @@ def _get_or_create_component_yml(self) -> str: if framework_yml and os.path.exists(framework_yml): self._create_backup(framework_yml) return framework_yml - + # Try project source directory project_yml = str(Path(self.config.project_src_dir) / "idf_component.yml") if os.path.exists(project_yml): self._create_backup(project_yml) return project_yml - + # Create new file in project source self._create_default_component_yml(project_yml) return project_yml - + def _create_backup(self, file_path: str) -> None: """ Create backup of a file with .orig extension. - + Creates a backup copy of the specified file by appending .orig to the filename. Only creates the backup if it doesn't already exist to preserve the original state. - + Args: file_path: Absolute path to the file to backup """ backup_path = f"{file_path}.orig" if not os.path.exists(backup_path): shutil.copy(file_path, backup_path) - + def _create_default_component_yml(self, file_path: str) -> None: """ Create a default idf_component.yml file with basic ESP-IDF dependency. - + Creates a new component YAML file with minimal default content specifying ESP-IDF version 5.1 or higher as the base dependency. This ensures compatibility with modern ESP-IDF features. - + Args: file_path: Absolute path where to create the new YAML file """ @@ -279,21 +300,21 @@ def _create_default_component_yml(self, file_path: str) -> None: "idf": ">=5.1" } } - + with open(file_path, 'w', encoding='utf-8') as f: yaml.dump(default_content, f) - + def _load_component_yml(self, file_path: str) -> Dict[str, Any]: """ Load and parse idf_component.yml file safely. - + Attempts to load and parse the YAML file using SafeLoader for security. Returns a default structure with empty dependencies if the file cannot be read or parsed. - + Args: file_path: Absolute path to the YAML file to load - + Returns: Parsed YAML data as dictionary, or default structure on failure """ @@ -302,15 +323,15 @@ def _load_component_yml(self, file_path: str) -> Dict[str, Any]: return yaml.load(f, Loader=SafeLoader) or {"dependencies": {}} except Exception: return {"dependencies": {}} - + def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: """ Save component data to YAML file safely. - + Attempts to write the component data dictionary to the specified YAML file. Handles errors gracefully by silently failing to prevent build interruption. - + Args: file_path: Absolute path to the YAML file to write data: Component data dictionary to serialize @@ -320,74 +341,74 @@ def _save_component_yml(self, file_path: str, data: Dict[str, Any]) -> None: yaml.dump(data, f) except Exception: pass - + def _remove_components(self, component_data: Dict[str, Any], components_to_remove: list) -> None: """ Remove specified components from the configuration. - + Iterates through the list of components to remove, checking if each exists in the dependencies and removing it if found. Tracks removed components for later cleanup operations and logs all actions. - + Args: component_data: Component configuration data dictionary components_to_remove: List of component names to remove """ dependencies = component_data.setdefault("dependencies", {}) - + for component in components_to_remove: component = component.strip() if not component: continue - + if component in dependencies: self.logger.log_change(f"Removed component: {component}") del dependencies[component] - + # Track for cleanup - convert to filesystem-safe name filesystem_name = self._convert_component_name_to_filesystem(component) self.removed_components.add(filesystem_name) else: self.logger.log_change(f"Component not found: {component}") - + def _add_components(self, component_data: Dict[str, Any], components_to_add: list) -> None: """ Add specified components to the configuration. - + Processes each component entry, parsing name and version information, and adds new components to the dependencies. Skips components that already exist and filters out entries that are too short to be valid. - + Args: component_data: Component configuration data dictionary components_to_add: List of component entries to add (format: name@version or name) """ dependencies = component_data.setdefault("dependencies", {}) - + for component in components_to_add: component = component.strip() - if len(component) <= 4: # Skip too short entries + if not component: # Skip empty entrys continue - + component_name, version = self._parse_component_entry(component) - + if component_name not in dependencies: dependencies[component_name] = {"version": version} self.logger.log_change(f"Added component: {component_name} ({version})") else: self.logger.log_change(f"Component already exists: {component_name}") - + def _parse_component_entry(self, entry: str) -> Tuple[str, str]: """ Parse component entry into name and version components. - + Splits component entries that contain version information (format: name@version) and returns both parts. If no version is specified, defaults to "*" for latest version. - + Args: entry: Component entry string (e.g., "espressif/esp_timer@1.0.0" or "espressif/esp_timer") - + Returns: Tuple containing (component_name, version) """ @@ -395,26 +416,26 @@ def _parse_component_entry(self, entry: str) -> Tuple[str, str]: name, version = entry.split("@", 1) return (name.strip(), version.strip()) return (entry.strip(), "*") - + def _convert_component_name_to_filesystem(self, component_name: str) -> str: """ Convert component name from registry format to filesystem format. - + Converts component names from ESP Component Registry format (using forward slashes) to filesystem-safe format (using double underscores) for directory operations. - + Args: component_name: Component name in registry format (e.g., "espressif/esp_timer") - + Returns: Filesystem-safe component name (e.g., "espressif__esp_timer") """ return component_name.replace("/", "__") - + def _backup_pioarduino_build_py(self) -> None: """ Create backup of the original pioarduino-build.py file. - + Creates a backup of the Arduino framework's build script before making modifications. Only operates when Arduino framework is active and creates MCU-specific backup names to avoid conflicts. @@ -427,93 +448,104 @@ def _backup_pioarduino_build_py(self) -> None: build_py_path = str(Path(self.config.arduino_libs_mcu) / "pioarduino-build.py") backup_path = str(Path(self.config.arduino_libs_mcu) / f"pioarduino-build.py.{self.config.mcu}") - + if os.path.exists(build_py_path) and not os.path.exists(backup_path): shutil.copy2(build_py_path, backup_path) - + def _cleanup_removed_components(self) -> None: """ Clean up removed components and restore original build file. - - Performs cleanup operations for all components that were removed, + + Performs optimized batch cleanup operations for all components that were removed, including removing include directories and cleaning up CPPPATH - entries from the build script. + entries from the build script in a single pass. """ - for component in self.removed_components: - self._remove_include_directory(component) - - self._remove_cpppath_entries() - - def _remove_include_directory(self, component: str) -> None: + if not self.removed_components: + return + + # Batch remove include directories + self._batch_remove_include_directories() + + # Single pass through build file for all components + self._batch_remove_cpppath_entries() + + def _batch_remove_include_directories(self) -> None: """ - Remove include directory for a specific component. - - Removes the component's include directory from the Arduino framework - libraries to prevent compilation errors and reduce build overhead. - - Args: - component: Component name in filesystem format + Remove multiple include directories in one optimized operation. + + Removes all component include directories efficiently without + individual file system calls for each component. """ - include_path = str(Path(self.config.arduino_libs_mcu) / "include" / component) - - if os.path.exists(include_path): - shutil.rmtree(include_path) - - def _remove_cpppath_entries(self) -> None: + include_base_path = Path(self.config.arduino_libs_mcu) / "include" + + for component in self.removed_components: + include_path = include_base_path / component + if include_path.exists(): + try: + shutil.rmtree(include_path) + except OSError: + pass # Continue with other components + + def _batch_remove_cpppath_entries(self) -> None: """ - Remove CPPPATH entries for removed components from pioarduino-build.py. - - Scans the Arduino build script and removes include path entries - for all components that were removed from the project. Uses - multiple regex patterns to catch different include path formats. + Remove CPPPATH entries for all components in single optimized file pass. + + Uses compiled regex patterns and processes all removed components + in a single pass through the build file for maximum efficiency. """ build_py_path = str(Path(self.config.arduino_libs_mcu) / "pioarduino-build.py") - + if not os.path.exists(build_py_path): return - + try: with open(build_py_path, 'r', encoding='utf-8') as f: content = f.read() - + original_content = content - - # Remove CPPPATH entries for each removed component - for component in self.removed_components: - patterns = [ - rf'.*join\([^,]*,\s*"include",\s*"{re.escape(component)}"[^)]*\),?\n', - rf'.*"include/{re.escape(component)}"[^,\n]*,?\n', - rf'.*"[^"]*include[^"]*{re.escape(component)}[^"]*"[^,\n]*,?\n' - ] - - for pattern in patterns: - content = re.sub(pattern, '', content) - + + # Create combined pattern for all components for maximum efficiency + escaped_components = [re.escape(comp) for comp in self.removed_components] + component_pattern = '|'.join(escaped_components) + + # Compile patterns once for all components + combined_patterns = [ + re.compile(rf'.*join\([^,]*,\s*"include",\s*"({component_pattern})"[^)]*\),?\n'), + re.compile(rf'.*"include/({component_pattern})"[^,\n]*,?\n'), + re.compile(rf'.*"[^"]*include[^"]*({component_pattern})[^"]*"[^,\n]*,?\n') + ] + + # Apply all patterns in single pass + for pattern in combined_patterns: + content = pattern.sub('', content) + + # Write changes if any were made if content != original_content: with open(build_py_path, 'w', encoding='utf-8') as f: f.write(content) - + except Exception: pass class LibraryIgnoreHandler: """ - Handles lib_ignore processing and include removal. - + Handles lib_ignore processing and include removal with optimized performance. + Manages the processing of lib_ignore entries from platformio.ini, converting library names to include paths and removing corresponding entries from the build script while protecting critical components. + Uses compiled regex patterns and caching for maximum performance. """ - + def __init__(self, config: ComponentManagerConfig, logger: ComponentLogger): """ - Initialize the library ignore handler. - + Initialize the library ignore handler with performance optimizations. + Sets up the handler with configuration and logging dependencies, - initializes tracking for ignored libraries, and prepares caching - for Arduino library mappings. - + initializes tracking for ignored libraries, and prepares optimized + caching and compiled patterns for maximum performance. + Args: config: Configuration manager instance for accessing paths and settings logger: Logger instance for recording library operations @@ -522,49 +554,74 @@ def __init__(self, config: ComponentManagerConfig, logger: ComponentLogger): self.logger = logger # Track ignored libraries for processing self.ignored_libs: Set[str] = set() - # Cache for Arduino library mappings (lazy loaded) + + # Performance optimization: Pre-compute critical components as set for O(1) lookup + self._critical_components = { + 'lwip', # Network stack + 'freertos', # Real-time OS + 'esp_system', # System functions + 'esp_common', # Common ESP functions + 'driver', # Hardware drivers + 'nvs_flash', # Non-volatile storage + 'spi_flash', # Flash memory access + 'esp_timer', # Timer functions + 'esp_event', # Event system + 'log', # Logging system + 'arduino_tinyusb', # Arduino TinyUSB library + 'tinyusb' # TinyUSB library + } + + # Pre-compute BT-related keywords as set for O(1) lookup + self._bt_keywords = { + 'BLE', 'BT', 'NIMBLE', 'BLUETOOTH', 'ESP32_BLE', 'ESP32BLE', + 'BLUETOOTHSERIAL', 'BLE_ARDUINO', 'ESP_BLE', 'ESP_BT' + } + + # Cache for expensive operations (lazy loaded) self._arduino_libraries_cache = None - + self._compiled_patterns_cache = {} + self._cleanup_patterns = None + def handle_lib_ignore(self) -> None: """ Handle lib_ignore entries from platformio.ini and remove corresponding includes. - + Main entry point for library ignore processing. Creates backup if needed, processes lib_ignore entries from the current environment, and removes - corresponding include paths from the build script. + corresponding include paths from the build script using optimized algorithms. """ # Create backup before processing lib_ignore if not self.ignored_libs: self._backup_pioarduino_build_py() - + # Get lib_ignore entries from current environment only lib_ignore_entries = self._get_lib_ignore_entries() - + if lib_ignore_entries: self.ignored_libs.update(lib_ignore_entries) self._remove_ignored_lib_includes() self.logger.log_change(f"Processed {len(lib_ignore_entries)} ignored libraries") - + def _get_lib_ignore_entries(self) -> List[str]: """ - Get lib_ignore entries from current environment configuration only. - + Get lib_ignore entries from current environment configuration with optimized filtering. + Extracts and processes lib_ignore entries from the platformio.ini configuration, converting library names to include directory names - and filtering out critical ESP32 components that should never be ignored. - + and filtering out critical ESP32 components using O(1) set lookups. + Returns: List of processed library names ready for include path removal """ try: # Get lib_ignore from current environment only lib_ignore = self.config.env.GetProjectOption("lib_ignore", []) - + if isinstance(lib_ignore, str): lib_ignore = [lib_ignore] elif lib_ignore is None: lib_ignore = [] - + # Clean and normalize entries cleaned_entries = [] for entry in lib_ignore: @@ -572,117 +629,81 @@ def _get_lib_ignore_entries(self) -> List[str]: if entry: # Convert library names to potential include directory names include_name = self._convert_lib_name_to_include(entry) - cleaned_entries.append(include_name) - - # Filter out critical ESP32 components that should never be ignored - critical_components = [ - 'lwip', # Network stack - 'freertos', # Real-time OS - 'esp_system', # System functions - 'esp_common', # Common ESP functions - 'driver', # Hardware drivers - 'nvs_flash', # Non-volatile storage - 'spi_flash', # Flash memory access - 'esp_timer', # Timer functions - 'esp_event', # Event system - 'log', # Logging system - 'arduino_tinyusb', # Arduino TinyUSB library - 'tinyusb' # TinyUSB library - ] - - filtered_entries = [] - for entry in cleaned_entries: - if entry not in critical_components: - filtered_entries.append(entry) - - return filtered_entries - + # Use optimized set lookup for critical components check (O(1) vs O(n)) + if include_name not in self._critical_components: + cleaned_entries.append(include_name) + + return cleaned_entries + except Exception: return [] - + def _has_bt_ble_dependencies(self) -> bool: """ - Check if lib_deps contains any BT/BLE related dependencies. - + Check if lib_deps contains any BT/BLE related dependencies using optimized search. + Scans the lib_deps configuration option for Bluetooth or BLE related keywords to determine if BT components should be protected from removal even if they appear in lib_ignore. - + Returns: True if BT/BLE dependencies are found in lib_deps """ try: # Get lib_deps from current environment lib_deps = self.config.env.GetProjectOption("lib_deps", []) - + if isinstance(lib_deps, str): lib_deps = [lib_deps] elif lib_deps is None: lib_deps = [] - - # Convert to string and check for BT/BLE keywords + + # Convert to string and check for BT/BLE keywords using set intersection lib_deps_str = ' '.join(str(dep) for dep in lib_deps).upper() - - bt_ble_keywords = ['BLE', 'BT', 'NIMBLE', 'BLUETOOTH'] + return any(keyword in lib_deps_str for keyword in self._bt_keywords) - return any(keyword in lib_deps_str for keyword in bt_ble_keywords) - except Exception: return False - + def _is_bt_related_library(self, lib_name: str) -> bool: """ - Check if a library name is related to Bluetooth/BLE functionality. - + Check if a library name is related to Bluetooth/BLE functionality using optimized lookup. + Examines library names for Bluetooth and BLE related keywords to determine if the library should be protected when BT dependencies - are present in the project. - + are present in the project. Uses pre-computed set for fast lookup. + Args: lib_name: Library name to check for BT/BLE relation - + Returns: True if library name contains BT/BLE related keywords """ lib_name_upper = lib_name.upper() - - bt_related_names = [ - 'BT', - 'BLE', - 'BLUETOOTH', - 'NIMBLE', - 'ESP32_BLE', - 'ESP32BLE', - 'BLUETOOTHSERIAL', - 'BLE_ARDUINO', - 'ESP_BLE', - 'ESP_BT' - ] - - return any(bt_name in lib_name_upper for bt_name in bt_related_names) - + return any(bt_keyword in lib_name_upper for bt_keyword in self._bt_keywords) + def _get_arduino_core_libraries(self) -> Dict[str, str]: """ Get all Arduino core libraries and their corresponding include paths. - + Scans the Arduino framework libraries directory to build a mapping of library names to their corresponding include paths. Reads library.properties files to get official library names. - + Returns: Dictionary mapping library names to include directory names """ libraries_mapping = {} - + # Path to Arduino Core Libraries afd = self.config.arduino_framework_dir if not afd: return libraries_mapping arduino_libs_dir = str(Path(afd).resolve() / "libraries") - + if not os.path.exists(arduino_libs_dir): return libraries_mapping - + try: for entry in os.listdir(arduino_libs_dir): lib_path = str(Path(arduino_libs_dir) / entry) @@ -694,26 +715,26 @@ def _get_arduino_core_libraries(self) -> Dict[str, str]: libraries_mapping[entry.lower()] = include_path # Also use directory name as key except Exception: pass - + return libraries_mapping - + def _get_library_name_from_properties(self, lib_dir: str) -> Optional[str]: """ Extract library name from library.properties file. - + Reads the library.properties file in the given directory and extracts the official library name from the 'name=' field. - + Args: lib_dir: Path to library directory containing library.properties - + Returns: Official library name or None if not found or readable """ prop_path = str(Path(lib_dir) / "library.properties") if not os.path.isfile(prop_path): return None - + try: with open(prop_path, 'r', encoding='utf-8') as f: for line in f: @@ -722,27 +743,27 @@ def _get_library_name_from_properties(self, lib_dir: str) -> Optional[str]: return line.split('=', 1)[1].strip() except Exception: pass - + return None - + def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: """ Map library name to corresponding include path. - + Converts Arduino library names to their corresponding ESP-IDF component include paths using an extensive mapping table. Handles common Arduino libraries and their ESP-IDF equivalents. - + Args: lib_name: Official library name from library.properties dir_name: Directory name of the library - + Returns: Corresponding ESP-IDF component include path name """ lib_name_lower = lib_name.lower().replace(' ', '').replace('-', '_') dir_name_lower = dir_name.lower() - + # Extended mapping list with Arduino Core Libraries extended_mapping = { # Core ESP32 mappings @@ -766,7 +787,7 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: 'json': 'cjson', 'mbedtls': 'mbedtls', 'openssl': 'openssl', - + # Arduino Core specific mappings (safe mappings that don't conflict with critical components) 'esp32blearduino': 'bt', 'esp32_ble_arduino': 'bt', @@ -834,89 +855,142 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: 'dsps_fft2r': 'espressif__esp-dsp', 'esp-dsp': 'espressif__esp-dsp' } - + # Check extended mapping first if lib_name_lower in extended_mapping: return extended_mapping[lib_name_lower] - + # Check directory name if dir_name_lower in extended_mapping: return extended_mapping[dir_name_lower] - + # Fallback: Use directory name as include path return dir_name_lower - + def _convert_lib_name_to_include(self, lib_name: str) -> str: """ - Convert library name to potential include directory name. - + Convert library name to potential include directory name with optimized fast paths. + Converts library names from platformio.ini lib_ignore entries to their corresponding include directory names. Uses Arduino - core library mappings and common naming conventions. - + core library mappings and common naming conventions with + performance optimizations for common cases like DSP. + Args: lib_name: Library name from lib_ignore configuration - + Returns: Converted include directory name for path removal """ - # Load Arduino Core Libraries on first call + lib_name_lower = lib_name.lower() + + # Fast path optimization for DSP components (most performance-critical case) + dsp_patterns = { + 'dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r', 'esp-dsp' + } + if lib_name_lower in dsp_patterns: + return 'espressif__esp-dsp' + + # Fast path for BT components + bt_patterns = { + 'ble', 'bluetooth', 'bluetoothserial', 'simpleble', 'esp-nimble-cpp' + } + if lib_name_lower in bt_patterns: + return 'bt' + + # Load Arduino Core Libraries on first call (lazy loading) if self._arduino_libraries_cache is None: self._arduino_libraries_cache = self._get_arduino_core_libraries() - - lib_name_lower = lib_name.lower() - - # Check Arduino Core Libraries first + + # Check Arduino Core Libraries cache if lib_name_lower in self._arduino_libraries_cache: return self._arduino_libraries_cache[lib_name_lower] - + + # Continue with full conversion logic for less common cases + return self._full_conversion_logic(lib_name_lower) + + def _full_conversion_logic(self, lib_name_lower: str) -> str: + """ + Full conversion logic for library names not handled by fast paths. + + Args: + lib_name_lower: Lowercase library name to convert + + Returns: + Converted include directory name + """ # Remove common prefixes and suffixes cleaned_name = lib_name_lower - + # Remove common prefixes prefixes_to_remove = ['lib', 'arduino-', 'esp32-', 'esp-'] for prefix in prefixes_to_remove: if cleaned_name.startswith(prefix): cleaned_name = cleaned_name[len(prefix):] - + # Remove common suffixes suffixes_to_remove = ['-lib', '-library', '.h'] for suffix in suffixes_to_remove: if cleaned_name.endswith(suffix): cleaned_name = cleaned_name[:-len(suffix)] - + # Check again with cleaned name if cleaned_name in self._arduino_libraries_cache: return self._arduino_libraries_cache[cleaned_name] - - # Direct mapping for common cases not in Arduino libraries - direct_mapping = { - 'ble': 'bt', - 'bluetooth': 'bt', - 'bluetoothserial': 'bt', - 'simpleble': 'bt', - 'esp-nimble-cpp': 'bt', - 'dsp': 'espressif__esp-dsp', - 'esp_dsp': 'espressif__esp-dsp', - 'dsps': 'espressif__esp-dsp', - 'fft2r': 'espressif__esp-dsp', - 'dsps_fft2r': 'espressif__esp-dsp', - 'esp-dsp': 'espressif__esp-dsp' - } - - if cleaned_name in direct_mapping: - return direct_mapping[cleaned_name] - + return cleaned_name - + + def _get_compiled_patterns(self, lib_name: str) -> List[Pattern]: + """ + Get pre-compiled regex patterns for a library name with caching. + + Compiles and caches regex patterns for library name matching + to avoid repeated compilation overhead during processing. + + Args: + lib_name: Library name to create patterns for + + Returns: + List of compiled regex patterns for the library + """ + if lib_name not in self._compiled_patterns_cache: + escaped_name = re.escape(lib_name) + patterns = [ + re.compile(rf'.*join\([^,]*,\s*"include",\s*"{escaped_name}"[^)]*\),?\n'), + re.compile(rf'.*"include/{escaped_name}"[^,\n]*,?\n'), + re.compile(rf'.*"[^"]*include[^"]*{escaped_name}[^"]*"[^,\n]*,?\n'), + re.compile(rf'.*"[^"]*/{escaped_name}/include[^"]*"[^,\n]*,?\n'), + re.compile(rf'.*"[^"]*{escaped_name}[^"]*include[^"]*"[^,\n]*,?\n'), + re.compile(rf'.*join\([^)]*"include"[^)]*"{escaped_name}"[^)]*\),?\n'), + re.compile(rf'.*"{escaped_name}/include"[^,\n]*,?\n'), + re.compile(rf'\s*"[^"]*[\\/]{escaped_name}[\\/][^"]*",?\n'), + re.compile(rf'.*Path\([^)]*\)\s*/\s*"include"\s*/\s*"{escaped_name}"[^,\n]*,?\n'), + re.compile(rf'.*Path\([^)]*{escaped_name}[^)]*\)\s*/\s*"include"[^,\n]*,?\n') + ] + self._compiled_patterns_cache[lib_name] = patterns + return self._compiled_patterns_cache[lib_name] + + def _get_cleanup_patterns(self) -> List[Pattern]: + """ + Get compiled cleanup patterns with caching. + + Returns: + List of compiled regex patterns for content cleanup + """ + if self._cleanup_patterns is None: + self._cleanup_patterns = [ + re.compile(r'\n\s*\n'), + re.compile(r',\s*\n\s*\]') + ] + return self._cleanup_patterns + def _remove_ignored_lib_includes(self) -> None: """ - Remove include entries for ignored libraries from pioarduino-build.py. + Remove include entries for ignored libraries using optimized batch processing. Processes the Arduino build script to remove CPPPATH entries for - all ignored libraries. Implements protection for BT/BLE components - when dependencies are detected. Uses multiple regex - patterns to catch different include path formats. + all ignored libraries using compiled regex patterns and batch processing. + Implements protection for BT/BLE components when dependencies are detected. """ build_py_path = str(Path(self.config.arduino_libs_mcu) / "pioarduino-build.py") @@ -924,99 +998,120 @@ def _remove_ignored_lib_includes(self) -> None: self.logger.log_change("Build file not found") return - # Check if BT/BLE dependencies exist in lib_deps + # Check if BT/BLE dependencies exist in lib_deps (single check) bt_ble_protected = self._has_bt_ble_dependencies() if bt_ble_protected: self.logger.log_change("BT/BLE protection enabled") try: + # Read file once with open(build_py_path, 'r', encoding='utf-8') as f: content = f.read() original_content = content total_removed = 0 - # Remove CPPPATH entries for each ignored library + # Pre-filter libraries to process (avoid processing protected libraries) + libs_to_process = [] for lib_name in self.ignored_libs: - # Skip BT-related libraries if BT/BLE dependencies are present if bt_ble_protected and self._is_bt_related_library(lib_name): self.logger.log_change(f"Protected BT library: {lib_name}") continue + libs_to_process.append(lib_name) - # Multiple patterns to catch different include formats - patterns = [ - rf'.*join\([^,]*,\s*"include",\s*"{re.escape(lib_name)}"[^)]*\),?\n', - rf'.*"include/{re.escape(lib_name)}"[^,\n]*,?\n', - rf'.*"[^"]*include[^"]*{re.escape(lib_name)}[^"]*"[^,\n]*,?\n', - rf'.*"[^"]*/{re.escape(lib_name)}/include[^"]*"[^,\n]*,?\n', - rf'.*"[^"]*{re.escape(lib_name)}[^"]*include[^"]*"[^,\n]*,?\n', - rf'.*join\([^)]*"include"[^)]*"{re.escape(lib_name)}"[^)]*\),?\n', - rf'.*"{re.escape(lib_name)}/include"[^,\n]*,?\n', - rf'\s*"[^"]*[\\/]{re.escape(lib_name)}[\\/][^"]*",?\n', - # pathlib-style: Path(...)/"include"/"" - rf'.*Path\([^)]*\)\s*/\s*"include"\s*/\s*"{re.escape(lib_name)}"[^,\n]*,?\n', - rf'.*Path\([^)]*{re.escape(lib_name)}[^)]*\)\s*/\s*"include"[^,\n]*,?\n' - ] - - removed_count = 0 - for pattern in patterns: - matches = re.findall(pattern, content) - if matches: - content = re.sub(pattern, '', content) - removed_count += len(matches) - - if removed_count > 0: - self.logger.log_change(f"Ignored library: {lib_name} ({removed_count} entries)") - total_removed += removed_count - - # Clean up empty lines and trailing commas - content = re.sub(r'\n\s*\n', '\n', content) - content = re.sub(r',\s*\n\s*\]', '\n]', content) - - # Validate and write changes - if self._validate_changes(original_content, content) and content != original_content: - with open(build_py_path, 'w', encoding='utf-8') as f: - f.write(content) - self.logger.log_change(f"Updated build file ({total_removed} total removals)") + # Batch process all libraries using compiled patterns + if libs_to_process: + content, total_removed = self._batch_remove_patterns(content, libs_to_process) + + # Clean up content once at the end + if total_removed > 0: + content = self._cleanup_content(content) + + # Validate and write changes + if self._validate_changes(original_content, content): + with open(build_py_path, 'w', encoding='utf-8') as f: + f.write(content) + self.logger.log_change(f"Updated build file ({total_removed} total removals)") - except (IOError, OSError) as e: - self.logger.log_change(f"Error processing libraries: {str(e)}") except Exception as e: - self.logger.log_change(f"Unexpected error processing libraries: {str(e)}") + self.logger.log_change(f"Error processing libraries: {str(e)}") - def _validate_changes(self, original_content: str, new_content: str) -> bool: + def _batch_remove_patterns(self, content: str, libs_to_process: List[str]) -> Tuple[str, int]: + """ + Process all libraries in batches using compiled patterns for optimal performance. + + Args: + content: File content to process + libs_to_process: List of library names to remove + + Returns: + Tuple of (modified_content, total_removed_count) """ - Validate that the changes are reasonable and safe. + total_removed = 0 + + for lib_name in libs_to_process: + patterns = self._get_compiled_patterns(lib_name) + removed_count = 0 - Performs sanity checks on the modified content to ensure that - the changes don't remove too much content or create invalid - modifications that could break the build process. + for pattern in patterns: + matches = pattern.findall(content) + if matches: + content = pattern.sub('', content) + removed_count += len(matches) + + if removed_count > 0: + self.logger.log_change(f"Ignored library: {lib_name} ({removed_count} entries)") + total_removed += removed_count + + return content, total_removed + + def _cleanup_content(self, content: str) -> str: + """ + Optimized content cleanup using compiled patterns. Args: - original_content: Original file content before modifications - new_content: Modified file content after processing + content: Content to clean up Returns: - True if changes are within acceptable limits and safe to apply + Cleaned content """ - original_lines = len(original_content.splitlines()) - new_lines = len(new_content.splitlines()) - removed_lines = original_lines - new_lines + cleanup_patterns = self._get_cleanup_patterns() + content = cleanup_patterns[0].sub('\n', content) + content = cleanup_patterns[1].sub('\n]', content) + return content + + def _validate_changes(self, original_content: str, new_content: str) -> bool: + """ + Validate that changes are safe and don't break the build file structure. + + Args: + original_content: Original file content + new_content: Modified file content - # Don't allow removing more than 50% of the file or negative changes - return not (removed_lines > original_lines * 0.5 or removed_lines < 0) + Returns: + True if changes are safe to apply + """ + # Basic validation - ensure we haven't broken basic Python syntax + if new_content != original_content: + # Check that we still have essential structure elements + essential_elements = ['CPPPATH', 'env.Append', '[', ']'] + return all(element in new_content for element in essential_elements) + return True def _backup_pioarduino_build_py(self) -> None: """ Create backup of the original pioarduino-build.py file. - - Creates a backup copy of the Arduino build script before making - modifications. Only operates when Arduino framework is active - and uses MCU-specific backup naming to avoid conflicts. + + Creates a backup of the Arduino framework's build script before + making modifications. Only operates when Arduino framework is active + and creates MCU-specific backup names to avoid conflicts. """ if "arduino" not in self.config.env.subst("$PIOFRAMEWORK"): return + if not self.config.arduino_libs_mcu: + return + build_py_path = str(Path(self.config.arduino_libs_mcu) / "pioarduino-build.py") backup_path = str(Path(self.config.arduino_libs_mcu) / f"pioarduino-build.py.{self.config.mcu}") @@ -1036,10 +1131,10 @@ class BackupManager: def __init__(self, config: ComponentManagerConfig): """ Initialize the backup manager with configuration access. - + Sets up the backup manager with access to configuration paths and settings needed for backup and restore operations. - + Args: config: Configuration manager instance providing access to paths """ @@ -1048,7 +1143,7 @@ def __init__(self, config: ComponentManagerConfig): def backup_pioarduino_build_py(self) -> None: """ Create backup of the original pioarduino-build.py file. - + Creates a backup copy of the Arduino framework's build script with MCU-specific naming to prevent conflicts between different ESP32 variants. Only creates backup if it doesn't already exist. @@ -1065,11 +1160,11 @@ def backup_pioarduino_build_py(self) -> None: def restore_pioarduino_build_py(self, target=None, source=None, env=None) -> None: """ Restore the original pioarduino-build.py from backup. - + Restores the original Arduino build script from the backup copy and removes the backup file. This is typically called during clean operations or when resetting the build environment. - + Args: target: Build target (unused, for PlatformIO compatibility) source: Build source (unused, for PlatformIO compatibility) @@ -1086,21 +1181,21 @@ def restore_pioarduino_build_py(self, target=None, source=None, env=None) -> Non class ComponentManager: """ Main component manager that orchestrates all operations. - + Primary interface for component management operations, coordinating between specialized handlers for components, libraries, and backups. Uses composition pattern to organize functionality into focused classes. """ - + def __init__(self, env): """ Initialize the ComponentManager with composition pattern. - + Creates and configures all specialized handler instances using the composition pattern for better separation of concerns and maintainability. Each handler focuses on a specific aspect of component management. - + Args: env: PlatformIO environment object containing project configuration """ @@ -1109,66 +1204,66 @@ def __init__(self, env): self.component_handler = ComponentHandler(self.config, self.logger) self.library_handler = LibraryIgnoreHandler(self.config, self.logger) self.backup_manager = BackupManager(self.config) - + def handle_component_settings(self, add_components: bool = False, remove_components: bool = False) -> None: """ Handle component operations by delegating to specialized handlers. - + Main entry point for component management operations. Coordinates component addition/removal and library ignore processing, then provides a summary of all changes made during the session. - + Args: add_components: Whether to process component additions from configuration remove_components: Whether to process component removals from configuration """ self.component_handler.handle_component_settings(add_components, remove_components) self.library_handler.handle_lib_ignore() - + # Print summary changes = self.logger.get_changes_summary() if changes: self.logger.log_change(f"Session completed with {len(changes)} changes") - + def handle_lib_ignore(self) -> None: """ Delegate lib_ignore handling to specialized handler. - + Provides direct access to library ignore processing for cases where only library handling is needed without component operations. """ self.library_handler.handle_lib_ignore() - + def restore_pioarduino_build_py(self, target=None, source=None, env=None) -> None: """ Delegate backup restoration to backup manager. - + Provides access to backup restoration functionality, typically used during clean operations or build environment resets. - + Args: target: Build target (unused, for PlatformIO compatibility) source: Build source (unused, for PlatformIO compatibility) env: Environment (unused, for PlatformIO compatibility) """ self.backup_manager.restore_pioarduino_build_py(target, source, env) - + def get_changes_summary(self) -> List[str]: """ Get summary of changes from logger. - + Provides access to the complete list of changes made during the current session for reporting or debugging purposes. - + Returns: List of change messages in chronological order """ return self.logger.get_changes_summary() - + def print_changes_summary(self) -> None: """ Print changes summary via logger. - + Outputs a formatted summary of all changes made during the session, useful for build reporting and debugging. """ From 85aff244997a79d56ad31a8dd9a3e5e37a4efa6b Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sun, 7 Sep 2025 17:39:12 +0200 Subject: [PATCH 13/13] Fix typos and enhance error handling in component manager --- builder/frameworks/component_manager.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 83ef42c98..118c1f508 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -387,7 +387,7 @@ def _add_components(self, component_data: Dict[str, Any], components_to_add: lis for component in components_to_add: component = component.strip() - if not component: # Skip empty entrys + if not component: # Skip empty entries continue component_name, version = self._parse_component_entry(component) @@ -524,8 +524,8 @@ def _batch_remove_cpppath_entries(self) -> None: with open(build_py_path, 'w', encoding='utf-8') as f: f.write(content) - except Exception: - pass + except Exception as e: + print(f"[ComponentManager] Error updating build file during CPPPATH cleanup: {e!s}") class LibraryIgnoreHandler: @@ -633,7 +633,7 @@ def _get_lib_ignore_entries(self) -> List[str]: if include_name not in self._critical_components: cleaned_entries.append(include_name) - return cleaned_entries + return sorted(set(cleaned_entries)) except Exception: return [] @@ -853,7 +853,9 @@ def _map_library_to_include_path(self, lib_name: str, dir_name: str) -> str: 'dsps': 'espressif__esp-dsp', 'fft2r': 'espressif__esp-dsp', 'dsps_fft2r': 'espressif__esp-dsp', - 'esp-dsp': 'espressif__esp-dsp' + 'esp-dsp': 'espressif__esp-dsp', + 'espressif/esp-dsp': 'espressif__esp-dsp', + 'espressif__esp-dsp': 'espressif__esp-dsp' } # Check extended mapping first @@ -886,7 +888,8 @@ def _convert_lib_name_to_include(self, lib_name: str) -> str: # Fast path optimization for DSP components (most performance-critical case) dsp_patterns = { - 'dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r', 'esp-dsp' + 'dsp', 'esp_dsp', 'dsps', 'fft2r', 'dsps_fft2r', 'esp-dsp', + 'espressif/esp-dsp', 'espressif__esp-dsp' } if lib_name_lower in dsp_patterns: return 'espressif__esp-dsp' @@ -1034,7 +1037,7 @@ def _remove_ignored_lib_includes(self) -> None: self.logger.log_change(f"Updated build file ({total_removed} total removals)") except Exception as e: - self.logger.log_change(f"Error processing libraries: {str(e)}") + self.logger.log_change(f"Error processing libraries: {e!s} ({e.__class__.__name__})") def _batch_remove_patterns(self, content: str, libs_to_process: List[str]) -> Tuple[str, int]: """