Merge branch 'fix/small-limit-for-kconfserver' into 'master'

fix: allow larger buffer size for commands (especially for idf.py confserver)

Closes IDFCI-3569 and IDF-13919

See merge request espressif/esp-idf!42312

Author: dobairoland

tools/idf_py_actions/core_ext.py

@@ -46,6 +46,37 @@ def build_target(target_name: str, ctx: Context, args: PropertyDict) -> None:
         ensure_build_directory(args, ctx.info_name)
         run_target(target_name, args, force_progression=GENERATORS[args.generator].get('force_progression', False))
 
+    def confserver_target(target_name: str, ctx: Context, args: PropertyDict, buffer_size: int) -> None:
+        """
+        Execute the idf.py confserver command with the specified buffer size.
+        """
+        ensure_build_directory(args, ctx.info_name)
+        if buffer_size < 2048:
+            yellow_print(
+                f'WARNING: The specified buffer size {buffer_size} KB is less than the '
+                'recommended minimum of 2048 KB for idf.py confserver. Consider increasing it to at least 2048 KB '
+                'by setting environment variable IDF_CONFSERVER_BUFFER_SIZE=<buffer size in KB> or by calling '
+                'idf.py confserver --buffer-size <buffer size in KB>.'
+            )
+        try:
+            run_target(
+                target_name,
+                args,
+                force_progression=GENERATORS[args.generator].get('force_progression', False),
+                buffer_size=buffer_size,
+            )
+        except ValueError as e:
+            if str(e) == 'Separator is not found, and chunk exceed the limit':
+                # Buffer size too small/one-line output of the command too long
+                raise FatalError(
+                    f'ERROR: Command failed with an error message "{e}". '
+                    'Try increasing the buffer size to 2048 (or higher) by setting environment variable '
+                    'IDF_CONFSERVER_BUFFER_SIZE=<buffer size in KB> or by calling '
+                    'idf.py confserver --buffer-size <buffer size in KB>.'
+                )
+            else:
+                raise
+
     def size_target(
         target_name: str, ctx: Context, args: PropertyDict, output_format: str, output_file: str, diff_map_file: str
     ) -> None:
@@ -481,9 +512,22 @@ def help_and_exit(action: str, ctx: Context, param: list, json_option: bool, add
                 ],
             },
             'confserver': {
-                'callback': build_target,
+                'callback': confserver_target,
                 'help': 'Run JSON configuration server.',
-                'options': global_options,
+                'options': global_options
+                + [
+                    {
+                        'names': ['--buffer-size'],
+                        'help': (
+                            'Set the buffer size (in KB) in order to accommodate initial confserver response.'
+                            'Default value and recommended minimum is 2048 (KB), but it might need to be '
+                            'increased for very large projects.'
+                        ),
+                        'type': int,
+                        'default': 2048,
+                        'envvar': 'IDF_CONFSERVER_BUFFER_SIZE',
+                    }
+                ],
             },
             'size': {
                 'callback': size_target,

tools/idf_py_actions/tools.py

@@ -298,6 +298,7 @@ def __init__(
         force_progression: bool = False,
         interactive: bool = False,
         convert_output: bool = False,
+        buffer_size: int | None = None,
     ) -> None:
         self.tool_name = tool_name
         self.args = args
@@ -310,6 +311,7 @@ def __init__(
         self.force_progression = force_progression
         self.interactive = interactive
         self.convert_output = convert_output
+        self.buffer_size = buffer_size or 256
 
     def __call__(self) -> None:
         def quote_arg(arg: str) -> str:
@@ -368,7 +370,7 @@ async def run_command(self, cmd: list, env_copy: dict) -> tuple[Process, str | N
             p = await asyncio.create_subprocess_exec(
                 *cmd,
                 env=env_copy,
-                limit=1024 * 1024,
+                limit=1024 * self.buffer_size,
                 cwd=self.cwd,
                 stdout=asyncio.subprocess.PIPE,
                 stderr=asyncio.subprocess.PIPE,
@@ -520,6 +522,7 @@ def run_target(
     custom_error_handler: FunctionType | None = None,
     force_progression: bool = False,
     interactive: bool = False,
+    buffer_size: int | None = None,
 ) -> None:
     """Run target in build directory."""
     if env is None:
@@ -546,6 +549,7 @@ def run_target(
         hints=not args.no_hints,
         force_progression=force_progression,
         interactive=interactive,
+        buffer_size=buffer_size,
     )()