Don't use temp file for responsefile

Author: tannewt

ports/zephyr-cp/cptools/cpbuild.py

@@ -8,6 +8,7 @@
 import re
 import tempfile
 import time
+from typing import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -109,7 +110,14 @@ def _create_semaphore():
 max_track = 0
 
 
-async def run_command(command, working_directory, description=None, check_hash=[], extradeps=[]):
+async def run_command(
+    command,
+    working_directory,
+    description=None,
+    check_hash=[],
+    extradeps=[],
+    responsefile: Optional[pathlib.Path] = None,
+):
     """
     Runs a command asynchronously. The command should ideally be a list of strings
     and pathlib.Path objects. If all of the paths haven't been modified since the last
@@ -122,7 +130,11 @@ async def run_command(command, working_directory, description=None, check_hash=[
 
     Paths in check_hash are hashed before and after the command. If the hash is
     the same, then the old mtimes are reset. This is helpful if a command may produce
-    the same result and you don't want the rest of the build impacted.
+    the same result and you don't want the rest of the build impacted
+
+    responsefile is used to store the command line arguments if they are too long for the OS.
+    The arguments will be replaced with @<responsefile> and tried again.
+    If None, commands that are too long will fail.
     """
     paths = []
     if isinstance(command, list):
@@ -133,9 +145,16 @@ async def run_command(command, working_directory, description=None, check_hash=[
             # if isinstance(part, list):
 
             command[i] = str(part)
-        command = " ".join(command)
+        command_string = " ".join(command)
 
-    command_hash = hashlib.sha3_256(command.encode("utf-8"))
+        # When on windows, use a responsefile if the command string is >= 8192
+        if os.name == "nt" and len(command_string) >= 8192:
+            responsefile.write_text("\n".join(command[1:]))
+            command_string = f"{command[0]}@{responsefile.name}"
+    else:
+        command_string = command
+
+    command_hash = hashlib.sha3_256(command_string.encode("utf-8"))
     command_hash.update(str(working_directory).encode("utf-8"))
     command_hash = command_hash.hexdigest()
 
@@ -187,21 +206,14 @@ async def run_command(command, working_directory, description=None, check_hash=[
 
     cancellation = None
     async with shared_semaphore:
-        for i, part in enumerate(command.split()):
-            if isinstance(part, str) and part.startswith("@"):
-                print(part)
-                with open(part[1:], "r") as f:
-                    print("Contents of file:", part[1:])
-                    content = f.read()
-                    print(content)
         global max_track
         if not tracks:
             max_track += 1
             tracks.append(max_track)
         track = tracks.pop()
         start_time = time.perf_counter_ns() // 1000
         process = await asyncio.create_subprocess_shell(
-            command,
+            command_string,
             stdout=asyncio.subprocess.PIPE,
             stderr=asyncio.subprocess.PIPE,
             cwd=working_directory,
@@ -247,22 +259,23 @@ async def run_command(command, working_directory, description=None, check_hash=[
             raise cancellation
         if description:
             logger.info(f"{description} ({run_reason})")
-            logger.debug(command)
+            logger.debug(command_string)
         else:
-            logger.info(f"{command} ({run_reason})")
+            logger.info(f"{command_string} ({run_reason})")
         if old_newest_file == newest_file:
             logger.error("No files were modified by the command.")
             raise RuntimeError()
     else:
         if command_hash in LAST_BUILD_TIMES:
             del LAST_BUILD_TIMES[command_hash]
+        logger.error(command_string)
+        logger.error(f"Return code: {process.returncode}")
         if stdout:
             logger.info(stdout.decode("utf-8").strip())
         if stderr:
             logger.warning(stderr.decode("utf-8").strip())
         if not stdout and not stderr:
             logger.warning("No output")
-        logger.error(command)
         if cancellation:
             raise cancellation
         raise RuntimeError()
@@ -332,21 +345,6 @@ def __init__(self, srcdir: pathlib.Path, builddir: pathlib.Path, cmake_args):
         self.ar = cmake_args["AR"]
         self.cflags = cmake_args.get("CFLAGS", "")
 
-        self._cflags_tempfile = tempfile.NamedTemporaryFile()
-
-        # Windows paths have / as separator but gcc wants them as escaped backslashes.
-        if os.name == "nt":
-            escaped_cflags = self.cflags.replace("/", "\\\\")
-            escaped_cflags = escaped_cflags.replace(" ", "\n")
-        else:
-            escaped_cflags = self.cflags
-        print("writing to:", self._cflags_tempfile.name)
-        print(escaped_cflags)
-        self._cflags_tempfile.write(escaped_cflags.encode())
-        self._cflags_tempfile.flush()
-        self.cflags_file = "@" + self._cflags_tempfile.name
-        print("cflags_file:", self.cflags_file)
-
         self.srcdir = srcdir
         self.builddir = builddir
 
@@ -355,26 +353,24 @@ async def preprocess(
     ):
         output_file.parent.mkdir(parents=True, exist_ok=True)
         depfile = output_file.parent / (output_file.name + ".d")
-        if depfile.exists():
-            pass
+        responsefile = output_file.parent / (output_file.name + ".rsp")
+
         await run_command(
             [
                 self.c_compiler,
                 "-E",
                 "-MMD",
-                "-MF",
-                depfile,
-                "-v",
+                f"-MF {depfile}",
                 "-c",
                 source_file,
-                self.cflags_file,
+                self.cflags,
                 *flags,
-                "-o",
-                output_file,
+                f"-o {output_file}",
             ],
             description=f"Preprocess {source_file.relative_to(self.srcdir)} -> {output_file.relative_to(self.builddir)}",
             working_directory=self.srcdir,
             check_hash=[output_file],
+            responsefile=responsefile,
         )
 
     async def compile(
@@ -386,6 +382,7 @@ async def compile(
             output_file = self.builddir / output_file
         output_file.parent.mkdir(parents=True, exist_ok=True)
         depfile = output_file.with_suffix(".d")
+        responsefile = output_file.with_suffix(".rsp")
         extradeps = []
         if depfile.exists():
             depfile_contents = depfile.read_text().split()
@@ -396,10 +393,11 @@ async def compile(
                     extradeps.append(pathlib.Path(dep))
                 else:
                     extradeps.append(self.srcdir / dep)
+
         await run_command(
             [
                 self.c_compiler,
-                self.cflags_file,
+                self.cflags,
                 "-MMD",
                 "-c",
                 source_file,
@@ -410,6 +408,7 @@ async def compile(
             description=f"Compile {source_file.relative_to(self.srcdir)} -> {output_file.relative_to(self.builddir)}",
             working_directory=self.srcdir,
             extradeps=extradeps,
+            responsefile=responsefile,
         )
 
     async def archive(self, objects: list[pathlib.Path], output_file: pathlib.Path):