Add support for recursive response files (#25390)

Fixes: #25387

Author: sbc100

test/common.py

@@ -1328,6 +1328,10 @@ def parse_wasm(self, filename):
         funcs.append(name)
     return imports, exports, funcs
 
+  def output_name(self, basename):
+    suffix = get_output_suffix(self.get_cflags())
+    return basename + suffix
+
   @classmethod
   def setUpClass(cls):
     super().setUpClass()

test/test_core.py

@@ -4135,10 +4135,6 @@ def do_basic_dylink_test(self, **kwargs):
       }
     ''', 'other says 11.', 'int sidey();', force_c=True, **kwargs)
 
-  def output_name(self, basename):
-    suffix = common.get_output_suffix(self.get_cflags())
-    return basename + suffix
-
   @needs_dylink
   @crossplatform
   def test_dylink_basics(self):
@@ -7275,15 +7271,6 @@ def test(expected, args=None, assert_returncode=0):
     test('|1|')
     test('|1|', args=['-DDIRECT'])
 
-  def test_response_file(self):
-    out_js = self.output_name('response_file')
-    response_data = '-o "%s" "%s"' % (out_js, test_file('hello_world.cpp'))
-    create_file('rsp_file', response_data.replace('\\', '\\\\'))
-    self.run_process([EMCC, "@rsp_file"] + self.get_cflags())
-    self.do_run(out_js, 'hello, world', no_build=True)
-
-    self.assertContained('response file not found: foo.txt', self.expect_fail([EMCC, '@foo.txt']))
-
   def test_linker_response_file(self):
     objfile = 'response_file.o'
     out_js = self.output_name('response_file')

test/test_other.py

@@ -9424,8 +9424,8 @@ def test_emar_response_file(self):
     create_file("file'2", ' ')
     create_file("hyvää päivää", ' ')
     create_file("snowman freezes covid ☃ 🦠", ' ')
-    rsp = response_file.create_response_file(("file'1", "file'2", "hyvää päivää", "snowman freezes covid ☃ 🦠"), shared.TEMP_DIR)
-    building.emar('cr', 'libfoo.a', ['@' + rsp])
+    create_file("tmp.rsp", response_file.create_response_file_contents(("file'1", "file'2", "hyvää päivää", "snowman freezes covid ☃ 🦠")))
+    building.emar('cr', 'libfoo.a', ['@tmp.rsp'])
 
   def test_response_file_bom(self):
     # Modern CMake version create response fils in UTF-8 but with BOM
@@ -13003,6 +13003,16 @@ def create_o(name, i):
     self.run_process(building.get_command_with_possible_response_file([EMCC, 'main.c'] + files))
     self.assertContained(str(count * (count - 1) // 2), self.run_js('a.out.js'))
 
+  @crossplatform
+  def test_response_file(self):
+    out_js = self.output_name('response_file')
+    response_data = '-o "%s" "%s"' % (out_js, test_file('hello_world.cpp'))
+    create_file('rsp_file', response_data.replace('\\', '\\\\'))
+    self.run_process([EMCC, "@rsp_file"] + self.get_cflags())
+    self.do_run(out_js, 'hello, world', no_build=True)
+
+    self.assertContained('emcc: error: @foo.txt: No such file or directory', self.expect_fail([EMCC, '@foo.txt']))
+
   # Tests that the filename suffix of the response files can be used to detect which encoding the file is.
   @crossplatform
   def test_response_file_encoding(self):
@@ -13020,6 +13030,13 @@ def test_response_file_encoding(self):
     open('a.rsp', 'w', encoding=preferred_encoding).write('äö.c') # Write a response file using Python preferred encoding
     self.run_process([EMCC, '@a.rsp']) # ... and test that it is properly autodetected.
 
+  @crossplatform
+  def test_response_file_recursive(self):
+    create_file('rsp2.txt', response_file.create_response_file_contents([test_file('hello_world.c'), '-o', 'hello.js']))
+    create_file('rsp1.txt', '@rsp2.txt\n')
+    self.run_process([EMCC, '@rsp1.txt'])
+    self.assertContained('hello, world!', self.run_js('hello.js'))
+
   def test_output_name_collision(self):
     # Ensure that the secondary filenames never collide with the primary output filename
     # In this case we explicitly ask for JS to be created in a file with the `.wasm` suffix.

tools/response_file.py

@@ -13,14 +13,9 @@
 DEBUG = int(os.environ.get('EMCC_DEBUG', '0'))
 
 
-def create_response_file(args, directory):
-  """Routes the given cmdline param list in args into a new response file and
-  returns the filename to it.
+def create_response_file_contents(args):
+  """Create response file contents based on list of arguments.
   """
-
-  response_fd, response_filename = tempfile.mkstemp(prefix='emscripten_', suffix='.rsp.utf-8', dir=directory, text=True)
-
-  # Backslashes and other special chars need to be escaped in the response file.
   escape_chars = ['\\', '\"']
   # When calling llvm-ar on Linux and macOS, single quote characters ' should be escaped.
   if not WINDOWS:
@@ -40,6 +35,18 @@ def escape(arg):
       arg = '"%s"' % arg
     contents += arg + '\n'
 
+  return contents
+
+
+def create_response_file(args, directory):
+  """Routes the given cmdline param list in args into a new response file and
+  returns the filename to it.
+  """
+  # Backslashes and other special chars need to be escaped in the response file.
+  contents = create_response_file_contents(args)
+
+  response_fd, response_filename = tempfile.mkstemp(prefix='emscripten_', suffix='.rsp.utf-8', dir=directory, text=True)
+
   with os.fdopen(response_fd, 'w', encoding='utf-8') as f:
     f.write(contents)
 
@@ -54,7 +61,7 @@ def escape(arg):
   return response_filename
 
 
-def read_response_file(response_filename):
+def expand_response_file(arg):
   """Reads a response file, and returns the list of cmdline params found in the
   file.
 
@@ -63,12 +70,19 @@ def read_response_file(response_filename):
   specified, first UTF-8 and then Python locale.getpreferredencoding() are
   attempted.
 
-  The parameter response_filename may start with '@'."""
-  if response_filename.startswith('@'):
-    response_filename = response_filename[1:]
+  The parameter `arg` is the command line argument to be expanded."""
+
+  if arg.startswith('@'):
+    response_filename = arg[1:]
+  elif arg.startswith('-Wl,@'):
+    response_filename = arg[5:]
+  else:
+    response_filename = None
 
-  if not os.path.exists(response_filename):
-    raise OSError("response file not found: %s" % response_filename)
+  # Is the argument is not a response file, or if the file does not exist
+  # just return orginal argument.
+  if not response_filename or not os.path.exists(response_filename):
+    return [arg]
 
   # Guess encoding based on the file suffix
   components = os.path.basename(response_filename).split('.')
@@ -97,20 +111,13 @@ def read_response_file(response_filename):
   if DEBUG:
     logging.warning(f'read response file {response_filename}: {args}')
 
-  return args
+  # Response file can be recursive so call substitute_response_files on the arguments
+  return substitute_response_files(args)
 
 
 def substitute_response_files(args):
   """Substitute any response files found in args with their contents."""
   new_args = []
   for arg in args:
-    if arg.startswith('@'):
-      new_args += read_response_file(arg)
-    elif arg.startswith('-Wl,@'):
-      for a in read_response_file(arg[5:]):
-        if a.startswith('-'):
-          a = '-Wl,' + a
-        new_args.append(a)
-    else:
-      new_args.append(arg)
+    new_args += expand_response_file(arg)
   return new_args