Merge branch 'master' into test/flagstests

Author: bdbaddog

CHANGES.txt

@@ -24,6 +24,8 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
   From Mats Wichmann:
     - Clean up C and C++ FLAGS tests. Tests which use a real compiler
       are now more clearly distinguished (-live.py suffix and docstring).
+    - runtest.py once again finds "external" tests, such as the tests for
+      tools in scons-contrib. An earlier rework had broken this.  Fixes #4699.
 
 
 RELEASE 4.9.1 -  Thu, 27 Mar 2025 11:40:20 -0700
@@ -33,6 +35,9 @@ RELEASE 4.9.1 -  Thu, 27 Mar 2025 11:40:20 -0700
     - Fixed a hang in `wait_for_process_to_die()` on Windows, affecting
       clean-up of the SCons daemon used for Ninja builds.
 
+  From Thaddeus Crews:
+    - Nodes are now treated as PathLike objects.
+
   From Mats Wichmann:
     - Fix typos in CCFLAGS test. Didn't affect the test itself, but
       didn't correctly apply the DefaultEnvironment speedup.

RELEASE.txt

@@ -29,6 +29,8 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY
 - List modifications to existing features, where the previous behavior
   wouldn't actually be considered a bug
 
+- Nodes are now treated as PathLike objects.
+
 FIXES
 -----
 
@@ -57,7 +59,8 @@ DOCUMENTATION
 DEVELOPMENT
 -----------
 
-- List visible changes in the way SCons is developed
+- runtest.py once again finds "external" tests, such as the tests for
+  tools in scons-contrib. An earlier rework had broken this.  Fixes #4699.
 
 Thanks to the following contributors listed below for their contributions to this release.
 ==========================================================================================

SCons/Builder.py

@@ -114,6 +114,7 @@
 from SCons.Debug import logInstanceCreation
 from SCons.Errors import InternalError, UserError
 from SCons.Executor import Executor
+from SCons.Node import Node
 
 class _Null:
     pass
@@ -487,10 +488,11 @@ def _adjustixes(self, files, pre, suf, ensure_suffix: bool=False):
             # fspath() is to catch PathLike paths. We avoid the simpler
             # str(f) so as not to "lose" files that are already Nodes:
             # TypeError: expected str, bytes or os.PathLike object, not File
-            with suppress(TypeError):
-                f = os.fspath(f)
-            if SCons.Util.is_String(f):
-                f = SCons.Util.adjustixes(f, pre, suf, ensure_suffix)
+            if not isinstance(f, Node):
+                with suppress(TypeError):
+                    f = os.fspath(f)
+                if SCons.Util.is_String(f):
+                    f = SCons.Util.adjustixes(f, pre, suf, ensure_suffix)
             result.append(f)
         return result
 

SCons/BuilderTests.py

@@ -702,7 +702,7 @@ def test_single_source(self) -> None:
         """Test Builder with single_source flag set"""
         def func(target, source, env) -> None:
             """create the file"""
-            with open(str(target[0]), "w"):
+            with open(target[0], "w"):
                 pass
             if len(source) == 1 and len(target) == 1:
                 env['CNT'][0] = env['CNT'][0] + 1
@@ -759,7 +759,7 @@ def test_lists(self) -> None:
         """Testing handling lists of targets and source"""
         def function2(target, source, env, tlist = [outfile, outfile2], **kw) -> int:
             for t in target:
-                with open(str(t), 'w') as f:
+                with open(t, 'w') as f:
                     f.write("function2\n")
             for t in tlist:
                 if t not in list(map(str, target)):
@@ -790,7 +790,7 @@ def function2(target, source, env, tlist = [outfile, outfile2], **kw) -> int:
 
         def function3(target, source, env, tlist = [sub1_out, sub2_out]) -> int:
             for t in target:
-                with open(str(t), 'w') as f:
+                with open(t, 'w') as f:
                     f.write("function3\n")
             for t in tlist:
                 if t not in list(map(str, target)):

SCons/Environment.xml

@@ -1248,7 +1248,7 @@ env.Command(
 
 import os
 def rename(env, target, source):
-    os.rename('.tmp', str(target[0]))
+    os.rename('.tmp', target[0])
 
 
 env.Command(
@@ -3626,7 +3626,7 @@ def create(target, source, env):
 
     Writes 'prefix=$SOURCE' into the file name given as $TARGET.
     """
-    with open(str(target[0]), 'wb') as f:
+    with open(target[0], 'wb') as f:
         f.write(b'prefix=' + source[0].get_contents() + b'\n')
 
 # Fetch the prefix= argument, if any, from the command line.

SCons/Node/__init__.py

@@ -622,6 +622,9 @@ def __init__(self) -> None:
         # what line in what file created the node, for example).
         Annotate(self)
 
+    def __fspath__(self) -> str:
+        return str(self)
+
     def disambiguate(self, must_exist: bool = False):
         return self
 

SCons/SConfTests.py

@@ -295,7 +295,7 @@ def test_TryAction(self) -> None:
         """Test SConf.TryAction
         """
         def actionOK(target, source, env):
-            with open(str(target[0]), "w") as f:
+            with open(target[0], "w") as f:
                 f.write("RUN OK\n")
             return None
         def actionFAIL(target, source, env) -> int:

bin/SConsExamples.py

@@ -547,7 +547,7 @@ def process(source_file, ofp):
                 elif line[:11] != "STRIP CCCOM":
                     ofp.write(line)
 
-    with open(str(target[0]), "w") as fp:
+    with open(target[0], "w") as fp:
         for src in map(str, source):
             process(src, fp)
             fp.write('debug = ' + ARGUMENTS.get('debug', '0') + '\\n')

runtest.py

@@ -29,7 +29,6 @@
 from io import StringIO
 from pathlib import Path, PurePath, PureWindowsPath
 from queue import Queue
-from typing import TextIO
 
 cwd = os.getcwd()
 debug: str | None = None
@@ -575,38 +574,34 @@ def footer(self, f):
     del os.environ['_JAVA_OPTIONS']
 
 
-# ---[ test discovery ]------------------------------------
-# This section figures out which tests to run.
+# ---[ Test Discovery ]------------------------------------
+# This section determines which tests to run based on three
+# mutually exclusive options:
+# 1. Reading test paths from a testlist file (--file or --retry option)
+# 2. Using test paths given as command line arguments
+# 3. Automatically finding all tests (--all option)
 #
-# The initial testlist is made by reading from the testlistfile,
-# if supplied, or by looking at the test arguments, if supplied,
-# or by looking for all test files if the "all" argument is supplied.
-# One of the three is required.
+# Test paths can specify either individual test files, or directories to
+# scan for tests. The following test types are recognized:
 #
-# Each test path, whichever of the three sources it comes from,
-# specifies either a test file or a directory to search for
-# SCons tests. SCons code layout assumes that any file under the 'SCons'
-# subdirectory that ends with 'Tests.py' is a unit test, and any Python
-# script (*.py) under the 'test' subdirectory is an end-to-end test.
-# We need to track these because they are invoked differently.
-# find_unit_tests and find_e2e_tests are used for this searching.
+# - Unit tests: Files ending in 'Tests.py' under the 'SCons' directory
+# - End-to-end tests: Python scripts (*.py) under the 'test' directory
+# - External tests: End-to-end tests in paths containing a 'test'
+#   component (not expected to be local)
 #
-# Note that there are some tests under 'SCons' that *begin* with
-# 'test_', but they're packaging and installation tests, not
-# functional tests, so we don't execute them by default.  (They can
-# still be executed by hand, though).
+# find_unit_tests() and find_e2e_tests() perform the directory scanning.
 #
-# Test exclusions, if specified, are then applied.
-
+# After the initial test list is built, any test exclusions specified via
+# --exclude-list are applied to produce the final test set.
 
 def scanlist(testfile):
     """ Process a testlist file """
     data = StringIO(testfile.read_text())
     tests = [t.strip() for t in data.readlines() if not t.startswith('#')]
     # in order to allow scanned lists to work whether they use forward or
-    # backward slashes, first create the object as a PureWindowsPath which
-    # accepts either, then use that to make a Path object to use for
-    # comparisons like "file in scanned_list".
+    # backward slashes, on non-Windows first create the object as a
+    # PureWindowsPath which accepts either, then use that to make a Path
+    # object for use in comparisons like "if file in scanned_list".
     if sys.platform == 'win32':
         return [Path(t) for t in tests if t]
     else:
@@ -635,7 +630,7 @@ def find_e2e_tests(directory):
         if 'sconstest.skip' in filenames:
             continue
 
-        # Slurp in any tests in exclude lists
+        # Gather up the data from any exclude lists
         excludes = []
         if ".exclude_tests" in filenames:
             excludefile = Path(dirpath, ".exclude_tests").resolve()
@@ -648,8 +643,7 @@ def find_e2e_tests(directory):
     return sorted(result)
 
 
-# initial selection:
-# if we have a testlist file read that, else hunt for tests.
+# Initial test selection:
 unittests = []
 endtests = []
 if args.testlistfile:
@@ -668,15 +662,16 @@ def find_e2e_tests(directory):
         # Clean up path removing leading ./ or .\
         name = str(path)
         if name.startswith('.') and name[1] in (os.sep, os.altsep):
-            path = path.with_name(tn[2:])
+            path = path.with_name(name[2:])
 
         if path.exists():
             if path.is_dir():
                 if path.parts[0] == "SCons" or path.parts[0] == "testing":
                     unittests.extend(find_unit_tests(path))
                 elif path.parts[0] == 'test':
                     endtests.extend(find_e2e_tests(path))
-                # else: TODO: what if user pointed to a dir outside scons tree?
+                elif args.external and 'test' in path.parts:
+                    endtests.extend(find_e2e_tests(path))
             else:
                 if path.match("*Tests.py"):
                     unittests.append(path)
@@ -703,7 +698,7 @@ def find_e2e_tests(directory):
 """)
     sys.exit(1)
 
-# ---[ test processing ]-----------------------------------
+# ---[ Test Processing ]-----------------------------------
 tests = [Test(t) for t in tests]
 
 if args.list_only:
@@ -826,10 +821,11 @@ def run_test(t, io_lock=None, run_async=True):
 
 
 class RunTest(threading.Thread):
-    """ Test Runner class.
+    """Test Runner thread.
 
-    One instance will be created for each job thread in multi-job mode
+    One will be created for each job in multi-job mode
     """
+
     def __init__(self, queue=None, io_lock=None, group=None, target=None, name=None):
         super().__init__(group=group, target=target, name=name)
         self.queue = queue

test/Actions/append.py

@@ -45,13 +45,13 @@
 env=Environment()
 
 def before(env, target, source):
-    with open(str(target[0]), "wb") as f:
+    with open(target[0], "wb") as f:
         f.write(b"Foo\\n")
     with open("before.txt", "wb") as f:
         f.write(b"Bar\\n")
 
 def after(env, target, source):
-    with open(str(target[0]), "rb") as fin, open("after%s", "wb") as fout:
+    with open(target[0], "rb") as fin, open("after%s", "wb") as fout:
         fout.write(fin.read())
 
 env.Prepend(LINKCOM=Action(before))