Common out Gradle/Maven properties file patching in our Python scripts

Author: steve-s

graalpython/com.oracle.graal.python.test/src/tests/standalone/test_gradle_plugin.py

@@ -98,8 +98,7 @@ def generate_app(self, target_dir):
                 if file.endswith(".j"):
                     shutil.move(os.path.join(root, file), os.path.join(root, file[0:len(file)- 1] + "java"))
 
-        util.patch_properties_file(os.path.join(target_dir, "gradle", "wrapper", "gradle-wrapper.properties"), self.env.get("GRADLE_DISTRIBUTION_URL_OVERRIDE"))
-
+        util.override_gradle_properties_file(target_dir)
         self.copy_build_files(target_dir)
 
     def check_filelist(self, target_dir, log, check_lib=True):

graalpython/com.oracle.graal.python.test/src/tests/standalone/test_maven_plugin.py

@@ -72,11 +72,11 @@ def generate_app(self, tmpdir, target_dir, target_name, pom_template=None):
         shutil.copy(os.path.join(mvnw_dir, "mvnw"), os.path.join(target_dir, "mvnw"))
         shutil.copy(os.path.join(mvnw_dir, "mvnw.cmd"), os.path.join(target_dir, "mvnw.cmd"))
         shutil.copytree(os.path.join(mvnw_dir, ".mvn"), os.path.join(target_dir, ".mvn"))
-        util.patch_properties_file(os.path.join(target_dir, ".mvn", "wrapper", "maven-wrapper.properties"), self.env.get("MAVEN_DISTRIBUTION_URL_OVERRIDE"))
+        util.override_maven_properties_file(target_dir)
 
     @unittest.skipUnless(util.is_maven_plugin_test_enabled, "ENABLE_MAVEN_PLUGIN_UNITTESTS is not true")
     def test_generated_app(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
+        with util.TemporaryTestDirectory() as tmpdir:
             target_name = "generated_app_test"
             target_dir = os.path.join(str(tmpdir), target_name)
             self.generate_app(tmpdir, target_dir, target_name)
@@ -140,7 +140,7 @@ def test_generated_app(self):
 
     @unittest.skipUnless(util.is_maven_plugin_test_enabled, "ENABLE_MAVEN_PLUGIN_UNITTESTS is not true")
     def test_generated_app_external_resources(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
+        with util.TemporaryTestDirectory() as tmpdir:
             target_name = "generated_app_ext_resources_test"
             target_dir = os.path.join(str(tmpdir), target_name)
             self.generate_app(tmpdir, target_dir, target_name)
@@ -193,7 +193,7 @@ def test_generated_app_external_resources(self):
 
     @unittest.skipUnless(util.is_maven_plugin_test_enabled, "ENABLE_MAVEN_PLUGIN_UNITTESTS is not true")
     def test_fail_without_graalpy_dep(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
+        with util.TemporaryTestDirectory() as tmpdir:
             target_name = "fail_without_graalpy_dep_test"
             target_dir = os.path.join(str(tmpdir), target_name)
             pom_template = os.path.join(os.path.dirname(__file__), "fail_without_graalpy_dep_pom.xml")
@@ -207,7 +207,7 @@ def test_fail_without_graalpy_dep(self):
 
     @unittest.skipUnless(util.is_maven_plugin_test_enabled, "ENABLE_MAVEN_PLUGIN_UNITTESTS is not true")
     def test_gen_launcher_and_venv(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
+        with util.TemporaryTestDirectory() as tmpdir:
             target_name = "gen_launcher_and_venv_test"
             target_dir = os.path.join(str(tmpdir), target_name)
             pom_template = os.path.join(os.path.dirname(__file__), "prepare_venv_pom.xml")
@@ -247,7 +247,7 @@ def check_tagfile(self, target_dir, expected):
 
     @unittest.skipUnless(util.is_maven_plugin_test_enabled, "ENABLE_MAVEN_PLUGIN_UNITTESTS is not true")
     def test_check_home(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
+        with util.TemporaryTestDirectory() as tmpdir:
             target_name = "check_home_test"
             target_dir = os.path.join(str(tmpdir), target_name)
             pom_template = os.path.join(os.path.dirname(__file__), "check_home_pom.xml")
@@ -340,7 +340,7 @@ def test_check_home(self):
 
     @unittest.skipUnless(util.is_maven_plugin_test_enabled, "ENABLE_MAVEN_PLUGIN_UNITTESTS is not true")
     def test_empty_packages(self):
-        with tempfile.TemporaryDirectory() as tmpdir:
+        with util.TemporaryTestDirectory() as tmpdir:
             target_name = "empty_packages_test"
             target_dir = os.path.join(str(tmpdir), target_name)
             pom_template = os.path.join(os.path.dirname(__file__), "check_packages_pom.xml")

graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py

@@ -333,15 +333,12 @@ def replace_in_file(file, str, replace_str):
     with open(file, "w") as f:
         f.write(contents.replace(str, replace_str))
 
-def patch_properties_file(properties_file, distribution_url_override):
-    if distribution_url_override:
-        new_lines = []
-        with(open(properties_file)) as f:
-            while line := f.readline():
-                line.strip()
-                if not line.startswith("#") and "distributionUrl" in line:
-                    new_lines.append(f"distributionUrl={distribution_url_override}\n")
-                else:
-                    new_lines.append(line)
-        with(open(properties_file, "w")) as f:
-            f.writelines(new_lines)
+
+def override_gradle_properties_file(gradle_project_root):
+    if override_file:=os.environ.get('GRADLE_PROPERTIES_OVERRIDE'):
+        shutil.copy(override_file, os.path.join(gradle_project_root, "gradle", "wrapper", "gradle-wrapper.properties"))
+
+
+def override_maven_properties_file(maven_project_root):
+    if override_file:=os.environ.get('MAVEN_PROPERTIES_OVERRIDE'):
+        shutil.copy(override_file, os.path.join(maven_project_root, ".mvn", "wrapper", "maven-wrapper.properties"))

mx.graalpython/mx_graalpython.py

@@ -41,6 +41,7 @@
 from pathlib import Path
 from textwrap import dedent
 
+import mx_graalpython_gradleproject
 import mx_urlrewrites
 
 if sys.version_info[0] < 3:
@@ -1276,9 +1277,11 @@ def graalpython_gate_runner(args, tasks):
                 mx_urlrewrites.rewriteurl('https://repo1.maven.org/maven2/'),
             ])
 
-            urls = get_wrapper_urls("graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties", ["distributionUrl"])
-            if "distributionUrl" in urls:
-                env["GRADLE_DISTRIBUTION_URL_OVERRIDE"] = mx_urlrewrites.rewriteurl(urls["distributionUrl"])
+            override_path = os.path.join(SUITE.get_mx_output_dir(), 'gradle-properties-override')
+            original_props_file = "graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties"
+            mx.copyfile(original_props_file, override_path)
+            mx_graalpython_gradleproject.patch_distributionUrl(override_path, original_props_file)
+            env['GRADLE_PROPERTIES_OVERRIDE'] = override_path
 
             env["org.graalvm.maven.downloader.version"] = version
             env["org.graalvm.maven.downloader.repository"] = f"{pathlib.Path(mvn_repo_path).as_uri()}/"
@@ -1309,9 +1312,11 @@ def graalpython_gate_runner(args, tasks):
                 mx_urlrewrites.rewriteurl('https://repo1.maven.org/maven2/'),
             ])
 
-            urls = get_wrapper_urls("graalpython/com.oracle.graal.python.test/src/tests/standalone/mvnw/.mvn/wrapper/maven-wrapper.properties", ["distributionUrl"])
-            if "distributionUrl" in urls:
-                env["MAVEN_DISTRIBUTION_URL_OVERRIDE"] = mx_urlrewrites.rewriteurl(urls["distributionUrl"])
+            override_path = os.path.join(SUITE.get_mx_output_dir(), 'maven-properties-override')
+            original_props_file = "graalpython/com.oracle.graal.python.test/src/tests/standalone/mvnw/.mvn/wrapper/maven-wrapper.properties"
+            mx.copyfile(original_props_file, override_path)
+            mx_graalpython_gradleproject.patch_distributionUrl(override_path, original_props_file, escape_colon=False)
+            env['MAVEN_PROPERTIES_OVERRIDE'] = override_path
 
             env["org.graalvm.maven.downloader.version"] = version
             env["org.graalvm.maven.downloader.repository"] = f"{pathlib.Path(mvn_repo_path).as_uri()}/"

mx.graalpython/mx_graalpython_gradleproject.py

@@ -48,6 +48,36 @@
 import mx_urlrewrites
 
 
+def gradle_wrapper_properties(project_root):
+    return os.path.join(project_root, "gradle", "wrapper", "gradle-wrapper.properties")
+
+
+def patch_distributionUrl(properties_file, original_file=None, escape_colon=True):
+    def escape(s):
+        return s.replace(':', '\\:') if escape_colon else s
+
+    def unescape(s):
+        return s.replace('\\:', ':') if escape_colon else s
+
+    original_file = original_file if original_file else properties_file
+    with open(properties_file, 'r') as f:
+        wrapper_properties = [l.strip() for l in f.readlines()]
+    found = False
+    for i, line in enumerate(wrapper_properties):
+        if line.strip().startswith('distributionUrl='):
+            url = unescape(line[len('distributionUrl='):])
+            new_url = mx_urlrewrites.rewriteurl(url)
+            mx.logv("Rewritten Gradle distribution URL to: " + new_url)
+            wrapper_properties[i] = 'distributionUrl=' + escape(new_url)
+            found = True
+            break
+    if not found:
+        mx.abort("Could not find 'distributionUrl' in " + original_file)
+    mx.logvv(f'Patched gradle-wrapper.properties to:\n' + '\n'.join(wrapper_properties))
+    with open(properties_file, 'w') as f:
+        f.write('\n'.join(wrapper_properties) + '\n')
+
+
 class GradlePluginProject(mx.Distribution, mx.ClasspathDependency):  # pylint: disable=too-many-instance-attributes
     """
     Mx project and distribution for Gradle plugins projects. These have properties of both mx Java projects and
@@ -274,24 +304,8 @@ def _create_build_script(self, version: str | None = None):
         """
         # Gradle wrapper
         shutil.copytree(os.path.join(self.subject.gradle_directory, "wrapper"), self.subject.get_output_root(), dirs_exist_ok=True)
-        properties_path = os.path.join("gradle", "wrapper", "gradle-wrapper.properties")
-        wrapper_properties_file = os.path.join(self.subject.get_output_root(), properties_path)
-        with open(wrapper_properties_file, 'r') as f:
-            wrapper_properties = [l.strip() for l in f.readlines()]
-        found = False
-        for i, line in enumerate(wrapper_properties):
-            if line.strip().startswith('distributionUrl='):
-                url = line[len('distributionUrl='):].replace('\\:', ':')
-                new_url = mx_urlrewrites.rewriteurl(url)
-                mx.logv("Rewritten Gradle distribution URL to: " + new_url)
-                wrapper_properties[i] = 'distributionUrl=' + new_url.replace(':', '\\:')
-                found = True
-                break
-        if not found:
-            mx.abort("Could not find 'distributionUrl' in " + os.path.join(self.subject.gradle_directory, "wrapper", properties_path))
-        mx.logvv(f'Patched {properties_path} to:\n' + '\n'.join(wrapper_properties))
-        with open(wrapper_properties_file, 'w') as f:
-            f.write('\n'.join(wrapper_properties))
+        patch_distributionUrl(gradle_wrapper_properties(self.subject.get_output_root()),
+                             original_file=gradle_wrapper_properties(os.path.join(self.subject.gradle_directory, "wrapper")))
 
         # build.gradle
         deps_decls = []
@@ -423,4 +437,4 @@ def _create_build_script(self, version: str | None = None):
         with open(settings_path, 'w+') as f:
             f.write(settings)
         mx.logv(f"Gradle settings written to {settings_path}")
-        mx.logvv(settings)
+        mx.logvv(settings)