Add custom source suffix and HTTP Headers for custom urls with authentication support via env var interpolation

portable-python.yml

@@ -29,7 +29,18 @@ cpython-additional-packages:
 # Uncomment to override cpython or a dependency source URL
 # Note: string "$version" will be replaced with version string (e.g. 1.2.3)
 # cpython-url: https://my-cpython-mirror/cpython-$version.tar.gz
+# cpython-http-headers:
+#  - Authorization: Bearer ${GITHUB_TOKEN}
 # zlib-url: https://my-zlib-mirror/zlib-$version.tar.gz
+# zlib-http-headers:
+#  - Authorization: Bearer ${GITHUB_TOKEN}
+#
+# The .tar.gz in projects public releases has additional files not present the tarball of the git tag
+# uuid-url: https://my-github-enterprise/api/v3/repos/opensource/libuuid/releases/assets/48151
+# uuid-src-suffix: .tar.gz
+# uuid-http-headers:
+#  - Authorization: Bearer ${GITHUB_TOKEN}
+#    Accept: application/octet-stream
 
 # Uncomment to override the ./configure arguments for a dependency
 # Note: this will replace the default arguments, not extend them

pyproject.toml

@@ -63,7 +63,7 @@ ignore = [
 order-by-type = false
 
 [tool.ruff.lint.mccabe]
-max-complexity = 14
+max-complexity = 18
 
 [tool.ruff.lint.pydocstyle]
 convention = "numpy"

src/portable_python/__init__.py

@@ -492,10 +492,22 @@ def is_usable_module(self, name):
     def cfg_version(self, default):
         return PPG.config.get_value("%s-version" % self.m_name) or default
 
+    def cfg_http_headers(self):
+        if config_http_headers := PPG.config.get_value("%s-http-headers" % self.m_name):
+            expanded_http_headers = {}
+            for header_dict in config_http_headers:
+                for key, value in header_dict.items():
+                    expanded_http_headers[os.path.expandvars(key)] = os.path.expandvars(value)
+            return expanded_http_headers
+
     def cfg_url(self, version):
         if config_url := PPG.config.get_value("%s-url" % self.m_name):
             url_template = Template(config_url)
-            return url_template.substitute(version=version)
+            url_subbed = url_template.substitute(version=version)
+            return os.path.expandvars(url_subbed)
+
+    def cfg_src_suffix(self):
+        return PPG.config.get_value("%s-src-suffix" % self.m_name)
 
     def cfg_configure(self, deps_lib_dir, deps_lib64_dir):
         if configure := PPG.config.get_value("%s-configure" % self.m_name):
@@ -510,6 +522,16 @@ def url(self):
         """Url of source tarball, if any"""
         return ""
 
+    @property
+    def headers(self):
+        """Headers for connecting to source url, if any"""
+        return self.cfg_http_headers()
+
+    @property
+    def src_suffix(self):
+        """Suffix of src archive for when URL doesn't end in the file extension"""
+        return self.cfg_src_suffix()
+
     @property
     def version(self):
         """Version to use"""
@@ -632,8 +654,15 @@ def compile(self):
                         self._finalize()
                         return
 
-                # Split on '#' for urls that include a checksum, such as #sha256=... fragment
-                basename = runez.basename(self.url, extension_marker="#")
+                # Some URL's may not end in file extension, such as with redirects.
+                # Github releases asset endpoint is this way .../releases/assets/48151
+                if not self.url.endswith((".zip", ".tar.gz")):
+                    suffix = self.src_suffix or ".tar.gz"
+                    basename = f"{self.m_name}-{self.version}.{suffix}"
+                else:
+                    # Split on '#' for urls that include a checksum, such as #sha256=... fragment
+                    basename = runez.basename(self.url, extension_marker="#")
+
                 path = self.setup.folders.sources / basename
                 if not path.exists():
                     proxies = {}
@@ -643,7 +672,8 @@ def compile(self):
                     https_proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy")
                     if https_proxy:
                         proxies["https"] = https_proxy
-                    RestClient().download(self.url, path, proxies=proxies)
+
+                    RestClient().download(self.url, path, proxies=proxies, headers=self.headers)
 
                 runez.decompress(path, self.m_src_build, simplify=True)
 

src/portable_python/external/__init__.py

@@ -6,7 +6,7 @@ class GettextTiny(ModuleBuilder):
 
     @property
     def url(self):
-        return f"https://github.com/sabotage-linux/gettext-tiny/archive/refs/tags/v{self.version}.tar.gz"
+        return self.cfg_url(self.version) or f"https://github.com/sabotage-linux/gettext-tiny/archive/refs/tags/v{self.version}.tar.gz"
 
     @property
     def version(self):