Add parallel update path for prereleases. (#206)

This allows us to make prerelease releases that are available from python.org and will automatically update through subsequent prereleases until the next real release. Prereleases are not added to the Store.

Author: zooba

_msbuild.py

@@ -102,7 +102,7 @@ def main_exe(name):
         ItemDefinition('Link',
             SubSystem='CONSOLE',
             DelayLoadDLLs=f'{DLL_NAME}.dll;ole32.dll;shell32.dll;advapi32.dll',
-            DisableSpecificWarnings=Prepend('4199;'),
+            AdditionalOptions=Prepend('/IGNORE:4199 '),
         ),
         INCLUDE_TMPDIR,
         Manifest('default.manifest'),
@@ -124,7 +124,7 @@ def mainw_exe(name):
         ItemDefinition('Link',
             SubSystem='WINDOWS',
             DelayLoadDLLs=f'{DLL_NAME}.dll;ole32.dll;shell32.dll;advapi32.dll',
-            DisableSpecificWarnings=Prepend('4199;'),
+            AdditionalOptions=Prepend('/IGNORE:4199 '),
         ),
         INCLUDE_TMPDIR,
         ItemDefinition('ClCompile', PreprocessorDefinitions=Prepend(f'EXE_NAME=L"{name}";')),
@@ -296,6 +296,12 @@ def _make_xyzw_version(v, sep="."):
     return sep.join(map(str, (v.major, v.minor, micro, 0)))
 
 
+def _is_prerelease(v):
+    from packaging.version import parse
+    v = parse(v)
+    return bool(v.pre)
+
+
 def _patch_appx_identity(source, dest, **new):
     from xml.etree import ElementTree as ET
     NS = {}
@@ -335,7 +341,7 @@ def update_file(file, content):
 def init_METADATA():
     import os, re
     _, sep, version = os.getenv("BUILD_SOURCEBRANCH", os.getenv("GITHUB_REF", "")).rpartition("/")
-    if sep and "." in version:
+    if "." in version:
         from packaging.version import parse
         try:
             # Looks like a version tag
@@ -359,6 +365,8 @@ def init_PACKAGE(tag=None):
     tmpdir = get_current_build_state().temp_dir
     INCLUDE_TMPDIR.options["AdditionalIncludeDirectories"] = Prepend(f"{tmpdir};")
 
+    pre = _is_prerelease(METADATA["Version"])
+
     # GENERATE _version MODULE
     ver_py = tmpdir / "_version.py"
     update_file(ver_py, f"__version__ = {METADATA['Version']!r}")
@@ -384,6 +392,9 @@ def init_PACKAGE(tag=None):
 
     appinstaller = tmpdir / "pymanager.appinstaller"
     _patch_appinstaller(PACKAGE.find("pymanager.appinstaller").source, appinstaller,
+        # Prerelease builds use a separate online appinstaller file
+        # Release builds upload to both files, to move beta users onto final
+        AppInstallerName="pymanager-preview" if pre else "pymanager",
         Version=appx_version,
         Publisher=appx_publisher,
         Url=appx_url,

ci/upload.py

@@ -8,7 +8,7 @@
 UPLOAD_URL_PREFIX = os.getenv("UPLOAD_URL_PREFIX", "https://www.python.org/ftp/")
 UPLOAD_PATH_PREFIX = os.getenv("UPLOAD_PATH_PREFIX", "/srv/www.python.org/ftp/")
 UPLOAD_URL = os.getenv("UPLOAD_URL")
-UPLOAD_DIR = os.getenv("UPLOAD_DIR")
+UPLOAD_DIR = os.getenv("UPLOAD_DIR", "dist")
 UPLOAD_HOST = os.getenv("UPLOAD_HOST", "")
 UPLOAD_HOST_KEY = os.getenv("UPLOAD_HOST_KEY", "")
 UPLOAD_KEYFILE = os.getenv("UPLOAD_KEYFILE", "")
@@ -126,6 +126,25 @@ def url2path(url):
     return UPLOAD_PATH_PREFIX + url[len(UPLOAD_URL_PREFIX) :]
 
 
+def appinstaller_uri_matches(file, name):
+    NS = {}
+    with open(file, "r", encoding="utf-8") as f:
+        NS = dict(e for _, e in ET.iterparse(f, events=("start-ns",)))
+    for k, v in NS.items():
+        ET.register_namespace(k, v)
+    NS["x"] = NS[""]
+
+    with open(file, "r", encoding="utf-8") as f:
+        xml = ET.parse(f)
+
+    self_uri = xml.find(".[@Uri]", NS).get("Uri")
+    if not self_uri:
+        print("##[error]Empty Uri attribute in appinstaller file")
+        sys.exit(2)
+
+    return self_uri.rpartition("/")[2].casefold() == name.casefold()
+
+
 def validate_appinstaller(file, uploads):
     NS = {}
     with open(file, "r", encoding="utf-8") as f:
@@ -141,10 +160,8 @@ def validate_appinstaller(file, uploads):
     if not self_uri:
         print("##[error]Empty Uri attribute in appinstaller file")
         sys.exit(2)
-    if not any(
-        u.casefold() == self_uri.casefold() and f == file
-        for f, u, _ in uploads
-    ):
+    upload_targets = [u for f, u, _ in uploads if f == file]
+    if not any(u.casefold() == self_uri.casefold() for u in upload_targets):
         print("##[error]Uri", self_uri, "in appinstaller file is not where "
               "the appinstaller file is being uploaded.")
         sys.exit(2)
@@ -164,6 +181,8 @@ def validate_appinstaller(file, uploads):
     print(file, "checked:")
     print("-", package_uri, "is part of this upload")
     print("-", self_uri, "is the destination of this file")
+    if len(upload_targets) > 1:
+        print(" - other destinations:", *(set(upload_targets) - set([self_uri])))
     print()
 
 
@@ -185,20 +204,35 @@ def purge(url):
         u = UPLOAD_URL + f.name
         UPLOADS.append((f, u, url2path(u)))
 else:
-    for pat in ("python-manager-*.msix", "python-manager-*.msi", "pymanager.appinstaller"):
+    for pat in ("python-manager-*.msix", "python-manager-*.msi"):
         for f in UPLOAD_DIR.glob(pat):
             u = UPLOAD_URL + f.name
             UPLOADS.append((f, u, url2path(u)))
 
+    # pymanager.appinstaller is always uploaded to the pymanager-preview URL,
+    # and where the file specifies a different location, is also updated as its
+    # own filename. Later validation checks that the URL listed in the file is
+    # one of the planned uploads. If we ever need to release an update for the
+    # "main" line but not prereleases, this code would have to be modified
+    # (but more likely we'd just immediately modify or replace
+    # 'pymanager.appinstaller' on the download server).
+    f = UPLOAD_DIR / "pymanager.appinstaller"
+    if f.is_file():
+        u = UPLOAD_URL + "pymanager-preview.appinstaller"
+        UPLOADS.append((f, u, url2path(u)))
+
+        if not appinstaller_uri_matches(f, "pymanager-preview.appinstaller"):
+            u = UPLOAD_URL + f.name
+            UPLOADS.append((f, u, url2path(u)))
+
 print("Planned uploads:")
 for f, u, p in UPLOADS:
     print(f"{f} -> {p}")
     print(f"  Final URL: {u}")
 print()
 
-for f, *_ in UPLOADS:
-    if f.match("*.appinstaller"):
-        validate_appinstaller(f, UPLOADS)
+for f in {f for f, *_ in UPLOADS if f.match("*.appinstaller")}:
+    validate_appinstaller(f, UPLOADS)
 
 for f, u, p in UPLOADS:
     print("Upload", f, "to", p)

make.py

@@ -21,9 +21,8 @@
 
 ref = "none"
 try:
-    if os.getenv("BUILD_SOURCEBRANCH"):
-        ref = os.getenv("BUILD_SOURCEBRANCH")
-    else:
+    ref = os.getenv("BUILD_SOURCEBRANCH", os.getenv("GITHUB_REF", ""))
+    if not ref:
         with subprocess.Popen(
             ["git", "describe", "HEAD", "--tags"],
             stdout=subprocess.PIPE,

src/pymanager/pymanager.appinstaller

@@ -2,7 +2,7 @@
 <AppInstaller
     xmlns="http://schemas.microsoft.com/appx/appinstaller/2018"
     Version="${Version}"
-    Uri="${Url}/pymanager.appinstaller">
+    Uri="${Url}/${AppInstallerName}.appinstaller">
 
     <MainPackage
         Name="PythonSoftwareFoundation.PythonManager"