Cache sha1 file

meta/common/__init__.py

@@ -1,8 +1,10 @@
 import os
 import os.path
 import datetime
+import hashlib
+import sys
 from urllib.parse import urlparse
-from typing import Any, Optional
+from typing import Any, Optional, Callable
 
 import requests
 from cachecontrol import CacheControl  # type: ignore
@@ -86,3 +88,37 @@ def default_session():
     sess.headers.update({"User-Agent": "PrismLauncherMeta/1.0"})
 
     return sess
+
+
+def remove_files(file_paths: list[str]) -> None:
+    for file_path in file_paths:
+        try:
+            if os.path.isfile(file_path):
+                os.remove(file_path)
+        except Exception as e:
+            print(e)
+
+
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def file_hash(
+    filename: str, hashtype: Callable[[], "hashlib._Hash"], blocksize: int = 65536
+) -> str:
+    hashtype = hashtype()
+    with open(filename, "rb") as f:
+        for block in iter(lambda: f.read(blocksize), b""):
+            hashtype.update(block)
+    return hashtype.hexdigest()
+
+
+def get_file_sha1_from_file(file_name: str, sha1_file: str) -> Optional[str]:
+    if os.path.isfile(sha1_file):
+        with open(sha1_file, "r") as file:
+            return file.read()
+
+    new_sha1 = file_hash(file_name, hashlib.sha1)
+    with open(sha1_file, "w") as file:
+        file.write(new_sha1)
+    return None

meta/run/generate_forge.py

@@ -1,11 +1,10 @@
 import os
 import re
-import sys
 from packaging import version as pversion
 from operator import attrgetter
 from typing import Collection
 
-from meta.common import ensure_component_dir, launcher_path, upstream_path
+from meta.common import ensure_component_dir, launcher_path, upstream_path, eprint
 from meta.common.forge import (
     FORGE_COMPONENT,
     INSTALLER_MANIFEST_DIR,
@@ -44,10 +43,6 @@
 ensure_component_dir(FORGE_COMPONENT)
 
 
-def eprint(*args, **kwargs):
-    print(*args, file=sys.stderr, **kwargs)
-
-
 # Construct a set of libraries out of a Minecraft version file, for filtering.
 mc_version_cache = {}
 

meta/run/generate_neoforge.py

@@ -1,11 +1,10 @@
 from copy import deepcopy
 import os
 import re
-import sys
 from operator import attrgetter
 from typing import Collection
 
-from meta.common import ensure_component_dir, launcher_path, upstream_path
+from meta.common import ensure_component_dir, launcher_path, upstream_path, eprint
 from meta.common.neoforge import (
     NEOFORGE_COMPONENT,
     INSTALLER_MANIFEST_DIR,
@@ -38,10 +37,6 @@
 ensure_component_dir(NEOFORGE_COMPONENT)
 
 
-def eprint(*args, **kwargs):
-    print(*args, file=sys.stderr, **kwargs)
-
-
 def version_from_build_system_installer(
     installer: MojangVersion,
     profile: NeoForgeInstallerProfileV2,

meta/run/index.py

@@ -2,7 +2,9 @@
 import os
 from operator import attrgetter
 
-from meta.common import launcher_path
+from meta.common import launcher_path, file_hash
+
+
 from meta.model import MetaVersion, MetaPackage
 from meta.model.index import (
     MetaPackageIndex,
@@ -14,15 +16,6 @@
 LAUNCHER_DIR = launcher_path()
 
 
-# take the hash type (like hashlib.md5) and filename, return hex string of hash
-def hash_file(hash_fn, file_name):
-    hash_instance = hash_fn()
-    with open(file_name, "rb") as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            hash_instance.update(chunk)
-    return hash_instance.hexdigest()
-
-
 # ignore these files when indexing versions
 ignore = {"index.json", "package.json", ".git", ".github"}
 
@@ -50,7 +43,7 @@ def hash_file(hash_fn, file_name):
             continue
         # parse and hash the version file
         filepath = LAUNCHER_DIR + "/%s/%s" % (package, filename)
-        filehash = hash_file(hashlib.sha256, filepath)
+        filehash = file_hash(filepath, hashlib.sha256)
         versionFile = MetaVersion.parse_file(filepath)
         is_recommended = versionFile.version in recommendedVersions
 

meta/run/update_fabric.py

@@ -29,14 +29,6 @@
 sess = default_session()
 
 
-def filehash(filename, hashtype, blocksize=65536):
-    h = hashtype()
-    with open(filename, "rb") as f:
-        for block in iter(lambda: f.read(blocksize), b""):
-            h.update(block)
-    return h.hexdigest()
-
-
 def get_maven_url(maven_key, server, ext):
     parts = maven_key.split(":", 3)
     maven_ver_url = (

meta/run/update_forge.py

@@ -7,7 +7,6 @@
 import json
 import os
 import re
-import sys
 import zipfile
 from contextlib import suppress
 from datetime import datetime
@@ -16,7 +15,15 @@
 
 from pydantic import ValidationError
 
-from meta.common import upstream_path, ensure_upstream_dir, default_session
+from meta.common import (
+    upstream_path,
+    ensure_upstream_dir,
+    default_session,
+    remove_files,
+    eprint,
+    file_hash,
+    get_file_sha1_from_file,
+)
 from meta.common.forge import (
     JARS_DIR,
     INSTALLER_INFO_DIR,
@@ -38,6 +45,7 @@
     InstallerInfo,
     ForgeLegacyInfo,
 )
+from meta.common.http import download_binary_file
 from meta.model.mojang import MojangVersion
 
 UPSTREAM_DIR = upstream_path()
@@ -53,18 +61,6 @@
 sess = default_session()
 
 
-def eprint(*args, **kwargs):
-    print(*args, file=sys.stderr, **kwargs)
-
-
-def filehash(filename, hashtype, blocksize=65536):
-    hashtype = hashtype()
-    with open(filename, "rb") as f:
-        for block in iter(lambda: f.read(blocksize), b""):
-            hashtype.update(block)
-    return hashtype.hexdigest()
-
-
 def get_single_forge_files_manifest(longversion):
     print(f"Getting Forge manifest for {longversion}")
     path_thing = UPSTREAM_DIR + "/forge/files_manifests/%s.json" % longversion
@@ -292,6 +288,24 @@ def main():
                 UPSTREAM_DIR + "/forge/version_manifests/%s.json" % version.long_version
             )
 
+            new_sha1 = None
+            sha1_file = jar_path + ".sha1"
+            if not os.path.isfile(jar_path):
+                remove_files([profile_path, installer_info_path])
+            else:
+                fileSha1 = get_file_sha1_from_file(jar_path, sha1_file)
+                try:
+                    rfile = sess.get(version.url() + ".sha1")
+                    rfile.raise_for_status()
+                    new_sha1 = rfile.text.strip()
+                    if fileSha1 != new_sha1:
+                        remove_files(
+                            [jar_path, profile_path, installer_info_path, sha1_file]
+                        )
+                except Exception as e:
+                    eprint("Failed to check sha1 %s" % version.url())
+                    eprint("Error is %s" % e)
+
             installer_refresh_required = not os.path.isfile(
                 profile_path
             ) or not os.path.isfile(installer_info_path)
@@ -300,11 +314,18 @@ def main():
                 # grab the installer if it's not there
                 if not os.path.isfile(jar_path):
                     eprint("Downloading %s" % version.url())
-                    rfile = sess.get(version.url(), stream=True)
-                    rfile.raise_for_status()
-                    with open(jar_path, "wb") as f:
-                        for chunk in rfile.iter_content(chunk_size=128):
-                            f.write(chunk)
+                    download_binary_file(sess, jar_path, version.url())
+                    if new_sha1 is None:
+                        try:
+                            rfile = sess.get(version.url() + ".sha1")
+                            rfile.raise_for_status()
+                            new_sha1 = rfile.text.strip()
+                        except Exception as e:
+                            eprint("Failed to download new sha1 %s" % version.url())
+                            eprint("Error is %s" % e)
+                    if new_sha1 is not None:  # this is in case the fetch failed
+                        with open(sha1_file, "w") as file:
+                            file.write(new_sha1)
 
             eprint("Processing %s" % version.url())
             # harvestables from the installer
@@ -355,8 +376,8 @@ def main():
             # installer info v1
             if not os.path.isfile(installer_info_path):
                 installer_info = InstallerInfo()
-                installer_info.sha1hash = filehash(jar_path, hashlib.sha1)
-                installer_info.sha256hash = filehash(jar_path, hashlib.sha256)
+                installer_info.sha1hash = file_hash(jar_path, hashlib.sha1)
+                installer_info.sha256hash = file_hash(jar_path, hashlib.sha256)
                 installer_info.size = os.path.getsize(jar_path)
                 installer_info.write(installer_info_path)
         else:
@@ -369,11 +390,7 @@ def main():
             if not os.path.isfile(LEGACYINFO_PATH):
                 # grab the jar/zip if it's not there
                 if not os.path.isfile(jar_path):
-                    rfile = sess.get(version.url(), stream=True)
-                    rfile.raise_for_status()
-                    with open(jar_path, "wb") as f:
-                        for chunk in rfile.iter_content(chunk_size=128):
-                            f.write(chunk)
+                    download_binary_file(sess, jar_path, version.url())
                 # find the latest timestamp in the zip file
                 tstamp = datetime.fromtimestamp(0)
                 with zipfile.ZipFile(jar_path) as jar:
@@ -383,8 +400,8 @@ def main():
                             tstamp = tstamp_new
                 legacy_info = ForgeLegacyInfo()
                 legacy_info.release_time = tstamp
-                legacy_info.sha1 = filehash(jar_path, hashlib.sha1)
-                legacy_info.sha256 = filehash(jar_path, hashlib.sha256)
+                legacy_info.sha1 = file_hash(jar_path, hashlib.sha1)
+                legacy_info.sha256 = file_hash(jar_path, hashlib.sha256)
                 legacy_info.size = os.path.getsize(jar_path)
                 legacy_info_list.number[key] = legacy_info
 

meta/run/update_neoforge.py

@@ -7,7 +7,6 @@
 import json
 import os
 import re
-import sys
 import zipfile
 from contextlib import suppress
 from datetime import datetime
@@ -17,7 +16,16 @@
 
 from pydantic import ValidationError
 
-from meta.common import upstream_path, ensure_upstream_dir, default_session
+from meta.common import (
+    upstream_path,
+    ensure_upstream_dir,
+    default_session,
+    remove_files,
+    eprint,
+    file_hash,
+    get_file_sha1_from_file,
+)
+from meta.common.http import download_binary_file
 from meta.common.neoforge import (
     JARS_DIR,
     INSTALLER_INFO_DIR,
@@ -47,18 +55,6 @@
 sess = default_session()
 
 
-def eprint(*args, **kwargs):
-    print(*args, file=sys.stderr, **kwargs)
-
-
-def filehash(filename, hashtype, blocksize=65536):
-    hashtype = hashtype()
-    with open(filename, "rb") as f:
-        for block in iter(lambda: f.read(blocksize), b""):
-            hashtype.update(block)
-    return hashtype.hexdigest()
-
-
 def find_nth(haystack, needle, n):
     start = haystack.find(needle)
     while start >= 0 and n > 1:
@@ -243,6 +239,24 @@ def main():
             UPSTREAM_DIR + "/neoforge/version_manifests/%s.json" % version.long_version
         )
 
+        new_sha1 = None
+        sha1_file = jar_path + ".sha1"
+        if not os.path.isfile(jar_path):
+            remove_files([profile_path, installer_info_path])
+        else:
+            fileSha1 = get_file_sha1_from_file(jar_path, sha1_file)
+            try:
+                rfile = sess.get(version.url() + ".sha1")
+                rfile.raise_for_status()
+                new_sha1 = rfile.text.strip()
+                if fileSha1 != new_sha1:
+                    remove_files(
+                        [jar_path, profile_path, installer_info_path, sha1_file]
+                    )
+            except Exception as e:
+                eprint("Failed to check sha1 %s" % version.url())
+                eprint("Error is %s" % e)
+
         installer_refresh_required = not os.path.isfile(
             profile_path
         ) or not os.path.isfile(installer_info_path)
@@ -252,16 +266,23 @@ def main():
             if not os.path.isfile(jar_path):
                 eprint("Downloading %s" % version.url())
                 try:
-                    rfile = sess.get(version.url(), stream=True)
-                    rfile.raise_for_status()
                     Path(jar_path).parent.mkdir(parents=True, exist_ok=True)
-                    with open(jar_path, "wb") as f:
-                        for chunk in rfile.iter_content(chunk_size=128):
-                            f.write(chunk)
+                    download_binary_file(sess, jar_path, version.url())
                 except Exception as e:
                     eprint("Failed to download %s" % version.url())
                     eprint("Error is %s" % e)
                     continue
+                if new_sha1 is None:
+                    try:
+                        rfile = sess.get(version.url() + ".sha1")
+                        rfile.raise_for_status()
+                        new_sha1 = rfile.text.strip()
+                    except Exception as e:
+                        eprint("Failed to download new sha1 %s" % version.url())
+                        eprint("Error is %s" % e)
+                if new_sha1 is not None:  # this is in case the fetch failed
+                    with open(sha1_file, "w") as file:
+                        file.write(new_sha1)
 
         eprint("Processing %s" % version.url())
         # harvestables from the installer
@@ -311,8 +332,8 @@ def main():
         # installer info v1
         if not os.path.isfile(installer_info_path):
             installer_info = InstallerInfo()
-            installer_info.sha1hash = filehash(jar_path, hashlib.sha1)
-            installer_info.sha256hash = filehash(jar_path, hashlib.sha256)
+            installer_info.sha1hash = file_hash(jar_path, hashlib.sha1)
+            installer_info.sha256hash = file_hash(jar_path, hashlib.sha256)
             installer_info.size = os.path.getsize(jar_path)
             installer_info.write(installer_info_path)