feat: finish Forge Metadata fetch

- parallelize legacy
- make loading a dotenv explicit

Signed-off-by: Rachel Powers <[email protected]>

Author: Ryex

libmcmeta/Cargo.toml

@@ -14,6 +14,7 @@ serde = { version = "1.0.160", features = ["derive"] }
 serde_json = "1.0.95"
 serde_valid = "0.15.0"
 serde_with = "2.3.2"
+time = { version = "0.3.20", features = ["serde-human-readable"] }
 tracing = "0.1.37"
 
 [features]

libmcmeta/src/models/forge.rs

@@ -4,7 +4,7 @@ use crate::models::{GradleSpecifier, MojangLibrary};
 use serde::{Deserialize, Serialize};
 use serde_valid::Validate;
 use serde_with::skip_serializing_none;
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 
 #[derive(Deserialize, Serialize, Clone, Debug, Validate)]
 pub struct ForgeMavenMetadata {
@@ -424,6 +424,8 @@ pub struct ForgeLegacyLibrary {
 #[skip_serializing_none]
 #[serde(deny_unknown_fields, rename_all = "camelCase")]
 pub struct ForgeLegacyVersionInfo {
+    #[serde(rename = "_comment_")]
+    pub comment: Option<Vec<String>>,
     pub id: String,
     pub time: String,
     pub release_time: String,
@@ -542,8 +544,8 @@ pub struct ForgeEntry {
     pub latest: Option<bool>,
     #[merge(strategy = merge::option::overwrite_some)]
     pub recommended: Option<bool>,
-    #[merge(strategy = merge::option_hashmap::recurse_some)]
-    pub files: Option<HashMap<String, ForgeFile>>,
+    #[merge(strategy = merge::option_btreemap::recurse_some)]
+    pub files: Option<BTreeMap<String, ForgeFile>>,
 }
 
 #[skip_serializing_none]
@@ -561,11 +563,11 @@ pub struct ForgeMCVersionInfo {
 #[derive(Deserialize, Serialize, Clone, Debug, Validate, Merge, Default)]
 #[serde(deny_unknown_fields)]
 pub struct DerivedForgeIndex {
-    #[merge(strategy = merge::hashmap::recurse)]
-    pub versions: HashMap<String, ForgeEntry>,
+    #[merge(strategy = merge::btreemap::recurse)]
+    pub versions: BTreeMap<String, ForgeEntry>,
     #[serde(rename = "by_mcversion")]
-    #[merge(strategy = merge::hashmap::recurse)]
-    pub by_mc_version: HashMap<String, ForgeMCVersionInfo>,
+    #[merge(strategy = merge::btreemap::recurse)]
+    pub by_mc_version: BTreeMap<String, ForgeMCVersionInfo>,
 }
 
 /// Example content
@@ -700,11 +702,11 @@ pub struct ForgeInstallerProfile {
 
 #[derive(Deserialize, Serialize, Clone, Debug, Validate, Merge, Default)]
 pub struct ForgeLegacyInfo {
-    #[serde(rename = "releaseTime")]
     #[merge(strategy = merge::option::overwrite_some)]
-    pub release_time: Option<String>,
+    #[serde(rename = "releaseTime", with = "time::serde::iso8601::option")]
+    pub release_time: Option<time::OffsetDateTime>,
     #[merge(strategy = merge::option::overwrite_some)]
-    pub size: Option<i32>,
+    pub size: Option<u64>,
     #[merge(strategy = merge::option::overwrite_some)]
     pub sha256: Option<String>,
     #[merge(strategy = merge::option::overwrite_some)]
@@ -787,7 +789,7 @@ pub struct ForgeProcessedVersion {
     pub build: i32,
     pub raw_version: String,
     pub mc_version: String,
-    pub mv_version_sane: String,
+    pub mc_version_sane: String,
     pub branch: Option<String>,
     pub installer_filename: Option<String>,
     pub installer_url: Option<String>,
@@ -803,7 +805,7 @@ impl ForgeProcessedVersion {
             build: entry.build,
             raw_version: entry.version.clone(),
             mc_version: entry.mc_version.clone(),
-            mv_version_sane: entry.mc_version.replacen("_pre", "-pre", 1),
+            mc_version_sane: entry.mc_version.replacen("_pre", "-pre", 1),
             branch: entry.branch.clone(),
             installer_filename: None,
             installer_url: None,

libmcmeta/src/models/mod.rs

@@ -465,7 +465,8 @@ pub struct MetaVersion {
     #[merge(strategy = merge::option::overwrite_some)]
     pub minecraft_arguments: Option<String>,
     #[merge(strategy = merge::option::overwrite_some)]
-    pub release_time: Option<String>,
+    #[serde(with = "time::serde::iso8601::option")]
+    pub release_time: Option<time::OffsetDateTime>,
     #[merge(strategy = merge::option_vec::append_some)]
     pub compatible_java_majors: Option<Vec<i32>>,
     #[merge(strategy = merge::option_vec::append_some)]
@@ -478,6 +479,14 @@ pub struct MetaVersion {
     pub additional_jvm_args: Option<Vec<String>>,
 }
 
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct MetaMcIndexEntry {
+    #[serde(with = "time::serde::iso8601")]
+    pub update_time: time::OffsetDateTime,
+    pub path: String,
+    pub hash: String,
+}
+
 pub mod validation {
     pub fn is_some<T>(obj: Option<T>) -> Result<(), serde_valid::validation::Error> {
         if !obj.is_some() {
@@ -549,8 +558,36 @@ pub mod merge {
         }
 
         pub fn overwrite_key<K: Eq + Hash, V>(left: &mut HashMap<K, V>, right: HashMap<K, V>) {
-            use std::collections::hash_map::Entry;
+            for (k, v) in right {
+                left.insert(k, v);
+            }
+        }
+    }
+
+    pub mod btreemap {
+        use std::collections::BTreeMap;
+        use std::hash::Hash;
+
+        pub fn recurse<K: Eq + Hash + Ord + PartialOrd, V: merge::Merge>(
+            left: &mut BTreeMap<K, V>,
+            right: BTreeMap<K, V>,
+        ) {
+            use std::collections::btree_map::Entry;
 
+            for (k, v) in right {
+                match left.entry(k) {
+                    Entry::Occupied(mut existing) => existing.get_mut().merge(v),
+                    Entry::Vacant(empty) => {
+                        empty.insert(v);
+                    }
+                }
+            }
+        }
+
+        pub fn overwrite_key<K: Eq + Hash + Ord + PartialOrd, V>(
+            left: &mut BTreeMap<K, V>,
+            right: BTreeMap<K, V>,
+        ) {
             for (k, v) in right {
                 left.insert(k, v);
             }
@@ -580,7 +617,6 @@ pub mod merge {
             left: &mut Option<HashMap<K, V>>,
             right: Option<HashMap<K, V>>,
         ) {
-            use std::collections::hash_map::Entry;
             if let Some(new) = right {
                 if let Some(original) = left {
                     hashmap::overwrite_key(original, new);
@@ -591,6 +627,38 @@ pub mod merge {
         }
     }
 
+    pub mod option_btreemap {
+        use super::btreemap;
+        use std::collections::BTreeMap;
+        use std::hash::Hash;
+
+        pub fn recurse_some<K: Eq + Hash + Ord + PartialOrd, V: merge::Merge>(
+            left: &mut Option<BTreeMap<K, V>>,
+            right: Option<BTreeMap<K, V>>,
+        ) {
+            if let Some(new) = right {
+                if let Some(original) = left {
+                    btreemap::recurse(original, new);
+                } else {
+                    *left = Some(new);
+                }
+            }
+        }
+
+        pub fn overwrite_key_some<K: Eq + Hash + Ord + PartialOrd, V>(
+            left: &mut Option<BTreeMap<K, V>>,
+            right: Option<BTreeMap<K, V>>,
+        ) {
+            if let Some(new) = right {
+                if let Some(original) = left {
+                    btreemap::overwrite_key(original, new);
+                } else {
+                    *left = Some(new);
+                }
+            }
+        }
+    }
+
     /// Merge strategies for `Option<Vec>`
     pub mod option_vec {
 

libmcmeta/src/models/mojang.rs

@@ -353,38 +353,55 @@ pub struct OldSnapshotIndex {
 pub struct LegacyOverrideEntry {
     main_class: Option<String>,
     applet_class: Option<String>,
-    release_time: Option<String>,
+    #[serde(with = "time::serde::iso8601::option")]
+    pub release_time: Option<time::OffsetDateTime>,
     #[serde(rename = "+traits")]
     additional_traits: Option<Vec<String>>,
     #[serde(rename = "+jvmArgs")]
     additional_jvm_args: Option<Vec<String>>,
 }
 
 impl LegacyOverrideEntry {
-    pub fn apply_onto_meta_version(self, meta_version: &MetaVersion, legacy: bool) {
+    pub fn apply_onto_meta_version(self, meta_version: &mut MetaVersion, legacy: bool) {
         // simply hard override classes
-        // meta_version.main_class = self.main_class
-        // meta_version.applet_class = self.applet_class
-        // # if we have an updated release time (more correct than Mojang), use it
-        // if self.release_time:
-        //     meta_version.release_time = self.release_time
-
-        // # add traits, if any
-        // if self.additional_traits:
-        //     if not meta_version.additional_traits:
-        //         meta_version.additional_traits = []
-        //     meta_version.additional_traits += self.additional_traits
-
-        // if self.additional_jvm_args:
-        //     if not meta_version.additional_jvm_args:
-        //         meta_version.additional_jvm_args = []
-        //     meta_version.additional_jvm_args += self.additional_jvm_args
-
-        // if legacy:
-        //     # remove all libraries - they are not needed for legacy
-        //     meta_version.libraries = None
-        //     # remove minecraft arguments - we use our own hardcoded ones
-        //     meta_version.minecraft_arguments = None
+
+        meta_version.main_class = self.main_class.clone();
+        meta_version.applet_class = self.applet_class.clone();
+
+        // if we have an updated release time (more correct than Mojang), use it
+        if let Some(release_time) = &self.release_time {
+            meta_version.release_time = Some(*release_time);
+        }
+
+        // add traits, if any
+        if let Some(mut additional_traits) = self.additional_traits {
+            if !meta_version.additional_traits.is_some() {
+                meta_version.additional_traits = Some(vec![]);
+            }
+            meta_version
+                .additional_traits
+                .as_mut()
+                .unwrap()
+                .append(&mut additional_traits);
+        }
+
+        if let Some(mut additional_jvm_args) = self.additional_jvm_args {
+            if !meta_version.additional_jvm_args.is_some() {
+                meta_version.additional_jvm_args = Some(vec![]);
+            }
+            meta_version
+                .additional_jvm_args
+                .as_mut()
+                .unwrap()
+                .append(&mut additional_jvm_args);
+        }
+
+        if legacy {
+            // remove all libraries - they are not needed for legacy
+            meta_version.libraries = None;
+            // remove minecraft arguments - we use our own hardcoded ones
+            meta_version.minecraft_arguments = None;
+        }
     }
 }
 
@@ -394,14 +411,15 @@ pub struct LegacyOverrideIndex {
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone, Validate)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct LibraryPatch {
     #[serde(rename = "match")]
     pub patch_match: Vec<GradleSpecifier>,
     #[serde(rename = "override")]
     pub patch_override: Option<Library>,
-    pub additionalLibraries: Option<Vec<Library>>,
+    pub additional_libraries: Option<Vec<Library>>,
     #[serde(default = "default_library_patch_patch_additional_libraries")]
-    pub patchAdditionalLibraries: bool,
+    pub patch_additional_libraries: bool,
 }
 
 fn default_library_patch_patch_additional_libraries() -> bool {
@@ -435,6 +453,7 @@ impl Deref for LibraryPatches {
 pub struct MojangArgumentObject {}
 
 #[derive(Deserialize, Serialize, Debug, Clone, Validate)]
+#[serde(untagged)]
 pub enum MojangArgument {
     String(String),
     Object(MojangArgumentObject),
@@ -477,6 +496,8 @@ fn mojang_logging_validate_type(
 #[derive(Deserialize, Serialize, Debug, Clone, Validate)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct MojangVersion {
+    #[serde(rename = "_comment_")]
+    pub comment: Option<Vec<String>>,
     pub id: String, // TODO: optional?
     pub arguments: Option<MojangArguments>,
     pub asset_index: Option<MojangAssets>,
@@ -489,8 +510,10 @@ pub struct MojangVersion {
     pub minecraft_arguments: Option<String>,
     #[validate(custom(mojang_version_validate_minimum_launcher_version))]
     pub minimum_launcher_version: Option<i32>,
-    pub release_time: Option<String>,
-    pub time: Option<String>,
+    #[serde(with = "time::serde::iso8601::option")]
+    pub release_time: Option<time::OffsetDateTime>,
+    #[serde(with = "time::serde::iso8601::option")]
+    pub time: Option<time::OffsetDateTime>,
     #[serde(rename = "type")]
     pub version_type: Option<String>,
     pub inherits_from: Option<String>,
@@ -533,7 +556,7 @@ impl MojangVersion {
         let mut main_jar = None;
         let mut addn_traits = None;
         let mut new_type = self.version_type.clone();
-        let mut compatible_java_majors = None;
+        let mut compatible_java_majors;
         if !self.id.is_empty() {
             let downloads = self.downloads.clone().expect("Missing downloads");
             let client_download = downloads

mcmeta/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-anyhow = "1.0.70"
+anyhow = { version = "1.0.70", features = ["backtrace"] }
 argparse = "0.2.2"
 axum = "0.6.15"
 clap = "4.2.1"
@@ -28,6 +28,7 @@ sha1 = "0.10.5"
 sha2 = "0.10.6"
 tempdir = "0.3.7"
 thiserror = "1.0.40"
+time = "0.3.20"
 tokio = { version = "1.27.0", features = ["full"] }
 tracing = "0.1.37"
 tracing-appender = "0.2.2"

mcmeta/src/main.rs

@@ -6,7 +6,7 @@ use axum::{routing::get, Extension, Router};
 use tracing::{debug, info};
 
 use anyhow::Result;
-use argparse::{ArgumentParser, Store};
+use argparse::{ArgumentParser, Store, StoreTrue};
 use dotenv::dotenv;
 use tracing_subscriber::{filter, prelude::*};
 
@@ -22,9 +22,8 @@ extern crate lazy_static;
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    dotenv().ok(); // This line loads the environment variables from the ".env" file.
-
     let mut config_path = "".to_string();
+    let mut use_dotenv = false;
     {
         // limit scope of argparse borrows
         let mut ap = ArgumentParser::new();
@@ -34,9 +33,18 @@ async fn main() -> Result<()> {
             Store,
             "Path to a json config file.",
         );
+        ap.refer(&mut use_dotenv).add_option(
+            &["--use-dotenv"],
+            StoreTrue,
+            "Load environment variables from a local .env",
+        );
         ap.parse_args_or_exit();
     }
 
+    if use_dotenv {
+        dotenv().ok();
+    }
+
     let config = Arc::new(ServerConfig::from_config(&config_path)?);
 
     let file_appender =