Merge branch 'main' of github.com:TravisWheelerLab/mdrepo-rs

Author: kyclark

Cargo.lock

@@ -154,7 +154,7 @@ pub const TRAJECTORY_FILE_EXTS: &[&str] = &[
 // - .vasp / POSCAR — VASP structure format
 // - .config / .cfg — DL_POLY config file
 // - .lmp — LAMMPS structure (alternate extension)
-pub const STRUCTURE_FILE_EXTS: &[&str] = &["pdb"];
+pub const STRUCTURE_FILE_EXTS: &[&str] = &["pdb", "gro"];
 
 // Here are common topology file extensions used in molecular dynamics simulations:
 // GROMACS

libmdrepo/src/constants.rs

@@ -7,7 +7,7 @@ use crate::schema::*;
 // ── helpers ───────────────────────────────────────────────────────────────────
 
 fn limit(n: Option<i64>) -> i64 {
-    n.unwrap_or(20).min(1000)
+    n.unwrap_or(200).min(1000)
 }
 
 // ── md_contribution ───────────────────────────────────────────────────────────
@@ -1695,7 +1695,10 @@ pub fn list_social_accounts(
     Ok((count, results))
 }
 
-pub fn get_social_account(conn: &mut PgConnection, rid: i32) -> QueryResult<SocialAccount> {
+pub fn get_social_account(
+    conn: &mut PgConnection,
+    rid: i32,
+) -> QueryResult<SocialAccount> {
     socialaccount_socialaccount::table
         .find(rid)
         .select(SocialAccount::as_select())
@@ -1766,7 +1769,10 @@ pub fn get_social_app(conn: &mut PgConnection, rid: i32) -> QueryResult<SocialAp
         .first(conn)
 }
 
-pub fn insert_social_app(conn: &mut PgConnection, new: NewSocialApp) -> QueryResult<SocialApp> {
+pub fn insert_social_app(
+    conn: &mut PgConnection,
+    new: NewSocialApp,
+) -> QueryResult<SocialApp> {
     diesel::insert_into(socialaccount_socialapp::table)
         .values(&new)
         .returning(SocialApp::as_returning())
@@ -1818,7 +1824,10 @@ pub fn list_social_app_sites(
     Ok((count, results))
 }
 
-pub fn get_social_app_site(conn: &mut PgConnection, rid: i64) -> QueryResult<SocialAppSite> {
+pub fn get_social_app_site(
+    conn: &mut PgConnection,
+    rid: i64,
+) -> QueryResult<SocialAppSite> {
     socialaccount_socialapp_sites::table
         .find(rid)
         .select(SocialAppSite::as_select())

mdr-db/src/ops.rs

@@ -92,28 +92,53 @@ fn run(args: Args) -> Result<()> {
     let mut trajectory_file_name: Option<String> = None;
     let mut structure_file_name: Option<String> = None;
     let mut topology_file_name: Option<String> = None;
-    for up_file in up_files.iter().filter(|f| f.is_primary) {
-        match up_file.file_type.as_str() {
-            "Trajectory" => trajectory_file_name = Some(up_file.filename.clone()),
-            "Structure" => structure_file_name = Some(up_file.filename.clone()),
-            "Topology" => topology_file_name = Some(up_file.filename.clone()),
-            _ => additional_files.push(metadata::AdditionalFile {
-                file_name: up_file.filename.clone(),
-                file_type: up_file.file_type.clone(),
-                description: up_file.description.clone(),
-            }),
+
+    for file in up_files {
+        if file.is_primary {
+            match file.file_type.as_str() {
+                "Trajectory" => {
+                    if trajectory_file_name.is_none() {
+                        trajectory_file_name = Some(file.filename.clone())
+                    } else {
+                        bail!("Found multiple primary trajectory files");
+                    }
+                }
+                "Structure" => {
+                    if structure_file_name.is_none() {
+                        structure_file_name = Some(file.filename.clone())
+                    } else {
+                        bail!("Found multiple primary structure files");
+                    }
+                }
+                "Topology" => {
+                    if topology_file_name.is_none() {
+                        topology_file_name = Some(file.filename.clone())
+                    } else {
+                        bail!("Found multiple primary topology files");
+                    }
+                }
+                _ => bail!(
+                    r#"Primary file type "{}" is not Trajectory/Structure/Topology"#,
+                    file.file_type
+                ),
+            }
+        } else {
+            additional_files.push(metadata::AdditionalFile {
+                file_name: file.filename.clone(),
+                file_type: file.file_type.clone(),
+                description: file.description.clone(),
+            })
         }
     }
 
-    if ![
-        &trajectory_file_name,
-        &structure_file_name,
-        &topology_file_name,
-    ]
-    .iter()
-    .all(|val| val.is_some())
-    {
-        bail!("Missing one of required files!")
+    for (field, val) in [
+        ("trajectory_file_name", &trajectory_file_name),
+        ("structure_file_name", &structure_file_name),
+        ("topology_file_name", &topology_file_name),
+    ] {
+        if val.is_none() {
+            bail!(r#""{field}" has no value"#)
+        }
     }
 
     let (_, ligands_res) =
@@ -258,6 +283,11 @@ fn run(args: Args) -> Result<()> {
         contributors,
     };
 
+    let errors = meta.check();
+    if !errors.is_empty() {
+        bail!("Errors!\n{}", errors.join("\n"));
+    }
+
     let mut out_file = open_outfile(&args.outfile)?;
     let format = args.format.clone().unwrap_or(guess_format(&args.outfile));
     write!(

mdr-gen-meta/src/main.rs

@@ -7,7 +7,6 @@ edition = "2024"
 anyhow = "1.0.100"
 chksum-md5 = "0.1.0"
 clap = { version = "4", features = ["derive"] }
-dotenv = "0.15.0"
 env_logger = "0.11.8"
 log = "0.4.29"
 multiset = "0.0.5"
@@ -21,6 +20,7 @@ urlencoding = "2.1.3"
 validator = { version = "0.20.0", features = ["derive"] }
 which = "8.0.0"
 libmdrepo = { path = "../libmdrepo" }
+dotenvy = "0.15.7"
 
 [dev-dependencies]
 assert_cmd = "2.1.2"

mdr-process/Cargo.toml

@@ -4,6 +4,7 @@ use crate::types::{
     UniprotEntry, UniprotResponse,
 };
 use anyhow::{anyhow, bail, Result};
+use dotenvy::dotenv;
 use libmdrepo::{
     common::{file_exists, get_md5, read_file},
     metadata::{self, Meta},
@@ -24,9 +25,7 @@ use which::which;
 // --------------------------------------------------
 pub fn process(args: &ProcessArgs) -> Result<()> {
     debug!("{args:?}");
-
-    // It's OK if there's no .env
-    let _ = dotenv::dotenv();
+    dotenv().ok();
     let input_dir = path::absolute(&args.dirname)?;
     let processed_dir = args
         .out_dir
@@ -403,19 +402,6 @@ pub fn sample_trajectory(
     Ok(())
 }
 
-// --------------------------------------------------
-//pub fn db_connection(server: &Server) -> Result<postgres::Client> {
-//    dotenv().ok();
-//    let env_key = match server {
-//        Server::Production => "PRODUCTION_DB_URL",
-//        Server::Staging => "STAGING_DB_URL",
-//    };
-//    dbg!(&env_key);
-//    let db_url = env::var(env_key).expect(&format!("{env_key} must be set"));
-//    postgres::Client::connect(&db_url, postgres::NoTls)
-//        .map_err(|_| anyhow!("Failed db connection"))
-//}
-
 // --------------------------------------------------
 pub fn make_import_json(
     meta_path: &PathBuf,

mdr-process/src/process.rs

@@ -3,6 +3,7 @@ use crate::{
     types::{ProcessArgs, ReprocessArgs},
 };
 use anyhow::{bail, Result};
+use dotenvy::dotenv;
 use libmdrepo::{common::file_exists, metadata::Meta};
 use log::{debug, info};
 use std::{
@@ -13,24 +14,27 @@ use std::{
 
 // --------------------------------------------------
 pub fn reprocess(args: &ReprocessArgs) -> Result<()> {
-    let _ = dotenv::dotenv();
+    dotenv().ok();
     let simulation_id = args.simulation_id;
     let mdrepo_id = format!("MDR{simulation_id:08}");
     debug!("Reprocessing simulation ID {mdrepo_id}");
 
     let data_dir = &args.work_dir.join(&mdrepo_id);
     if !data_dir.is_dir() {
-        fs::create_dir_all(&args.work_dir)?;
+        fs::create_dir_all(&data_dir)?;
     }
 
     let server = &args.server;
     let irods_dir =
         format!("/iplant/home/shared/mdrepo/{server}/release/{mdrepo_id}/original");
     let irods_dir = Path::new(&irods_dir);
+    debug!(r#"irods_dir = "{}""#, irods_dir.display());
 
-    let meta_path = data_dir.join("mdrepo-metadata.toml");
-    let meta = Meta::from_file(&meta_path)?;
+    let meta_filename = "mdrepo-metadata.toml";
+    let meta_local_path = data_dir.join(meta_filename);
+    irods_fetch(&irods_dir.join(meta_filename), &meta_local_path)?;
 
+    let meta = Meta::from_file(&meta_local_path)?;
     for filename in &[
         &meta.trajectory_file_name,
         &meta.structure_file_name,
@@ -67,21 +71,26 @@ pub fn reprocess(args: &ReprocessArgs) -> Result<()> {
 
 // --------------------------------------------------
 fn irods_fetch(irods_path: &Path, local_path: &PathBuf) -> Result<()> {
-    if !file_exists(local_path) {
-        debug!(
-            r#"Get "{}" -> "{}""#,
-            irods_path.display(),
-            local_path.display()
-        );
-        let cmd = Command::new("gocmd")
-            .args([
-                "get",
-                irods_path.to_string_lossy().as_ref(),
-                local_path.to_string_lossy().as_ref(),
-            ])
-            .output()?;
-        if !cmd.status.success() {
-            bail!(str::from_utf8(&cmd.stderr)?.to_string());
+    debug!(
+        r#"Get "{}" -> "{}""#,
+        irods_path.display(),
+        local_path.display()
+    );
+
+    if file_exists(local_path) {
+        debug!("Already downloaded");
+    } else {
+        let mut cmd = Command::new("gocmd");
+        cmd.args([
+            "get",
+            irods_path.to_string_lossy().as_ref(),
+            local_path.to_string_lossy().as_ref(),
+        ]);
+        debug!("Running {cmd:?}");
+        let output = cmd.output()?;
+
+        if !output.status.success() {
+            bail!(str::from_utf8(&output.stderr)?.to_string());
         }
     }
     Ok(())

mdr-process/src/reprocess.rs

@@ -3,14 +3,15 @@ use crate::{
     types::{ProcessArgs, TicketArgs},
 };
 use anyhow::{anyhow, bail, Result};
+use dotenvy::dotenv;
 use log::{debug, info};
 use std::{env, fs, path::PathBuf, process::Command};
 use which::which;
 
 // --------------------------------------------------
 pub fn process(args: &TicketArgs) -> Result<()> {
     debug!("{args:?}");
-    let _ = dotenv::dotenv();
+    dotenv().ok();
     let script_dir = &args.script_dir.clone().unwrap_or(PathBuf::from(
         env::var("SCRIPT_DIR").map_err(|e| anyhow!("SCRIPT_DIR: {e}"))?,
     ));

mdr-process/src/ticket.rs

@@ -122,11 +122,11 @@ pub struct TicketArgs {
 #[command(alias = "re")]
 pub struct ReprocessArgs {
     /// Simulation ID
-    #[arg(value_name = "INT")]
+    #[arg(short, long, value_name = "INT")]
     pub simulation_id: u32,
 
     /// Script directory
-    #[arg(short('S'), long, value_name = "SCRIPTS")]
+    #[arg(long, value_name = "SCRIPTS")]
     pub script_dir: Option<PathBuf>,
 
     // TODO: Think of adding "server" to this path?
@@ -144,7 +144,7 @@ pub struct ReprocessArgs {
     pub json_dir: Option<PathBuf>,
 
     /// Server
-    #[arg(short, long, value_name = "SERVER", default_value = "staging")]
+    #[arg(short('S'), long, value_name = "SERVER", default_value = "staging")]
     pub server: Server,
 }