Merge pull request #205 from pbashyal-nmdp/cli_improvements

Verify correct IMGT DB version

Author: mmaiers-nmdp

pyard/__init__.py

@@ -24,6 +24,7 @@
 from .pyard import ARD
 from .blender import blender as dr_blender
 from .broad_splits import find_splits as find_broad_splits
+from .misc import get_imgt_db_versions as db_versions
 
 __author__ = """NMDP Bioinformatics"""
 __version__ = "0.9.1"

pyard/db.py

@@ -22,8 +22,11 @@
 #
 import pathlib
 import sqlite3
+import sys
 from typing import Tuple, Dict, Set, List
 
+from pyard.misc import get_imgt_db_versions
+
 
 def get_pyard_db_install_directory():
     return pathlib.Path.home() / ".pyard"
@@ -49,17 +52,25 @@ def create_db_connection(data_dir, imgt_version, ro=False):
         # If in read-only mode, make sure the db file exists
         if not pathlib.Path(db_filename).exists():
             raise RuntimeError(f"Reference Database {db_filename}  not available.")
-
-    # Create the data directory if it doesn't exist
-    if not pathlib.Path(data_dir).exists():
-        pathlib.Path(data_dir).mkdir(parents=True, exist_ok=True)
-
-    if ro:
         # Open the database in read-only mode
         file_uri = f"file:{db_filename}?mode=ro"
         # Multiple threads can access the same connection since it's only ro
         return sqlite3.connect(file_uri, check_same_thread=False, uri=True)
 
+    # Check the imgt_version is a valid IMGT DB Version
+    # by querying the IMGT site
+    if imgt_version != "Latest":
+        if not pathlib.Path(db_filename).exists():
+            all_imgt_versions = get_imgt_db_versions()
+            if imgt_version not in all_imgt_versions:
+                raise ValueError(
+                    f"{imgt_version} is not a valid IMGT database version."
+                )
+
+    # Create the data directory if it doesn't exist
+    if not pathlib.Path(data_dir).exists():
+        pathlib.Path(data_dir).mkdir(parents=True, exist_ok=True)
+
     # Open the database for read/write
     file_uri = f"file:{db_filename}"
     return sqlite3.connect(file_uri, uri=True)

pyard/misc.py

@@ -1,4 +1,9 @@
 # List of expression characters
+import pathlib
+from typing import List
+
+from pyard import db
+
 expression_chars = ["N", "Q", "L", "S"]
 # List of P and G characters
 PandG_chars = ["P", "G"]
@@ -64,3 +69,28 @@ def get_P_name(a: str) -> str:
     if last_char in PandG_chars + expression_chars:
         a = a[:-1]
     return ":".join(a.split(":")[0:2]) + "P"
+
+
+def get_imgt_db_versions() -> List[str]:
+    import urllib.request
+    import json
+
+    req = urllib.request.Request(
+        url="https://api.github.com/repos/ANHIG/IMGTHLA/branches?per_page=100"
+    )
+    res = urllib.request.urlopen(req, timeout=5)
+    if res.status == 200:
+        json_body = json.loads(res.read())
+        versions = list(map(lambda x: x["name"], json_body))
+        return versions
+
+
+def get_data_dir(data_dir):
+    if data_dir:
+        path = pathlib.Path(data_dir)
+        if not path.exists() or not path.is_dir():
+            raise RuntimeError(f"{data_dir} is not a valid directory")
+        data_dir = path
+    else:
+        data_dir = db.get_pyard_db_install_directory()
+    return data_dir

pyard/pyard.py

@@ -147,7 +147,8 @@ def __del__(self):
         Close the db connection, when ARD instance goes away
         :return:
         """
-        self.db_connection.close()
+        if hasattr(self, "db_connection") and self.db_connection:
+            self.db_connection.close()
 
     @functools.lru_cache(maxsize=max_cache_size)
     def redux(self, allele: str, redux_type: VALID_REDUCTION_TYPES, reping=True) -> str:

scripts/pyard-import

@@ -23,10 +23,11 @@
 #
 import argparse
 import pathlib
+import sys
 
 import pyard
 from pyard import db, data_repository
-import pandas as pd
+from pyard.misc import get_data_dir
 
 
 def get_imgt_version(imgt_version):
@@ -40,18 +41,10 @@ def get_imgt_version(imgt_version):
     return "Latest"
 
 
-def get_data_dir(data_dir):
-    if data_dir:
-        path = pathlib.Path(data_dir)
-        if not path.exists() or not path.is_dir():
-            raise RuntimeError(f"{data_dir} is not a valid directory")
-    else:
-        data_dir = db.get_pyard_db_install_directory()
-    return data_dir
-
-
 def get_v2_v3_mapping(v2_v3_mapping):
     if v2_v3_mapping:
+        import pandas as pd
+
         path = pathlib.Path(v2_v3_mapping)
         if not path.exists() or not path.is_file():
             raise RuntimeError(f"{data_dir} is not a valid file")
@@ -62,21 +55,52 @@ def get_v2_v3_mapping(v2_v3_mapping):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        usage="""[--db-version <IMGT DB Version>]\n
-        [--data-dir <directory for db file>]\n
-        [--v2-to-v3-mapping <V2 to V3 mapping CSV file>]""",
         description="""
         py-ard tool to generate reference SQLite database.
         Allows updating db with custom V2 to V3 mappings.
+        Displays the list of available IMGT database versions.
         """,
     )
-    parser.add_argument("--db-version", dest="imgt_version")
-    parser.add_argument("--data-dir", dest="data_dir")
-    parser.add_argument("--v2-to-v3-mapping", dest="v2_v3_mapping")
-    parser.add_argument("--refresh-mac", dest="refresh_mac", action="store_true")
-    parser.add_argument("--re-install", dest="reinstall", action="store_true")
+    parser.add_argument(
+        "--list",
+        dest="show_versions",
+        action="store_true",
+        help="Show Versions of available IMGT Databases",
+    )
+    parser.add_argument(
+        "--db-version",
+        dest="imgt_version",
+        help="Import supplied IMGT_VERSION DB Version",
+    )
+    parser.add_argument(
+        "--data-dir",
+        dest="data_dir",
+        help="Data directory to store imported data",
+    )
+    parser.add_argument(
+        "--v2-to-v3-mapping", dest="v2_v3_mapping", help="V2 to V3 mapping CSV file"
+    )
+    parser.add_argument(
+        "--refresh-mac",
+        dest="refresh_mac",
+        action="store_true",
+        help="Only refresh MAC data",
+    )
+    parser.add_argument(
+        "--re-install",
+        dest="reinstall",
+        action="store_true",
+        help="reinstall a fresh version of database",
+    )
     args = parser.parse_args()
 
+    if args.show_versions:
+        versions = pyard.db_versions()
+        print("Available IMGT Versions:")
+        for version in versions:
+            print(f"  {version}")
+        sys.exit(0)
+
     imgt_version = get_imgt_version(args.imgt_version)
     # print(imgt_version)
 
@@ -94,8 +118,13 @@ if __name__ == "__main__":
             db_fullname.unlink(missing_ok=True)
 
     print(f"Importing IMGT database version: {imgt_version}")
-    ard = pyard.ARD(imgt_version=imgt_version, data_dir=data_dir)
+    try:
+        ard = pyard.ARD(imgt_version=imgt_version, data_dir=data_dir)
+    except ValueError as e:
+        print(f"Error importing version {imgt_version}:", e)
+        sys.exit(1)
     print(f"Import complete for database version: {imgt_version}")
+    # We don't need ard object anymore
     del ard
 
     if v2_to_v3_dict:

scripts/pyard-status

@@ -23,33 +23,34 @@
 #
 import argparse
 import os
-import pathlib
 import re
 
+import pyard
 from pyard import db, data_repository
+from pyard.misc import get_data_dir
 
 
-def get_data_dir(data_dir):
-    if data_dir:
-        path = pathlib.Path(data_dir)
-        if not path.exists() or not path.is_dir():
-            raise RuntimeError(f"{data_dir} is not a valid directory")
-        data_dir = path
-    else:
-        data_dir = db.get_pyard_db_install_directory()
-    return data_dir
+def get_latest_imgt_version() -> int:
+    """
+    Gets the list of db versions and returns the maximum
+    version numbered db
+    @return: int
+    """
+    return max(map(int, pyard.db_versions()[:-1]))
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        usage="""
-        [--data-dir <directory for db file>]\n
-        """,
         description="""
         py-ard tool to provide a status report for reference SQLite databases.
         """,
     )
-    parser.add_argument("--data-dir", dest="data_dir")
+    parser.add_argument(
+        "--data-dir",
+        dest="data_dir",
+        help="Data directory to store imported data",
+    )
+
     args = parser.parse_args()
     data_dir = get_data_dir(args.data_dir)
     # print(data_dir)
@@ -61,9 +62,24 @@ if __name__ == "__main__":
             # eg: get 3440 from 'pyard-3440.sqlite3'
             match = imgt_regex.match(filename)
             imgt_version = match.group(1)  # Get first group
-            db_connection = db.create_db_connection(data_dir, imgt_version)
+            db_connection = db.create_db_connection(data_dir, imgt_version, ro=True)
             print("-" * 43)
-            print(f"IMGT DB Version: {imgt_version}")
+            if imgt_version == "Latest":
+                db_version = data_repository.get_db_version(db_connection)
+                print(f"IMGT DB Version: {imgt_version} ({db_version})")
+                latest_version = get_latest_imgt_version()
+                if latest_version == db_version:
+                    print(
+                        f"You're up to date. {db_version} is the most recent version."
+                    )
+                else:
+                    print(f"There is a newer IMGT release than version {db_version}")
+                    print(
+                        f"Upgrade to latest version '{latest_version}'",
+                        "with 'pyard-import --re-install'",
+                    )
+            else:
+                print(f"IMGT DB Version: {imgt_version}")
             print("-" * 43)
             print(f"|{'Table Name':20}|{'Rows':20}|")
             print(f"|{'-' * 41}|")