Merge pull request #218 from pbashyal-nmdp/update_for_performance

Performance Updates

Author: mmaiers-nmdp

api-spec.yaml

@@ -2,7 +2,7 @@ openapi: 3.0.3
 info:
   title: ARD Reduction
   description: Reduce to ARD Level
-  version: "1.0.0rc2"
+  version: "1.0.0rc3"
 servers:
   - url: 'http://localhost:8080'
 tags:

api.py

@@ -17,7 +17,7 @@ def validate_controller():
             return {"message": "gl_string not provided"}, 404
         # Validate
         try:
-            ard.isvalid_gl(gl_string)
+            ard.validate(gl_string)
             return {"valid": True}, 200
         except InvalidAlleleError as e:
             return {

extras/reduce_conf.json

@@ -54,5 +54,7 @@
   "map_drb345_to_drbx": false,
   "apply_compression": "gzip",
   "generate_glstring": true,
+  "redux_cache_size": 1000,
+  "reduced_column_prefix": "reduced_",
   "verbose_log": true
 }

pyard/__init__.py

@@ -27,7 +27,7 @@
 from .misc import get_imgt_db_versions as db_versions
 
 __author__ = """NMDP Bioinformatics"""
-__version__ = "1.0.0rc2"
+__version__ = "1.0.0rc3"
 
 
 def init(

pyard/ard.py

@@ -26,7 +26,7 @@
 import sys
 from typing import Iterable, List
 
-from . import broad_splits
+from . import broad_splits, smart_sort
 from . import data_repository as dr
 from . import db
 from .exceptions import InvalidAlleleError, InvalidMACError, InvalidTypingError
@@ -41,7 +41,6 @@
     expression_chars,
     DEFAULT_CACHE_SIZE,
 )
-from .smart_sort import smart_sort_comparator
 
 default_config = {
     "reduce_serology": True,
@@ -56,6 +55,7 @@
     "verbose_log": True,
 }
 
+
 # Typing information
 
 
@@ -95,17 +95,16 @@ def __init__(
         self.ars_mappings = dr.generate_ars_mapping(self.db_connection, imgt_version)
         # Load Alleles and XX Codes
         (
-            self.valid_alleles,
-            self.who_alleles,
-            self.xx_codes,
-            self.who_group,
-            self.exp_alleles,
+            self.code_mappings,
+            self.allele_group,
         ) = dr.generate_alleles_and_xx_codes_and_who(
             self.db_connection, imgt_version, self.ars_mappings
         )
 
         # Generate short nulls from WHO mapping
-        self.shortnulls = dr.generate_short_nulls(self.db_connection, self.who_group)
+        self.shortnulls = dr.generate_short_nulls(
+            self.db_connection, self.code_mappings.who_group
+        )
 
         # Load Serology mappings
         broad_splits.broad_splits_ser_mapping = (
@@ -128,6 +127,12 @@ def __init__(
                 self._redux_allele
             )
             self.redux = functools.lru_cache(maxsize=max_cache_size)(self.redux)
+            self.is_mac = functools.lru_cache(maxsize=max_cache_size)(self.is_mac)
+            self.smart_sort_comparator = functools.lru_cache(maxsize=max_cache_size)(
+                smart_sort.smart_sort_comparator
+            )
+        else:
+            self.smart_sort_comparator = smart_sort.smart_sort_comparator
 
         # reference data is read-only and can be frozen
         # Works only for Python >= 3.9
@@ -213,8 +218,10 @@ def _redux_allele(
             # new redux_type which is full WHO expansion
             if self._is_who_allele(allele):
                 return allele
-            if allele in self.who_group:
-                return self.redux("/".join(self.who_group[allele]), redux_type)
+            if allele in self.code_mappings.who_group:
+                return self.redux(
+                    "/".join(self.code_mappings.who_group[allele]), redux_type
+                )
             else:
                 return allele
         elif redux_type == "exon":
@@ -254,8 +261,7 @@ def _redux_allele(
             else:
                 raise InvalidAlleleError(f"{allele} is an invalid allele.")
 
-    @staticmethod
-    def sorted_unique_gl(gls: List[str], delim: str) -> str:
+    def _sorted_unique_gl(self, gls: List[str], delim: str) -> str:
         """
         Make a list of sorted unique GL Strings separated by delim.
         As the list may itself contains elements that are separated by the
@@ -272,7 +278,7 @@ def sorted_unique_gl(gls: List[str], delim: str) -> str:
         if delim == "+":
             # No need to make unique. eg. homozygous cases are valid for SLUGs
             return delim.join(
-                sorted(gls, key=functools.cmp_to_key(smart_sort_comparator))
+                sorted(gls, key=functools.cmp_to_key(self.smart_sort_comparator))
             )
 
         # generate a unique list over a delimiter
@@ -282,7 +288,7 @@ def sorted_unique_gl(gls: List[str], delim: str) -> str:
             all_gls += gl.split(delim)
         unique_gls = set(all_gls)
         return delim.join(
-            sorted(unique_gls, key=functools.cmp_to_key(smart_sort_comparator))
+            sorted(unique_gls, key=functools.cmp_to_key(self.smart_sort_comparator))
         )
 
     @functools.lru_cache(maxsize=DEFAULT_CACHE_SIZE)
@@ -302,28 +308,28 @@ def redux(self, glstring: str, redux_type: VALID_REDUCTION_TYPES) -> str:
 
         self.validate(glstring)
 
-        if re.search(r"\^", glstring):
-            return self.sorted_unique_gl(
+        if "^" in glstring:
+            return self._sorted_unique_gl(
                 [self.redux(a, redux_type) for a in glstring.split("^")], "^"
             )
 
-        if re.search(r"\|", glstring):
-            return self.sorted_unique_gl(
+        if "|" in glstring:
+            return self._sorted_unique_gl(
                 [self.redux(a, redux_type) for a in glstring.split("|")], "|"
             )
 
-        if re.search(r"\+", glstring):
-            return self.sorted_unique_gl(
+        if "+" in glstring:
+            return self._sorted_unique_gl(
                 [self.redux(a, redux_type) for a in glstring.split("+")], "+"
             )
 
-        if re.search("~", glstring):
-            return self.sorted_unique_gl(
+        if "~" in glstring:
+            return self._sorted_unique_gl(
                 [self.redux(a, redux_type) for a in glstring.split("~")], "~"
             )
 
-        if re.search("/", glstring):
-            return self.sorted_unique_gl(
+        if "/" in glstring:
+            return self._sorted_unique_gl(
                 [self.redux(a, redux_type) for a in glstring.split("/")], "/"
             )
 
@@ -353,11 +359,13 @@ def redux(self, glstring: str, redux_type: VALID_REDUCTION_TYPES) -> str:
             if self.is_XX(glstring, loc_antigen, code):
                 if is_hla_prefix:
                     reduced_alleles = self.redux(
-                        "/".join(self.xx_codes[loc_antigen]), redux_type
+                        "/".join(self.code_mappings.xx_codes[loc_antigen]), redux_type
                     )
                     return "/".join(["HLA-" + a for a in reduced_alleles.split("/")])
                 else:
-                    return self.redux("/".join(self.xx_codes[loc_antigen]), redux_type)
+                    return self.redux(
+                        "/".join(self.code_mappings.xx_codes[loc_antigen]), redux_type
+                    )
 
         # Handle MAC
         if self._config["reduce_MAC"] and self.is_mac(glstring):
@@ -389,7 +397,7 @@ def validate(self, glstring):
         :return: boolean indicating success
         """
         try:
-            return self.isvalid_gl(glstring)
+            return self._is_valid_gl(glstring)
         except InvalidAlleleError as e:
             raise InvalidTypingError(
                 f"{glstring} is not valid GL String. \n {e.message}", e
@@ -402,7 +410,7 @@ def is_XX(self, glstring: str, loc_antigen: str = None, code: str = None) -> boo
                 loc_antigen, code = loc_allele[0], loc_allele[1]
             else:
                 return False
-        return code == "XX" and loc_antigen in self.xx_codes
+        return code == "XX" and loc_antigen in self.code_mappings.xx_codes
 
     def is_serology(self, allele: str) -> bool:
         """
@@ -423,6 +431,7 @@ def is_serology(self, allele: str) -> bool:
 
         return db.is_valid_serology(self.db_connection, allele)
 
+    @functools.lru_cache(maxsize=DEFAULT_CACHE_SIZE)
     def is_mac(self, allele: str) -> bool:
         """
         MAC has non-digit characters after the : character.
@@ -468,15 +477,15 @@ def _is_who_allele(self, allele):
         :param allele: Allele to test
         :return: bool to indicate if allele is valid
         """
-        return allele in self.who_alleles
+        return allele in self.allele_group.who_alleles
 
     def _is_valid_allele(self, allele):
         """
         Test if allele is valid in the current imgt database
         :param allele: Allele to test
         :return: bool to indicate if allele is valid
         """
-        return allele in self.valid_alleles
+        return allele in self.allele_group.alleles
 
     def is_shortnull(self, allele):
         """
@@ -493,7 +502,7 @@ def is_exp_allele(self, allele):
         :param allele: Allele to test
         :return: bool to indicate if allele is valid
         """
-        return allele in self.exp_alleles
+        return allele in self.allele_group.exp_alleles
 
     def _get_alleles(self, code, locus_antigen) -> Iterable[str]:
         """
@@ -574,9 +583,9 @@ def _map_v2_to_v3(self, v2_allele):
             v3_allele = self._predict_v3(v2_allele)
         return v3_allele
 
-    def isvalid(self, allele: str) -> bool:
+    def _is_valid(self, allele: str) -> bool:
         """
-        Determines validity of an allele
+        Determines validity of an allele in various forms
 
         :param allele: An HLA allele.
         :type: str
@@ -617,7 +626,7 @@ def isvalid(self, allele: str) -> bool:
             return self._is_valid_allele(allele)
         return True
 
-    def isvalid_gl(self, glstring: str) -> bool:
+    def _is_valid_gl(self, glstring: str) -> bool:
         """
         Determines validity of glstring
 
@@ -627,62 +636,23 @@ def isvalid_gl(self, glstring: str) -> bool:
         :rtype: bool
         """
 
-        if re.search(r"\^", glstring):
-            return all(map(self.isvalid_gl, glstring.split("^")))
-        if re.search(r"\|", glstring):
-            return all(map(self.isvalid_gl, glstring.split("|")))
-        if re.search(r"\+", glstring):
-            return all(map(self.isvalid_gl, glstring.split("+")))
-        if re.search("~", glstring):
-            return all(map(self.isvalid_gl, glstring.split("~")))
-        if re.search("/", glstring):
-            return all(map(self.isvalid_gl, glstring.split("/")))
+        if "^" in glstring:
+            return all(map(self._is_valid_gl, glstring.split("^")))
+        if "|" in glstring:
+            return all(map(self._is_valid_gl, glstring.split("|")))
+        if "+" in glstring:
+            return all(map(self._is_valid_gl, glstring.split("+")))
+        if "~" in glstring:
+            return all(map(self._is_valid_gl, glstring.split("~")))
+        if "/" in glstring:
+            return all(map(self._is_valid_gl, glstring.split("/")))
 
         # what falls through here is an allele
-        is_valid_allele = self.isvalid(glstring)
+        is_valid_allele = self._is_valid(glstring)
         if not is_valid_allele:
             raise InvalidAlleleError(f"{glstring} is not a valid Allele")
         return is_valid_allele
 
-    def mac_toG(self, allele: str) -> str:
-        """
-        Does ARS reduction with allele and ARS type
-
-        :param allele: An HLA allele.
-        :type: str
-        :return: ARS reduced allele
-        :rtype: str
-        """
-        locus_antigen, code = allele.split(":")
-        if HLA_regex.search(allele):
-            locus_antigen = locus_antigen.split("-")[1]  # Remove HLA- prefix
-        if db.is_valid_mac_code(self.db_connection, code):
-            alleles = self._get_alleles(code, locus_antigen)
-            group = [self.toG(a) for a in alleles]
-            if "X" in group:
-                raise InvalidMACError(f"{allele} is an invalid MAC.")
-            else:
-                return "/".join(group)
-        else:
-            raise InvalidMACError(f"{allele} is an invalid MAC.")
-
-    def toG(self, allele: str) -> str:
-        """
-        Does ARS reduction to the G group level
-
-        :param allele: An HLA allele.
-        :type: str
-        :return: ARS G reduced allele
-        :rtype: str
-        """
-        if allele in self.ars_mappings.g_group:
-            if allele in self.ars_mappings.dup_g:
-                return self.ars_mappings.dup_g[allele]
-            else:
-                return self.ars_mappings.g_group[allele]
-        else:
-            return "X"
-
     def expand_mac(self, mac_code: str):
         """
         Expands mac codes