Shortnulls (#154)

* shortnulls implementation

* shortnulls passes all tests

* more meaningful variable names, fixed error that caused invalids on first run

* made shortnull data structure consistent

* even more pythonic use of set, change back the location of the database

* now with more pythonsim

Author: mmaiers-nmdp

pyard/data_repository.py

@@ -211,7 +211,15 @@ def generate_alleles_and_xx_codes_and_who(db_connection: sqlite3.Connection, img
                                 ('allele_1d', 'allele_list'))
         xx_codes = {k: v.split('/') for k, v in xx_codes.items()}
 
-        return valid_alleles, who_alleles, xx_codes, who_group
+        shortnulls = db.load_dict(db_connection, 'shortnulls', 
+                                ('shortnull', 'allele_list'))
+        shortnulls = {k: v.split('/') for k, v in shortnulls.items()}
+
+        exp_alleles = db.load_dict(db_connection, 'exp_alleles', 
+                                ('exp_allele', 'allele_list'))
+        exp_alleles = {k: v.split('/') for k, v in exp_alleles.items()}
+
+        return valid_alleles, who_alleles, xx_codes, who_group, shortnulls, exp_alleles
 
     # Create a Pandas DataFrame from the mac_code list file
     # Skip the header (first 6 lines) and use only the Allele column
@@ -229,6 +237,12 @@ def generate_alleles_and_xx_codes_and_who(db_connection: sqlite3.Connection, img
     exp_alleles = allele_df[allele_df['Allele'].apply(
         lambda a: a[-1] in expression_chars and number_of_fields(a) > 2)]
     exp_alleles = exp_alleles.groupby('2d').apply(expression_reduce).dropna()
+
+    #flat_exp_alleles = {k: '/'.join(sorted(v, key=functools.cmp_to_key(smart_sort_comparator)))
+    #                 for k, v in exp_alleles.items()}
+    db.save_dict(db_connection, 'exp_alleles', exp_alleles,
+                 ('exp_allele', 'allele_list'))
+
     # Create valid set of alleles:
     # All full length alleles
     # All 3rd and 2nd field versions of longer alleles
@@ -304,7 +318,37 @@ def generate_alleles_and_xx_codes_and_who(db_connection: sqlite3.Connection, img
     db.save_dict(db_connection, 'who_group', flat_who_group,
                  columns=('who', 'allele_list'))
 
-    return valid_alleles, who_alleles, xx_codes, who_group
+    # shortnulls
+    # scan WHO alleles for those with expression characters and make shortnull mappings 
+    # DRB4*01:03N | DRB4*01:03:01:02N/DRB4*01:03:01:13N 
+    # DRB5*01:08N | DRB5*01:08:01N/DRB5*01:08:02N 
+    shortnulls = dict()
+    for who in who_group:
+        # e.g. DRB4*01:03
+        expression_alleles=[]
+        expression_chars_found = set()
+        if who[-1] not in expression_chars and who[-1] not in ['G', 'P'] and ":" in who:
+            for an_allele in who_group[who]:
+                # if an allele in a who_group has an expression character but the group allele doesnt, 
+                # add it to shortnulls
+                last_char = an_allele[-1]
+                if last_char in expression_chars:
+                    # e.g. DRB4*01:03:01:02N
+                    expression_chars_found.add(last_char)
+                    # add this allele to the set that this short null exapands to 
+                    expression_alleles.append(an_allele) 
+            # only create a shortnull if there is one expression character in this who_group
+            # there is nothing to be done for who_groups that have both Q and L for example
+            if expression_alleles:
+                if len(expression_chars_found) ==1:
+                    # e.g. DRB4*01:03N 
+                    a_shortnull = who + list(expression_chars_found)[0]
+                    shortnulls[a_shortnull] = "/".join(expression_alleles)
+
+    db.save_dict(db_connection, 'shortnulls', shortnulls, ('shortnull', 'allele_list'))
+    shortnulls = {k: v.split('/') for k, v in shortnulls.items()}
+
+    return valid_alleles, who_alleles, xx_codes, who_group, shortnulls, exp_alleles
 
 
 def generate_mac_codes(db_connection: sqlite3.Connection, refresh_mac: bool):

pyard/pyard.py

@@ -30,7 +30,7 @@
 from . import data_repository as dr
 from .smart_sort import smart_sort_comparator
 from .exceptions import InvalidAlleleError, InvalidMACError, InvalidTypingError
-from .misc import get_n_field_allele
+from .misc import get_n_field_allele, get_2field_allele
 
 HLA_regex = re.compile("^HLA-")
 
@@ -47,6 +47,7 @@
     "reduce_P": True,
     "reduce_XX": True,
     "reduce_MAC": True,
+    "reduce_shortnull": True,
     "map_drb345_to_drbx": True,
     "verbose_log": True
 }
@@ -100,7 +101,7 @@ def __init__(self, imgt_version: str = 'Latest', data_dir: str = None, config: d
         # Load ARS mappings
         self.ars_mappings = dr.generate_ars_mapping(self.db_connection, imgt_version)
         # Load Alleles and XX Codes
-        self.valid_alleles, self.who_alleles, self.xx_codes, self.who_group = \
+        self.valid_alleles, self.who_alleles, self.xx_codes, self.who_group, self.shortnulls, self.exp_alleles = \
             dr.generate_alleles_and_xx_codes_and_who(self.db_connection, imgt_version, self.ars_mappings)
 
         # Load Serology mappings
@@ -210,6 +211,8 @@ def redux(self, allele: str, redux_type: VALID_REDUCTION_TYPES) -> str:
                 # If ambiguous, reduce to G group level
                 return self.redux(allele, 'lgx')
         else:
+            # TODO: make this an explicit lookup to the g_group or p_group table
+            # just having a shorter name be valid is not stringent enough
             if allele.endswith(('P', 'G')):
                 allele = allele[:-1]
             if self._is_valid_allele(allele):
@@ -311,6 +314,11 @@ def redux_gl(self, glstring: str, redux_type: VALID_REDUCTION_TYPES) -> str:
             else:
                 raise InvalidMACError(f"{glstring} is an invalid MAC.")
 
+        # Handle shortnulls
+        if self._config["reduce_shortnull"] and self.is_shortnull(glstring):
+            return self.redux_gl("/".join(self.shortnulls[glstring]), redux_type)
+            #return self.redux_gl(self.shortnulls[glstring], redux_type)
+
         return self.redux(glstring, redux_type)
 
     def is_XX(self, glstring: str, loc_antigen: str = None, code: str = None) -> bool:
@@ -393,6 +401,23 @@ def _is_valid_allele(self, allele):
         """
         return allele in self.valid_alleles
 
+    def is_shortnull(self, allele):
+        """
+        Test if allele is valid in list of shortnull alleles and 
+        the reduce_shortnull is configured to True (WMDA rules)
+        :param allele: Allele to test
+        :return: bool to indicate if allele is valid
+        """
+        return allele in self.shortnulls and self._config["reduce_shortnull"]
+
+    def is_exp_allele(self, allele):
+        """
+        Test if allele is valid as a shortening (WHO rules)
+        :param allele: Allele to test
+        :return: bool to indicate if allele is valid
+        """
+        return allele in self.exp_alleles
+
     def _get_alleles(self, code, locus_antigen) -> Iterable[str]:
         """
         Look up allele code in database and generate alleles
@@ -411,7 +436,7 @@ def _get_alleles(self, code, locus_antigen) -> Iterable[str]:
         else:
             alleles = [f'{locus_antigen}:{a}' for a in alleles]
 
-        return filter(self._is_valid_allele, alleles)
+        return list(filter(self._is_valid_allele, alleles))
 
     def _get_alleles_from_serology(self, serology) -> Iterable[str]:
         alleles = db.serology_to_alleles(self.db_connection, serology)
@@ -491,16 +516,19 @@ def isvalid(self, allele: str) -> bool:
         if not self.is_mac(allele) and \
                 not self.is_XX(allele) and \
                 not self.is_serology(allele) and \
-                not self.is_v2(allele):
+                not self.is_v2(allele) and \
+                not self.is_shortnull(allele):
             # Alleles ending with P or G are valid_alleles
             if allele.endswith(('P', 'G')):
                 # remove the last character
                 allele = allele[:-1]
                 if self._is_valid_allele(allele):
                     return True
-                else:
-                    # reduce to 2 field for things like DPB1*28:01:01G
-                    allele = ':'.join(allele.split(':')[0:2])
+                else: 
+                    allele = get_2field_allele(allele)
+                    if self._is_valid_allele(allele):
+                        return True
+
             return self._is_valid_allele(allele)
         return True