Variant Rank Score Normalization

** Variant Ranking **

Normalize variant rank score by means of max-min normalization.

This is preferable because it makes output values
agnostic to the number of plugins and their scoring
behavior.

Max and min is computed as the total sum of upper and lower
bounds of each plugin.

The VCF attribute 'RankScoreNormalized' (bounded (0, 1))
is added to the scored VCF file.

No changes to existing 'RankScore' behavior, which
is still present in VCF.

** Compound Ranking **

This patch makes CompoundScore class use the additional
RankScoreNormalized VCF field for scoring purposes.
This patch adds a CompoundsNormalized INFO field to VCF,
based on RankScoreNormalized.

* Export RankScore bounds MIN, MAX to VCF file
  This is required by CompoundScore because it thresholds and
  edits RankScore(Normalized) during the scoring process.

* Configure CompoundScorer to use normalized rank score
  in addition to legacy RankScore.

* Apply CompoundScorer custom thresholds normalized
  to RankScore Min-Max bounds.

Signed-off-by: Tor Björgen <tor.bjorgen@scilifelab.se>

Author: torbjorgen

docs/commands/score-variants.md

@@ -0,0 +1,16 @@
+# Score Variant
+
+## Rank Score Normalization
+
+The rank score is MAXMIN normalized into range (0, 1) according to the following formula:
+
+```
+RankScoreNormalized = (RankScore - CategorySumMin) / (CategorySumMax - CategorySumMin)
+```
+where `RankScore` is the sum of rank score across categories (including rules such as min, max, sum etc)
+`RankScore = SUM(Score_category_n) for 0...n categories`
+and `CategorySumMin` is the sum of minimal score values for all categories,
+i. e `CategorySumMin = SUM(CategoryMin_n) for 0...n categories`.
+The same applies to `CategorySumMax = SUM(CategoryMax_n) for 0...n categories`.
+
+Refer to `score_variants.py::score()` method for implementation details.

genmod/commands/score_compounds.py

@@ -80,6 +80,13 @@ def compound(context, variant_file, silent, outfile, vep, processes, temp_dir):
     header_line = head.header
     individuals = head.individuals
 
+    add_metadata(head,
+                 'info',
+                 'CompoundsNormalized',
+                 annotation_number='.',
+                 entry_type='String',
+                 description='Rank score as provided by compound analysis, based on RankScoreNormalized. family_id:rank_score')
+
     ###################################################################
     ### The task queue is where all jobs(in this case batches that  ###
     ### represents variants in a region) is put. The consumers will ###

genmod/commands/score_variants.py

@@ -27,7 +27,7 @@
 print_headers, HeaderParser, get_variant_dict, get_info_dict)
 
 from genmod.score_variants import (ConfigParser, get_category_score, 
-check_plugins)
+check_plugins, as_normalized_max_min, RANK_SCORE_TYPES)
 
 from genmod import __version__
 
@@ -133,14 +133,22 @@ def score(context, variant_file, family_id, family_file, family_type, score_conf
         logger.warning("Variants already scored according to VCF header")
         logger.info("Please check VCF file")
         context.abort()
-    
+
+    for rank_score_type, rank_score_description in RANK_SCORE_TYPES.items():
+        add_metadata(head,
+                     'info',
+                     rank_score_type,
+                     annotation_number='.',
+                     entry_type='String',
+                     description=rank_score_description)
+
     add_metadata(
         head,
         'info',
-        'RankScore',
-        annotation_number='.', 
-        entry_type='String', 
-        description="The rank score for this variant in this family. family_id:rank_score."
+        'RankScoreMinMax',
+        annotation_number='.',
+        entry_type='String',
+        description="The rank score MIN-MAX bounds. family_id:min:max."
     )
 
     if rank_results:
@@ -169,26 +177,46 @@ def score(context, variant_file, family_id, family_file, family_type, score_conf
             rank_score = 0
             # This is for printing results to vcf:
             category_scores = []
+            # Keep track for per-category min max scores for normalization purposes
+            category_scores_max: float = 0.0
+            category_scores_min: float = 0.0
             for category in score_categories:
-                category_score = get_category_score(
+                category_score, category_score_min, category_score_max = get_category_score(
                     variant=variant, 
                     category=category, 
                     config_parser=config_parser, 
                     csq_format=csq_format
                 )
                 logger.debug("Adding category score {0} to rank_score".format(category_score))
-                
                 rank_score += category_score
                 logger.debug("Updating rank score to {0}".format(rank_score))
+                category_scores_min += category_score_min
+                category_scores_max += category_score_max
 
                 category_scores.append(str(category_score))
 
-            
+            # Normalize ranks score (across all categories)
+            rank_score_normalized: float = as_normalized_max_min(score=float(rank_score),
+                                                                 min_score_value=category_scores_min,
+                                                                 max_score_value=category_scores_max)
+
             variant = add_vcf_info(
                 keyword = 'RankScore',
                 variant_dict=variant,
                 annotation="{0}:{1}".format(family_id, rank_score)
             )
+
+            variant: dict = add_vcf_info(
+                keyword = 'RankScoreNormalized',
+                variant_dict=variant,
+                annotation="{0}:{1}".format(family_id, rank_score_normalized)
+            )
+
+            variant: dict = add_vcf_info(
+                keyword = 'RankScoreMinMax',
+                variant_dict=variant,
+                annotation="{0}:{1}:{2}".format(family_id, category_scores_min, category_scores_max)
+            )
 
             if rank_results:
                 variant = add_vcf_info(

genmod/score_variants/__init__.py

@@ -2,6 +2,7 @@
 
 from .score_function import ScoreFunction
 from .config_parser import ConfigParser
-from .score_variant import get_category_score
+from .score_variant import get_category_score, as_normalized_max_min
 from .compound_scorer import CompoundScorer
-from .check_plugins import check_plugins
+from .check_plugins import check_plugins
+from .rank_score_variant_definitions import RANK_SCORE_TYPES, RANK_SCORE_TYPE_NAMES