Merge pull request #29 from InvalidPointer/master

Added pyml2ds module

Author: Jon Walker

.gitignore

@@ -32,6 +32,7 @@ dist
 *.egg-info
 .eggs
 .pypirc
+*.pyc
 
 ## tox testing tool
 .tox

src/sasctl/utils/pyml2ds/__init__.py

@@ -0,0 +1 @@
+from .pyml2ds import pyml2ds

src/sasctl/utils/pyml2ds/basic/__init__.py

@@ -0,0 +1 @@
+from .tree import TreeParser

src/sasctl/utils/pyml2ds/basic/tree.py

@@ -0,0 +1,167 @@
+import abc, six
+import sys
+import unicodedata
+
+@six.add_metaclass(abc.ABCMeta)
+class TreeParser:
+    """Abstract class for parsing decision tree.
+
+    Attributes
+    ----------
+    d : dict
+        Dictionary for storing node hierarchy. Used not in all models. 
+    out_transform : string
+        Output transformation for generated value. For example, for logreg is used: 1 / (1 + exp(-{0})), where {0} stands for resulting gbvalue.
+    out_var_name : string
+        Name used for output variable.
+    """
+    def __init__(self):
+        self.d = {}
+
+
+    """Custom init method. Need to be called before using TreeParser.
+
+    Attributes
+    ----------
+    root : node
+        Tree root node.
+    tree_id : int
+        Id of current tree.
+    """
+    def init(self, root, tree_id=0):
+        self._root = root
+        self._node = root
+        self._tree_id = tree_id
+
+        self._depth = -1
+        self._indent = 4
+
+
+    def _gen_dict(self):
+        self.d = {}
+
+    
+    def _get_indent(self):
+        return " " * self._indent * self._depth
+
+
+    @abc.abstractmethod
+    def _not_leaf(self):
+        pass
+
+
+    @abc.abstractmethod
+    def _get_var(self):
+        pass
+
+
+    @abc.abstractmethod
+    def _go_left(self):
+        pass
+
+
+    @abc.abstractmethod
+    def _go_right(self):
+        pass
+
+
+    @abc.abstractmethod
+    def _left_node(self):
+        pass
+
+
+    @abc.abstractmethod
+    def _right_node(self):
+        pass
+
+
+    @abc.abstractmethod
+    def _missing_node(self):
+        pass
+
+
+    @abc.abstractmethod
+    def _split_value(self):
+        pass
+
+
+    @abc.abstractmethod
+    def _decision_type(self):
+        pass
+
+
+    @abc.abstractmethod
+    def _leaf_value(self):
+        pass
+
+
+    def _remove_diacritic(self, input):
+        if sys.hexversion >= 0x3000000:
+            output = unicodedata.normalize('NFKD', input).encode('ASCII', 'ignore').decode()
+        else:
+            # On Python < 3.0.0
+            if type(input) == str:
+                input = unicode(input, 'ISO-8859-1')
+            output = unicodedata.normalize('NFKD', input).encode('ASCII', 'ignore')
+        
+        return output
+
+
+    """Recursively parses tree node and writes generated SAS code to file.
+
+    Attributes
+    ----------
+    f : file object
+        Open file for writing output SAS code.
+    node: node
+        Tree node to process.
+    """
+    def parse_node(self, f, node=None, test=False):
+        if test:
+            self.test = True
+
+        pnode = self._node
+        if node is not None:
+            self._node = node
+        else:
+            self._node = self._root
+
+        if self._node == self._root:
+            self.d = dict()
+            self._gen_dict()
+
+        self._depth += 1
+
+        if self._not_leaf():
+            var = self._get_var()[:32]
+            var = self._remove_diacritic(var)
+            
+            split_value = self._split_value()
+            if self.test:
+                split_value = int(float(split_value))
+
+            cond = ""
+            if self._go_left():
+                cond = "missing({}) or ".format(var)
+            elif self._go_right():
+                cond = "not missing({}) and ".format(var)
+            else:
+                f.write(self._get_indent() + "if (missing({})) then do;\n".format(var))
+                self.parse_node(f, node=self._missing_node())
+                f.write(self._get_indent() + "end;\n")
+            
+            f.write(self._get_indent() + "if ({}{} {} {}) then do;\n".format(cond, var, self._decision_type(), split_value))
+            self.parse_node(f, node=self._left_node())
+            f.write(self._get_indent() + "end;\n")
+            f.write(self._get_indent() + "else do;\n")
+            self.parse_node(f, node=self._right_node())
+            f.write(self._get_indent() + "end;\n")
+        else:
+            leaf_value = self._leaf_value()
+            if self.test:
+                leaf_value = int(float(leaf_value))
+
+            f.write(self._get_indent() + "treeValue{} = {};\n".format(self._tree_id, leaf_value))
+        
+        self._node = pnode
+        self._depth -= 1

src/sasctl/utils/pyml2ds/connectors/__init__.py

@@ -0,0 +1 @@
+from .ensembles import XgbParser, LightgbmParser, PmmlParser

src/sasctl/utils/pyml2ds/connectors/ensembles/__init__.py

@@ -0,0 +1,3 @@
+from .xgb import XgbParser
+from .lgb import LightgbmParser
+from .pmml import PmmlParser

src/sasctl/utils/pyml2ds/connectors/ensembles/core.py

@@ -0,0 +1,47 @@
+import abc, six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class EnsembleParser:
+    """Abstract class for parsing decision tree ensebmles.
+
+    Attributes
+    ----------
+    out_transform : string
+        Output transformation for generated value. For example, for logreg is used: 1 / (1 + exp(-{0})), where {0} stands for resulting gbvalue.
+    out_var_name : string
+        Name used for output variable.
+    """
+    def __init__(self, out_transform="{0}", out_var_name="P_TARGET"):
+        self.out_transform = out_transform
+        self.out_var_name = out_var_name
+
+
+    @abc.abstractmethod
+    def _iter_trees(self):
+        pass
+
+    
+    def _aggregate(self, booster_count):
+        return "treeValue = sum({});\n".format(', '.join(["treeValue%d" % i for i in range(booster_count)]))
+
+
+    """Translates gradient boosting model and writes SAS scoring code to file.
+
+    Attributes
+    ----------
+    f : file object
+        Open file for writing output SAS code.
+    """
+    def translate(self, f, test=False):
+        for booster_id, tree in self._iter_trees():
+            f.write("/* Parsing tree {}*/\n".format(booster_id))
+            
+            self._tree_parser.init(tree, booster_id)
+            self._tree_parser.parse_node(f, test=test)
+            
+            f.write("\n")
+        
+        f.write("/* Getting target probability */\n")
+        f.write(self._aggregate(booster_id + 1))
+        f.write("{} = {};\n".format(self.out_var_name, self.out_transform.format("treeValue")))

src/sasctl/utils/pyml2ds/connectors/ensembles/lgb.py

@@ -0,0 +1,75 @@
+from sasctl.utils.pyml2ds.basic import TreeParser
+from .core import EnsembleParser
+
+
+class LightgbmTreeParser(TreeParser):
+    """Class for parsing lightgbm tree.
+    """
+
+    def _not_leaf(self):
+        return 'split_feature' in self._node
+
+
+    def _get_var(self):
+        return self._features[self._node['split_feature']]
+
+
+    def _go_left(self):
+        return self._node['default_left']
+
+    
+    def _go_right(self):
+        return (not self._node['default_left'])
+
+
+    def _left_node(self):
+        return self._node['left_child']
+
+    
+    def _right_node(self):
+        return self._node['right_child']
+
+
+    def _missing_node(self):
+        return None
+
+    
+    def _split_value(self):
+        return self._node['threshold']
+
+
+    def _decision_type(self):
+        return self._node['decision_type']
+
+
+    def _leaf_value(self):
+        return self._node['leaf_value']
+
+
+class LightgbmParser(EnsembleParser):
+    """Class for parsing lightgbm model.
+
+    Parameters
+    ----------
+    booster : lightgbm.basic.Booster
+        Booster of lightgbm model.
+    """
+    def __init__(self, booster):
+        super(LightgbmParser, self).__init__()
+
+        self._booster = booster
+        
+        self._dump = booster.dump_model()
+        if self._dump['objective'] != 'binary sigmoid:1':
+            raise Exception("Unfortunately only binary sigmoid objective function is supported right now. Your objective is %s. Please, open an issue at https://gitlab.sas.com/from-russia-with-love/lgb2sas." % self.dump['objective'])
+
+        self._features = self._dump['feature_names']
+        self.out_transform = "1 / (1 + exp(-{0}))"
+
+        self._tree_parser = LightgbmTreeParser()
+        self._tree_parser._features = self._features
+
+
+    def _iter_trees(self):
+        for tree in self._dump['tree_info']:
+            yield tree['tree_index'], tree['tree_structure']

src/sasctl/utils/pyml2ds/connectors/ensembles/pmml.py

@@ -0,0 +1,78 @@
+from sasctl.utils.pyml2ds.basic import TreeParser
+from .core import EnsembleParser
+
+
+class PmmlTreeParser(TreeParser):
+    """Class for parsing pmml gradient boosting tree.
+    """
+
+    def _not_leaf(self):
+        return self._node.get('defaultChild')
+
+
+    def _get_var(self):
+        return self._node.find('Node').find('SimplePredicate').get('field')
+
+
+    def _go_left(self):
+        return (self._node.find('Node').get('id') == self._node.get('defaultChild'))
+
+    
+    def _go_right(self):
+        return (not self._node.find('Node').get('id') == self._node.get('defaultChild'))
+
+
+    def _left_node(self):
+        return self._node.findall('Node')[0]
+
+    
+    def _right_node(self):
+        return self._node.findall('Node')[1]
+
+
+    def _missing_node(self):
+        return None
+
+    
+    def _split_value(self):
+        return self._node.find('Node').find('SimplePredicate').get('value')
+
+
+    def _decision_type(self):
+        ops = {'lessThan': '<', 'lessOrEqual': '<=', 'greaterThan': '>', 'greaterOrEqual': '>='}
+        return ops[self._node.find('Node').find('SimplePredicate').get('operator')]
+
+
+    def _leaf_value(self):
+        return self._node.get('score')
+
+
+class PmmlParser(EnsembleParser):
+    """Class for parsing pmml gradient boosting models.
+
+    Parameters
+    ----------
+    tree_root : etree.Element
+        Root node of pmml gradient boosting forest.
+    """
+    def __init__(self, tree_root):
+        super(PmmlParser, self).__init__()
+
+        self._tree_root = tree_root
+        for elem in tree_root.getiterator():
+            if not hasattr(elem.tag, 'find'): continue
+            i = elem.tag.find('}')
+            if i >= 0:
+                elem.tag = elem.tag[i+1:]
+        
+        self._forest = tree_root.find('MiningModel/Segmentation')[0].find('MiningModel')
+        
+        rescaleConstant = self._forest.find('Targets/Target').get('rescaleConstant')
+        self.out_transform = "1 / (1 + exp(-{}))".format("({0} + " + "{})".format(rescaleConstant))
+
+        self._tree_parser = PmmlTreeParser()
+
+
+    def _iter_trees(self):
+        for booster_id, tree_elem in enumerate(self._forest.find('Segmentation')):
+            yield booster_id, tree_elem.find('TreeModel/Node')