"polish document"

Author: dzhwinter

doc/design/evaluator.md

@@ -15,35 +15,44 @@ Currently, every operation is expressed in the graph. we divide the evaluator pr
 3. Merge the mini-batch statistics to form the evaluation result for multiple mini-batches. When it comes to distributed training/Multi-GPU training, aggregate the value from different devices.
 
 ### Implementation
-This design is shown in python API. There would be an abstract python interface and multiple inheritances for each evaluation method.
+This design is shown in python API. 
+Each metric operator need to caculate the metric statistic and return the batch aware states, Python side responsible for accumulate the states for each pass. 
 
+    
 ```python
 class Evaluator(object):
     """
     Evaluator Base class.
     """
-    def __init__(self):
+    def __init__(self, name, **kwargs):
        """
        Different evaluator may has different metric states. E.g, Accuracy need two variables, total and right sample counts.
        Auc need four variables, `true_positives`,
-         `true_negatives`, `false_positives` and `false_negatives`. So every evaluator should create its needed variables and append the related mini-batch operator to main_program
+         `true_negatives`, `false_positives` and `false_negatives`. So every evaluator should create its needed variables and append to main_program
 
        The initialization of Evaluator should be responsible for:
        create metric states and append to the main_program
-       add mini-batch evaluator caculate operators to the main_program
-       add increment operator to accumulate the metric states
        """ 
        pass
 
-    def clear(self):
+    def _update_ops(self, input, label, **kwargs)
+       """
+       Add mini-batch evaluator caculate operators to the main_program.
+       Add increment operator to accumulate the metric states.
+       """
+    
+
+    def reset(self, executor, program=None):
       """
-      clear metric states at the begin of each pass/user specified batch
+      Reset metric states at the begin of each pass/user specified batch number.
+      Execute the reset_program to reset the states.
       """
-      return init_program
+      
 
-    def evaluate(self):
+    def eval(self, executor, program=None):
       """
       Merge the mini-batch statistics to form the evaluation result for multiple mini-batches.
+      Execute the eval_program and return the result.
       """
-      return eval_program
+      return eval_result
 ```

python/paddle/v2/framework/evaluator.py

@@ -181,43 +181,6 @@ def eval(self, executor, program=None):
         return np.array(out[0])
 
 
-# Demo for composing low level ops to compute the F1 metric
-class FScore(Evaluator):
-    def __init__(self, input, label, beta=1.0, **kwargs):
-        super(F1, self).__init__("FScore", **kwargs)
-        block = self._program.global_block()
-        g_tp = block.create_var(
-            name=unique_name("Tp"), persistable=True, dtype="int64", shape=[1])
-        g_fn = block.create_var(
-            name=unique_name("Fn"), persistable=True, dtype="int64", shape=[1])
-        g_fp = block.create_var(
-            name=unique_name("Fp"), persistable=True, dtype="int64", shape=[1])
-
-        self._states["Tp"] = g_tp
-        self._states["Fp"] = g_fp
-        self._states["Fn"] = g_fn
-
-    def _update_ops(self):
-        block = self._program.global_block()
-        equal_out = block.create_var()
-        block.append_op(
-            type="equal",
-            inputs={"X": [input],
-                    "Y": [label]},
-            outputs={"Out": equal_out})
-
-        positive = block.create_var()
-        block.append_op(
-            type="sequence_pool",
-            inputs={"X": [equal_out]},
-            outputs={"Out": positive},
-            attrs={"pooltype": "SUM"})
-        batch = block.create_var(
-            name=feed_var_name,
-            type=core.VarDesc.VarType.FEED_MINIBATCH,
-            persistable=True)
-
-
 # FIXME(dzh): add a decorator to call _update_ops automatically
 def accuracy(*args, **kwargs):
     cls = Accuracy(*args, **kwargs)