Merge pull request #8365 from wanghaoshuang/seq_error

Add sequence error output to edit distance evaluator

Author: wanghaoshuang

python/paddle/fluid/evaluator.py

@@ -22,6 +22,7 @@
 __all__ = [
     'Accuracy',
     'ChunkEvaluator',
+    'EditDistance',
 ]
 
 
@@ -211,7 +212,7 @@ def eval(self, executor, eval_program=None):
 class EditDistance(Evaluator):
     """
     Accumulate edit distance sum and sequence number from mini-batches and
-    compute the average edit_distance of all batches.
+    compute the average edit_distance and instance error of all batches.
 
     Args:
         input: the sequences predicted by network.
@@ -227,14 +228,12 @@ class EditDistance(Evaluator):
         for epoch in PASS_NUM:
             distance_evaluator.reset(exe)
             for data in batches:
-                loss, sum_distance = exe.run(fetch_list=[cost] + distance_evaluator.metrics)
-                avg_distance = distance_evaluator.eval(exe)
-            pass_distance = distance_evaluator.eval(exe)
+                loss = exe.run(fetch_list=[cost])
+            distance, instance_error = distance_evaluator.eval(exe)
 
         In the above example:
-        'sum_distance' is the sum of the batch's edit distance.
-        'avg_distance' is the average of edit distance from the firt batch to the current batch.
-        'pass_distance' is the average of edit distance from all the pass.
+        'distance' is the average of the edit distance in a pass.
+        'instance_error' is the instance error rate in a pass.
 
     """
 
@@ -244,25 +243,45 @@ def __init__(self, input, label, ignored_tokens=None, **kwargs):
         if main_program.current_block().idx != 0:
             raise ValueError("You can only invoke Evaluator in root block")
 
-        self.total_error = self.create_state(
-            dtype='float32', shape=[1], suffix='total_error')
+        self.total_distance = self.create_state(
+            dtype='float32', shape=[1], suffix='total_distance')
         self.seq_num = self.create_state(
             dtype='int64', shape=[1], suffix='seq_num')
-        error, seq_num = layers.edit_distance(
+        self.instance_error = self.create_state(
+            dtype='int64', shape=[1], suffix='instance_error')
+        distances, seq_num = layers.edit_distance(
             input=input, label=label, ignored_tokens=ignored_tokens)
-        #error = layers.cast(x=error, dtype='float32')
-        sum_error = layers.reduce_sum(error)
-        layers.sums(input=[self.total_error, sum_error], out=self.total_error)
+
+        zero = layers.fill_constant(shape=[1], value=0.0, dtype='float32')
+        compare_result = layers.equal(distances, zero)
+        compare_result_int = layers.cast(x=compare_result, dtype='int')
+        seq_right_count = layers.reduce_sum(compare_result_int)
+        instance_error_count = layers.elementwise_sub(
+            x=seq_num, y=seq_right_count)
+        total_distance = layers.reduce_sum(distances)
+        layers.sums(
+            input=[self.total_distance, total_distance],
+            out=self.total_distance)
         layers.sums(input=[self.seq_num, seq_num], out=self.seq_num)
-        self.metrics.append(sum_error)
+        layers.sums(
+            input=[self.instance_error, instance_error_count],
+            out=self.instance_error)
+        self.metrics.append(total_distance)
+        self.metrics.append(instance_error_count)
 
     def eval(self, executor, eval_program=None):
         if eval_program is None:
             eval_program = Program()
         block = eval_program.current_block()
         with program_guard(main_program=eval_program):
-            total_error = _clone_var_(block, self.total_error)
+            total_distance = _clone_var_(block, self.total_distance)
             seq_num = _clone_var_(block, self.seq_num)
+            instance_error = _clone_var_(block, self.instance_error)
             seq_num = layers.cast(x=seq_num, dtype='float32')
-            out = layers.elementwise_div(x=total_error, y=seq_num)
-        return np.array(executor.run(eval_program, fetch_list=[out])[0])
+            instance_error = layers.cast(x=instance_error, dtype='float32')
+            avg_distance = layers.elementwise_div(x=total_distance, y=seq_num)
+            avg_instance_error = layers.elementwise_div(
+                x=instance_error, y=seq_num)
+            result = executor.run(
+                eval_program, fetch_list=[avg_distance, avg_instance_error])
+        return np.array(result[0]), np.array(result[1])

python/paddle/fluid/layers/nn.py

@@ -2479,10 +2479,7 @@ def __check_input(x, y):
     return out
 
 
-def edit_distance(input,
-                  label,
-                  normalized=False,
-                  ignored_tokens=None,
+def edit_distance(input, label, normalized=True, ignored_tokens=None,
                   name=None):
     """
     EditDistance operator computes the edit distances between a batch of