Skip to content

Commit 35eb011

Browse files
dzhwinterreyoung
authored andcommitted
"module document non-layer doc" (#11598)
* "add some api reference" * "fix based on preview" * "fix evaluator" * "remove template doc"
1 parent e6cfb2f commit 35eb011

File tree

4 files changed

+386
-98
lines changed

4 files changed

+386
-98
lines changed

python/paddle/fluid/evaluator.py

Lines changed: 74 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,12 @@ def _clone_var_(block, var):
4141

4242
class Evaluator(object):
4343
"""
44-
Base Class for all evaluators
44+
Warning: better to use the fluid.metrics.* things, more
45+
flexible support via pure Python and Operator, and decoupled
46+
with executor. Short doc are intended to urge new user
47+
start from Metrics.
48+
49+
Base Class for all evaluators.
4550
4651
Args:
4752
name(str): The name of evaluator. such as, "accuracy". Used for generate
@@ -69,6 +74,10 @@ def __init__(self, name, **kwargs):
6974
def reset(self, executor, reset_program=None):
7075
"""
7176
reset metric states at the begin of each pass/user specified batch
77+
78+
Args:
79+
executor(Executor|ParallelExecutor): a executor for executing the reset_program
80+
reset_program(Program): a single Program for reset process
7281
"""
7382
if reset_program is None:
7483
reset_program = Program()
@@ -85,15 +94,16 @@ def reset(self, executor, reset_program=None):
8594
def eval(self, executor, eval_program=None):
8695
"""
8796
Evaluate the statistics merged by multiple mini-batches.
97+
Args:
98+
executor(Executor|ParallelExecutor): a executor for executing the eval_program
99+
eval_program(Program): a single Program for eval process
88100
"""
89101
raise NotImplementedError()
90102

91-
def create_state(self, suffix, dtype, shape):
103+
def _create_state(self, suffix, dtype, shape):
92104
"""
93105
Create state variable.
94106
95-
NOTE: It is not a public API.
96-
97107
Args:
98108
suffix(str): the state suffix.
99109
dtype(str|core.VarDesc.VarType): the state data type
@@ -113,9 +123,35 @@ def create_state(self, suffix, dtype, shape):
113123

114124
class ChunkEvaluator(Evaluator):
115125
"""
126+
Warning: This would be deprecated in the future. Please use fluid.metrics.ChunkEvaluator
127+
instead.
128+
116129
Accumulate counter numbers output by chunk_eval from mini-batches and
117130
compute the precision recall and F1-score using the accumulated counter
118131
numbers.
132+
For some basics of chunking, please refer to
133+
'Chunking with Support Vector Machines <https://aclanthology.info/pdf/N/N01/N01-1025.pdf>'.
134+
135+
Args:
136+
input (Variable): prediction output of the network.
137+
label (Variable): label of the test data set.
138+
chunk_scheme (str): can be IOB/IOE/IOBES and IO. See the chunk_eval op for details.
139+
num_chunk_types (int): the number of chunk type.
140+
excluded_chunk_types (list): A list including chunk type ids, indicating chunk types that are not counted.
141+
142+
Returns:
143+
tuple: tuple containing: precision, recall, f1_score
144+
145+
Examples:
146+
.. code-block:: python
147+
148+
exe = fluid.executor(place)
149+
evaluator = fluid.Evaluator.ChunkEvaluator(input, label)
150+
for epoch in PASS_NUM:
151+
evaluator.reset(exe)
152+
for data in batches:
153+
loss = exe.run(fetch_list=[cost])
154+
distance, instance_error = distance_evaluator.eval(exe)
119155
"""
120156

121157
def __init__(
@@ -130,11 +166,11 @@ def __init__(
130166
if main_program.current_block().idx != 0:
131167
raise ValueError("You can only invoke Evaluator in root block")
132168

133-
self.num_infer_chunks = self.create_state(
169+
self.num_infer_chunks = self._create_state(
134170
dtype='int64', shape=[1], suffix='num_infer_chunks')
135-
self.num_label_chunks = self.create_state(
171+
self.num_label_chunks = self._create_state(
136172
dtype='int64', shape=[1], suffix='num_label_chunks')
137-
self.num_correct_chunks = self.create_state(
173+
self.num_correct_chunks = self._create_state(
138174
dtype='int64', shape=[1], suffix='num_correct_chunks')
139175
precision, recall, f1_score, num_infer_chunks, num_label_chunks, num_correct_chunks = layers.chunk_eval(
140176
input=input,
@@ -178,6 +214,8 @@ def eval(self, executor, eval_program=None):
178214

179215
class EditDistance(Evaluator):
180216
"""
217+
Warning: This would be deprecated in the future. Please use fluid.metrics.EditDistance
218+
instead.
181219
Accumulate edit distance sum and sequence number from mini-batches and
182220
compute the average edit_distance and instance error of all batches.
183221
@@ -188,15 +226,16 @@ class EditDistance(Evaluator):
188226
ignored_tokens(list of int): Tokens that should be removed before
189227
calculating edit distance.
190228
191-
Example:
229+
Examples:
230+
.. code-block:: python
192231
193-
exe = fluid.executor(place)
194-
distance_evaluator = fluid.Evaluator.EditDistance(input, label)
195-
for epoch in PASS_NUM:
196-
distance_evaluator.reset(exe)
197-
for data in batches:
198-
loss = exe.run(fetch_list=[cost])
199-
distance, instance_error = distance_evaluator.eval(exe)
232+
exe = fluid.executor(place)
233+
distance_evaluator = fluid.Evaluator.EditDistance(input, label)
234+
for epoch in PASS_NUM:
235+
distance_evaluator.reset(exe)
236+
for data in batches:
237+
loss = exe.run(fetch_list=[cost])
238+
distance, instance_error = distance_evaluator.eval(exe)
200239
201240
In the above example:
202241
'distance' is the average of the edit distance in a pass.
@@ -210,11 +249,11 @@ def __init__(self, input, label, ignored_tokens=None, **kwargs):
210249
if main_program.current_block().idx != 0:
211250
raise ValueError("You can only invoke Evaluator in root block")
212251

213-
self.total_distance = self.create_state(
252+
self.total_distance = self._create_state(
214253
dtype='float32', shape=[1], suffix='total_distance')
215-
self.seq_num = self.create_state(
254+
self.seq_num = self._create_state(
216255
dtype='int64', shape=[1], suffix='seq_num')
217-
self.instance_error = self.create_state(
256+
self.instance_error = self._create_state(
218257
dtype='int64', shape=[1], suffix='instance_error')
219258
distances, seq_num = layers.edit_distance(
220259
input=input, label=label, ignored_tokens=ignored_tokens)
@@ -256,9 +295,10 @@ def eval(self, executor, eval_program=None):
256295

257296
class DetectionMAP(Evaluator):
258297
"""
298+
Warning: This would be deprecated in the future. Please use fluid.metrics.DetectionMAP
299+
instead.
259300
Calculate the detection mean average precision (mAP).
260301
261-
TODO (Dang Qingqing): update the following doc.
262302
The general steps are as follows:
263303
1. calculate the true positive and false positive according to the input
264304
of detection and labels.
@@ -293,17 +333,18 @@ class DetectionMAP(Evaluator):
293333
- 11point: the 11-point interpolated average precision.
294334
- integral: the natural integral of the precision-recall curve.
295335
296-
Example:
336+
Examples:
337+
.. code-block:: python
297338
298-
exe = fluid.executor(place)
299-
map_evaluator = fluid.Evaluator.DetectionMAP(input,
300-
gt_label, gt_box, gt_difficult)
301-
cur_map, accum_map = map_evaluator.get_map_var()
302-
fetch = [cost, cur_map, accum_map]
303-
for epoch in PASS_NUM:
304-
map_evaluator.reset(exe)
305-
for data in batches:
306-
loss, cur_map_v, accum_map_v = exe.run(fetch_list=fetch)
339+
exe = fluid.executor(place)
340+
map_evaluator = fluid.Evaluator.DetectionMAP(input,
341+
gt_label, gt_box, gt_difficult)
342+
cur_map, accum_map = map_evaluator.get_map_var()
343+
fetch = [cost, cur_map, accum_map]
344+
for epoch in PASS_NUM:
345+
map_evaluator.reset(exe)
346+
for data in batches:
347+
loss, cur_map_v, accum_map_v = exe.run(fetch_list=fetch)
307348
308349
In the above example:
309350
@@ -340,9 +381,10 @@ def __init__(self,
340381
evaluate_difficult=evaluate_difficult,
341382
ap_version=ap_version)
342383

343-
self.create_state(dtype='int32', shape=None, suffix='accum_pos_count')
344-
self.create_state(dtype='float32', shape=None, suffix='accum_true_pos')
345-
self.create_state(dtype='float32', shape=None, suffix='accum_false_pos')
384+
self._create_state(dtype='int32', shape=None, suffix='accum_pos_count')
385+
self._create_state(dtype='float32', shape=None, suffix='accum_true_pos')
386+
self._create_state(
387+
dtype='float32', shape=None, suffix='accum_false_pos')
346388

347389
self.has_state = None
348390
var = self.helper.create_variable(

python/paddle/fluid/layers/__init__.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@
2828
from math_op_patch import *
2929
import detection
3030
from detection import *
31-
import metric
32-
from metric import *
31+
import metric_op
32+
from metric_op import *
3333
from learning_rate_scheduler import *
3434

3535
__all__ = []
@@ -41,5 +41,5 @@
4141
__all__ += ops.__all__
4242
__all__ += device.__all__
4343
__all__ += detection.__all__
44-
__all__ += metric.__all__
44+
__all__ += metric_op.__all__
4545
__all__ += learning_rate_scheduler.__all__

python/paddle/fluid/layers/metric.py renamed to python/paddle/fluid/layers/metric_op.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ def auc(input, label, curve='ROC', num_thresholds=200):
126126
topk_out, topk_indices = nn.topk(input, k=k)
127127
auc_out = helper.create_tmp_variable(dtype="float32")
128128
helper.append_op(
129-
type="accuracy",
129+
type="auc",
130130
inputs={
131131
"Out": [topk_out],
132132
"Indices": [topk_indices],

0 commit comments

Comments
 (0)