Merge pull request #1958 from linjieccc/op_path

wawltor · web-flow · commit 519f5527304d · 2022-04-15T13:11:59.000+08:00
Optimize task path for Taskflow
diff --git a/docs/model_zoo/taskflow.md b/docs/model_zoo/taskflow.md
@@ -438,10 +438,10 @@ from paddlenlp import Taskflow
 >>> from paddlenlp import Taskflow
 >>> nptag = Taskflow("knowledge_mining", model="nptag")
 >>> nptag("糖醋排骨")
->>> [{'text': '糖醋排骨', 'label': '菜品'}]
+[{'text': '糖醋排骨', 'label': '菜品'}]
 
-nptag(["糖醋排骨", "红曲霉菌"])
->>> [{'text': '糖醋排骨', 'label': '菜品'}, {'text': '红曲霉菌', 'label': '微生物'}]
+>>> nptag(["糖醋排骨", "红曲霉菌"])
+[{'text': '糖醋排骨', 'label': '菜品'}, {'text': '红曲霉菌', 'label': '微生物'}]
 
 # 使用`linking`输出粗粒度类别标签`category`，即WordTag的词汇标签。
 >>> nptag = Taskflow("knowledge_mining", model="nptag", linking=True)
@@ -471,8 +471,7 @@ nptag(["糖醋排骨", "红曲霉菌"])
 [{'source': '遇到逆竟时，我们必须勇于面对，而且要愈挫愈勇。', 'target': '遇到逆境时，我们必须勇于面对，而且要愈挫愈勇。', 'errors': [{'position': 3, 'correction': {'竟': '境'}}]}]
 
 # 批量预测
->>> corrector(['遇到逆竟时，我们必须勇于面对，而且要愈挫愈勇。',
-                '人生就是如此，经过磨练才能让自己更加拙壮，才能使自己更加乐观。'])
+>>> corrector(['遇到逆竟时，我们必须勇于面对，而且要愈挫愈勇。', '人生就是如此，经过磨练才能让自己更加拙壮，才能使自己更加乐观。'])
 [{'source': '遇到逆竟时，我们必须勇于面对，而且要愈挫愈勇。', 'target': '遇到逆境时，我们必须勇于面对，而且要愈挫愈勇。', 'errors': [{'position': 3, 'correction': {'竟': '境'}}]}, {'source': '人生就是如此，经过磨练才能让自己更加拙壮，才能使自己更加乐观。', 'target': '人生就是如此，经过磨练才能让自己更加茁壮，才能使自己更加乐观。', 'errors': [{'position': 18, 'correction': {'拙': '茁'}}]}]
 ```
 
@@ -628,16 +627,17 @@ nptag(["糖醋排骨", "红曲霉菌"])
 
 |                           任务名称                           |                           默认路径                           |                                                              |
 | :----------------------------------------------------------: | :----------------------------------------------------------: | :----------------------------------------------------------: |
-|         `Taskflow("word_segmentation", mode="base")`         |      `$HOME/.paddlenlp/taskflow/word_segmentation/lac`       | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/lexical_analysis) |
-|       `Taskflow("word_segmentation", mode="accurate")`       |    `$HOME/.paddlenlp/taskflow/word_segmentation/wordtag`     | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/text_to_knowledge/ernie-ctm) |
-|                `Taskflow("ner", mode="fast")`                |             `$HOME/.paddlenlp/taskflow/ner/lac`              | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/lexical_analysis) |
-|              `Taskflow("ner", mode="accurate")`              |           `$HOME/.paddlenlp/taskflow/ner/wordtag`            | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/text_to_knowledge/ernie-ctm) |
-|     `Taskflow("text_correction", model="csc-ernie-1.0")`     |  `$HOME/.paddlenlp/taskflow/text_correction/csc-ernie-1.0`   | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/text_correction/ernie-csc) |
+|         `Taskflow("word_segmentation", mode="base")`         |             `$HOME/.paddlenlp/taskflow/lac`                  | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/lexical_analysis) |
+|       `Taskflow("word_segmentation", mode="accurate")`       |             `$HOME/.paddlenlp/taskflow/wordtag`              | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/text_to_knowledge/ernie-ctm) |
+|       `Taskflow("pos_tagging")`                              |             `$HOME/.paddlenlp/taskflow/lac`                  | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/lexical_analysis) |
+|                `Taskflow("ner", mode="fast")`                |             `$HOME/.paddlenlp/taskflow/lac`                  | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/lexical_analysis) |
+|              `Taskflow("ner", mode="accurate")`              |             `$HOME/.paddlenlp/taskflow/wordtag`              | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/text_to_knowledge/ernie-ctm) |
+|     `Taskflow("text_correction", model="ernie-csc")`     |  `$HOME/.paddlenlp/taskflow/text_correction/ernie-csc`   | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/text_correction/ernie-csc) |
 |      `Taskflow("dependency_parsing", model="ddparser")`      |   `$HOME/.paddlenlp/taskflow/dependency_parsing/ddparser`    | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/dependency_parsing/ddparser) |
 | `Taskflow("dependency_parsing", model="ddparser-ernie-1.0")` | `$HOME/.paddlenlp/taskflow/dependency_parsing/ddparser-ernie-1.0` | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/dependency_parsing/ddparser) |
 | `Taskflow("dependency_parsing", model="ddparser-ernie-gram-zh")` | `$HOME/.paddlenlp/taskflow/dependency_parsing/ddparser-ernie-gram-zh` | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/dependency_parsing/ddparser) |
 | `Taskflow("sentiment_analysis", model="skep_ernie_1.0_large_ch")` | `$HOME/.paddlenlp/taskflow/sentiment_analysis/skep_ernie_1.0_large_ch` | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/sentiment_analysis/skep) |
-|       `Taskflow("knowledge_mining", model="wordtag")`        |     `$HOME/.paddlenlp/taskflow/knowledge_mining/wordtag`     | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/text_to_knowledge/ernie-ctm) |
+|       `Taskflow("knowledge_mining", model="wordtag")`        |             `$HOME/.paddlenlp/taskflow/wordtag`              | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/text_to_knowledge/ernie-ctm) |
 |        `Taskflow("knowledge_mining", model="nptag")`         |      `$HOME/.paddlenlp/taskflow/knowledge_mining/nptag`      | [示例](https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/text_to_knowledge/nptag) |
 
 </div></details>  
@@ -647,10 +647,10 @@ nptag(["糖醋排骨", "红曲霉菌"])
 
 这里我们以命名实体识别`Taskflow("ner", mode="accurate")`为例，展示如何定制自己的模型。
 
-调用`Taskflow`接口后，程序自动将相关文件下载到`$HOME/.paddlenlp/taskflow/ner/wordtag/`，该默认路径包含以下文件:
+调用`Taskflow`接口后，程序自动将相关文件下载到`$HOME/.paddlenlp/taskflow/wordtag/`，该默认路径包含以下文件:
 
 ```text
-$HOME/.paddlenlp/taskflow/ner/wordtag/
+$HOME/.paddlenlp/taskflow/wordtag/
 ├── model_state.pdparams # 默认模型参数文件
 ├── model_config.json # 默认模型配置文件
 └── tags.txt # 默认标签文件
diff --git a/paddlenlp/taskflow/dependency_parsing.py b/paddlenlp/taskflow/dependency_parsing.py
@@ -84,7 +84,7 @@ class DDParserTask(Task):
 
     resource_files_names = {
         "model_state": "model_state.pdparams",
-        "word_vocab": "vocab.json",
+        "word_vocab": "word_vocab.json",
         "rel_vocab": "rel_vocab.json",
     }
     resource_files_urls = {
diff --git a/paddlenlp/taskflow/task.py b/paddlenlp/taskflow/task.py
@@ -33,9 +33,10 @@ class Task(metaclass=abc.ABCMeta):
         kwargs (dict, optional): Additional keyword arguments passed along to the specific task. 
     """
 
-    def __init__(self, model, task, **kwargs):
+    def __init__(self, model, task, priority_path=None, **kwargs):
         self.model = model
         self.task = task
+        self.priority_path = priority_path
         self.kwargs = kwargs
         self._usage = ""
         # The dygraph model instantce 
@@ -50,6 +51,9 @@ def __init__(self, model, task, **kwargs):
             'task_flag'] if 'task_flag' in self.kwargs else self.model
         if 'task_path' in self.kwargs:
             self._task_path = self.kwargs['task_path']
+        elif self.priority_path:
+            self._task_path = os.path.join(self._home_path, "taskflow",
+                                           self.priority_path)
         else:
             self._task_path = os.path.join(self._home_path, "taskflow",
                                            self.task, self.model)
diff --git a/paddlenlp/taskflow/taskflow.py b/paddlenlp/taskflow/taskflow.py
@@ -39,33 +39,80 @@
 warnings.simplefilter(action='ignore', category=Warning, lineno=0, append=False)
 
 TASKS = {
+    'dependency_parsing': {
+        "models": {
+            "ddparser": {
+                "task_class": DDParserTask,
+                "task_flag": 'dependency_parsing-biaffine',
+            },
+            "ddparser-ernie-1.0": {
+                "task_class": DDParserTask,
+                "task_flag": 'dependency_parsing-ernie-1.0',
+            },
+            "ddparser-ernie-gram-zh": {
+                "task_class": DDParserTask,
+                "task_flag": 'dependency_parsing-ernie-gram-zh',
+            },
+        },
+        "default": {
+            "model": "ddparser",
+        }
+    },
+    'dialogue': {
+        "models": {
+            "plato-mini": {
+                "task_class": DialogueTask,
+                "task_flag": "dialogue-plato-mini"
+            },
+        },
+        "default": {
+            "model": "plato-mini",
+        }
+    },
     "knowledge_mining": {
         "models": {
             "wordtag": {
                 "task_class": WordTagTask,
                 "task_flag": 'knowledge_mining-wordtag',
+                "task_priority_path": "wordtag",
             },
             "nptag": {
                 "task_class": NPTagTask,
                 "task_flag": 'knowledge_mining-nptag',
             },
         },
         "default": {
-            "model": "wordtag"
+            "model": "wordtag",
+        }
+    },
+    "lexical_analysis": {
+        "models": {
+            "lac": {
+                "task_class": LacTask,
+                "hidden_size": 128,
+                "emb_dim": 128,
+                "task_flag": 'lexical_analysis-gru_crf',
+                "task_priority_path": "lac",
+            }
+        },
+        "default": {
+            "model": "lac"
         }
     },
     "ner": {
         "modes": {
             "accurate": {
                 "task_class": NERWordTagTask,
                 "task_flag": "ner-wordtag",
+                "task_priority_path": "wordtag",
                 "linking": False,
             },
             "fast": {
                 "task_class": NERLACTask,
                 "hidden_size": 128,
                 "emb_dim": 128,
                 "task_flag": "ner-lac",
+                "task_priority_path": "lac",
             }
         },
         "default": {
@@ -77,69 +124,37 @@
             "gpt-cpm-large-cn": {
                 "task_class": PoetryGenerationTask,
                 "task_flag": 'poetry_generation-gpt-cpm-large-cn',
+                "task_priority_path": "gpt-cpm-large-cn",
             },
         },
         "default": {
             "model": "gpt-cpm-large-cn",
         }
     },
-    "question_answering": {
-        "models": {
-            "gpt-cpm-large-cn": {
-                "task_class": QuestionAnsweringTask,
-                "task_flag": 'question_answering-gpt-cpm-large-cn',
-            },
-        },
-        "default": {
-            "model": "gpt-cpm-large-cn",
-        }
-    },
-    "lexical_analysis": {
+    "pos_tagging": {
         "models": {
             "lac": {
-                "task_class": LacTask,
+                "task_class": POSTaggingTask,
                 "hidden_size": 128,
                 "emb_dim": 128,
-                "task_flag": 'lexical_analysis-gru_crf',
+                "task_flag": 'pos_tagging-gru_crf',
+                "task_priority_path": "lac",
             }
         },
         "default": {
             "model": "lac"
         }
     },
-    "word_segmentation": {
-        "modes": {
-            "fast": {
-                "task_class": SegJiebaTask,
-                "task_flag": "word_segmentation-jieba",
-            },
-            "base": {
-                "task_class": SegLACTask,
-                "hidden_size": 128,
-                "emb_dim": 128,
-                "task_flag": "word_segmentation-gru_crf",
-            },
-            "accurate": {
-                "task_class": SegWordTagTask,
-                "task_flag": "word_segmentation-wordtag",
-                "linking": False,
-            },
-        },
-        "default": {
-            "mode": "base"
-        }
-    },
-    "pos_tagging": {
+    "question_answering": {
         "models": {
-            "lac": {
-                "task_class": POSTaggingTask,
-                "hidden_size": 128,
-                "emb_dim": 128,
-                "task_flag": 'pos_tagging-gru_crf',
-            }
+            "gpt-cpm-large-cn": {
+                "task_class": QuestionAnsweringTask,
+                "task_flag": 'question_answering-gpt-cpm-large-cn',
+                "task_priority_path": "gpt-cpm-large-cn",
+            },
         },
         "default": {
-            "model": "lac"
+            "model": "gpt-cpm-large-cn",
         }
     },
     'sentiment_analysis': {
@@ -157,34 +172,15 @@
             "model": "bilstm"
         }
     },
-    'dependency_parsing': {
-        "models": {
-            "ddparser": {
-                "task_class": DDParserTask,
-                "task_flag": 'dependency_parsing-biaffine',
-            },
-            "ddparser-ernie-1.0": {
-                "task_class": DDParserTask,
-                "task_flag": 'dependency_parsing-ernie-1.0',
-            },
-            "ddparser-ernie-gram-zh": {
-                "task_class": DDParserTask,
-                "task_flag": 'dependency_parsing-ernie-gram-zh',
-            },
-        },
-        "default": {
-            "model": "ddparser"
-        }
-    },
     'text_correction': {
         "models": {
-            "csc-ernie-1.0": {
+            "ernie-csc": {
                 "task_class": CSCTask,
-                "task_flag": "text_correction-csc-ernie-1.0"
+                "task_flag": "text_correction-ernie-csc"
             },
         },
         "default": {
-            "model": "csc-ernie-1.0"
+            "model": "ernie-csc"
         }
     },
     'text_similarity': {
@@ -198,15 +194,28 @@
             "model": "simbert-base-chinese"
         }
     },
-    'dialogue': {
-        "models": {
-            "plato-mini": {
-                "task_class": DialogueTask,
-                "task_flag": "dialogue-plato-mini"
+    "word_segmentation": {
+        "modes": {
+            "fast": {
+                "task_class": SegJiebaTask,
+                "task_flag": "word_segmentation-jieba",
+            },
+            "base": {
+                "task_class": SegLACTask,
+                "hidden_size": 128,
+                "emb_dim": 128,
+                "task_flag": "word_segmentation-gru_crf",
+                "task_priority_path": "lac",
+            },
+            "accurate": {
+                "task_class": SegWordTagTask,
+                "task_flag": "word_segmentation-wordtag",
+                "task_priority_path": "wordtag",
+                "linking": False,
             },
         },
         "default": {
-            "model": "plato-mini"
+            "mode": "base"
         }
     },
 }
@@ -247,6 +256,13 @@ def __init__(self, task, model=None, mode=None, device_id=0, **kwargs):
             )), "The {} name:{} is not in task:[{}]".format(tag, model, task)
         else:
             self.model = TASKS[task]['default'][ind_tag]
+
+        if "task_priority_path" in TASKS[self.task][tag][self.model]:
+            self.priority_path = TASKS[self.task][tag][self.model][
+                "task_priority_path"]
+        else:
+            self.priority_path = None
+
         # Set the device for the task
         device = get_env_device()
         if device == 'cpu' or device_id == -1:
@@ -261,7 +277,10 @@ def __init__(self, task, model=None, mode=None, device_id=0, **kwargs):
         self.kwargs = kwargs
         task_class = TASKS[self.task][tag][self.model]['task_class']
         self.task_instance = task_class(
-            model=self.model, task=self.task, **self.kwargs)
+            model=self.model,
+            task=self.task,
+            priority_path=self.priority_path,
+            **self.kwargs)
         task_list = TASKS.keys()
         Taskflow.task_list = task_list
 
@@ -297,7 +316,7 @@ def from_segments(self, *inputs):
         return results
 
     def interactive_mode(self, max_turn):
-        with self.task_instance.interactive_mode(max_turn=3):
+        with self.task_instance.interactive_mode(max_turn):
             while True:
                 human = input("[Human]:").strip()
                 if human.lower() == "exit":
diff --git a/paddlenlp/taskflow/text_correction.py b/paddlenlp/taskflow/text_correction.py
@@ -58,7 +58,7 @@
 
          """
 
-TASK_MODEL_MAP = {"csc-ernie-1.0": "ernie-1.0"}
+TASK_MODEL_MAP = {"ernie-csc": "ernie-1.0"}
 
 
 class CSCTask(Task):
@@ -75,13 +75,13 @@ class CSCTask(Task):
         "pinyin_vocab": "pinyin_vocab.txt"
     }
     resource_files_urls = {
-        "csc-ernie-1.0": {
+        "ernie-csc": {
             "model_state": [
-                "https://bj.bcebos.com/paddlenlp/taskflow/text_correction/csc-ernie-1.0/model_state.pdparams",
+                "https://bj.bcebos.com/paddlenlp/taskflow/text_correction/ernie-csc/model_state.pdparams",
                 "cdc53e7e3985ffc78fedcdf8e6dca6d2"
             ],
             "pinyin_vocab": [
-                "https://bj.bcebos.com/paddlenlp/taskflow/text_correction/csc-ernie-1.0/pinyin_vocab.txt",
+                "https://bj.bcebos.com/paddlenlp/taskflow/text_correction/ernie-csc/pinyin_vocab.txt",
                 "5599a8116b6016af573d08f8e686b4b2"
             ],
         }
diff --git a/paddlenlp/utils/downloader.py b/paddlenlp/utils/downloader.py

Original file line number	Diff line number	Diff line change
`@@ -84,7 +84,7 @@ class DDParserTask(Task):`
`84`	`84`
`85`	`85`	`resource_files_names = {`
`86`	`86`	`"model_state": "model_state.pdparams",`
`87`		`- "word_vocab": "vocab.json",`
	`87`	`+ "word_vocab": "word_vocab.json",`
`88`	`88`	`"rel_vocab": "rel_vocab.json",`
`89`	`89`	`}`
`90`	`90`	`resource_files_urls = {`