Skip to content

Commit 583f724

Browse files
authored
fix c_ops compatibility (#1450)
* fix c_ops compatibility with initial breakdowns and track and signalling compatibility issues under paddle 2.2.1
1 parent d1140bb commit 583f724

File tree

1 file changed

+14
-2
lines changed

1 file changed

+14
-2
lines changed

paddlenlp/experimental/faster_tokenizer.py

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,16 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
import importlib
16+
1517
import paddle
1618
import paddle.fluid.core as core
1719
import paddle.nn as nn
1820
from paddle.fluid.layer_helper import LayerHelper
1921
from paddle.fluid.framework import in_dygraph_mode
2022
from paddlenlp.utils.downloader import get_path_from_url
2123
from paddlenlp.transformers import BertTokenizer, ErnieTokenizer, RobertaTokenizer
22-
from paddle import _C_ops
24+
from paddlenlp.utils.log import logger
2325

2426
__all__ = ["to_tensor", "to_vocab_buffer", "FasterTokenizer"]
2527

@@ -77,6 +79,15 @@ class FasterTokenizer(nn.Layer):
7779

7880
def __init__(self, vocab, do_lower_case=False, is_split_into_words=False):
7981
super(FasterTokenizer, self).__init__()
82+
83+
try:
84+
self.mod = importlib.import_module("paddle._C_ops")
85+
except Exception as e:
86+
logger.warning(
87+
f"The paddlepaddle version is {paddle.__version__}, not the latest. "
88+
"Please upgrade the paddlepaddle package (>= 2.2.1).")
89+
self.mod = importlib.import_module("paddle.fluid.core.ops")
90+
8091
vocab_buffer = to_vocab_buffer(vocab, "vocab")
8192
self.register_buffer("vocab", vocab_buffer, persistable=True)
8293

@@ -94,11 +105,12 @@ def forward(self,
94105
if text_pair is not None:
95106
if isinstance(text_pair, list) or isinstance(text_pair, tuple):
96107
text_pair = to_tensor(list(text_pair))
97-
input_ids, seg_ids = _C_ops.faster_tokenizer(
108+
input_ids, seg_ids = self.mod.faster_tokenizer(
98109
self.vocab, text, text_pair, "do_lower_case",
99110
self.do_lower_case, "max_seq_len", max_seq_len,
100111
"pad_to_max_seq_len", pad_to_max_seq_len, "is_split_into_words",
101112
self.is_split_into_words)
113+
102114
return input_ids, seg_ids
103115

104116
attrs = {

0 commit comments

Comments
 (0)