don not add extra CombineSegment (#610)

pavel-esir · web-flow · commit eabd467039c6 · 2026-02-13T12:41:18.000Z
diff --git a/python/openvino_tokenizers/hf_parser.py b/python/openvino_tokenizers/hf_parser.py
@@ -289,7 +289,7 @@ def post_tokenization(self) -> None:
         ):
             self.add_truncation()
             self.pipeline.add_steps(
-                CombineSegmentsStep([Sequence() for _ in range(self.number_of_inputs)], add_special_tokens=False)
+                CombineSegmentsStep([Sequence()], add_special_tokens=False)
             )
             self.add_padding(use_max_padding=self.use_max_padding)
             return

Original file line number	Diff line number	Diff line change
`@@ -289,7 +289,7 @@ def post_tokenization(self) -> None:`
`289`	`289`	`):`
`290`	`290`	`self.add_truncation()`
`291`	`291`	`self.pipeline.add_steps(`
`292`		`- CombineSegmentsStep([Sequence() for _ in range(self.number_of_inputs)], add_special_tokens=False)`
	`292`	`+ CombineSegmentsStep([Sequence()], add_special_tokens=False)`
`293`	`293`	`)`
`294`	`294`	`self.add_padding(use_max_padding=self.use_max_padding)`
`295`	`295`	`return`