Predict SER, RE KIE with my ocr

[minhduc01168]

I want to write code that predicts SER, RE with my OCR model? But I haven't determined what the input format of the KIE ser model is? Can you tell me or is there any tutorial you know that can help me? Thank you very much

[GreatV]

maybe you can refer:

• https://github.com/PaddlePaddle/PaddleOCR/blob/main/doc/doc_en/kie_en.md
• https://github.com/Jacky0111/PaddleOCR-Tutorial

[minhduc01168]

@GreatV Do you have any documentation on the SER model input format so I can create my ocr model output accordingly to load into the ser model?

[GreatV]

You may need to dive into the code

• https://github.com/PaddlePaddle/PaddleOCR/blob/main/tools/infer_kie.py
• https://github.com/PaddlePaddle/PaddleOCR/blob/main/tools/infer_kie_token_ser.py
• https://github.com/PaddlePaddle/PaddleOCR/blob/main/tools/infer_kie_token_ser_re.py

[minhduc01168]

@GreatV I have rewritten the ocr function in paddleocr.py file:
def ocr_v2(
self,
img,
json_path=None,
det=False,
rec=False,
cls=False,
bin=False,
inv=False,
alpha_color=(255, 255, 255),
slice={},
):
"""
OCR with PaddleOCR

    args:
        img: img for OCR, support ndarray, img_path and list or ndarray
        json_path: path to the JSON file for OCR information
        det: use text detection or not. If False, only rec will be exec. Default is True
        rec: use text recognition or not. If False, only det will be exec. Default is True
        cls: use angle classifier or not. Default is True. If True, the text with rotation of 180 degrees can be recognized. If no text is rotated by 180 degrees, use cls=False to get better performance. Text with rotation of 90 or 270 degrees can be recognized even if cls=False.
        bin: binarize image to black and white. Default is False.
        inv: invert image colors. Default is False.
        alpha_color: set RGB color Tuple for transparent parts replacement. Default is pure white.
        slice: use sliding window inference for large images, det and rec must be True. Requires int values for slice["horizontal_stride"], slice["vertical_stride"], slice["merge_x_thres"], slice["merge_y_thres] (See doc/doc_en/slice_en.md). Default is {}.
    """
    assert isinstance(img, (np.ndarray, list, str, bytes))
    if isinstance(img, list) and det == True:
        logger.error("When input a list of images, det must be false")
        exit(0)
    if cls == True and self.use_angle_cls == False:
        logger.warning(
            "Since the angle classifier is not initialized, it will not be used during the forward process"
        )

    img, flag_gif, flag_pdf = check_img(img, alpha_color)
    # for infer pdf file
    if isinstance(img, list) and flag_pdf:
        if self.page_num > len(img) or self.page_num == 0:
            imgs = img
        else:
            imgs = img[: self.page_num]
    else:
        imgs = [img]

    if json_path:
        text_all, list_box, list_score = json_extract(json_path)
        list_box_new = []
        for box_line in list_box:
            merged_boxes = merge_two_groups_of_boxes(box_line)
            list_box_new.append(merged_boxes)

        list_box_kie = []
        for box_line in list_box_new:
            list_box_line = []
            for box_subline in box_line:
                box_kie = [[box_subline[0], box_subline[1]], [box_subline[2], box_subline[1]],
                           [box_subline[2], box_subline[3]], [box_subline[0], box_subline[3]]]
                list_box_line.append(box_kie)
            list_box_kie.append(list_box_line)

        list_score_new = []
        for score_line in list_score:
            score_line_new = []
            for score_subline in score_line:
                if len(score_subline) > 0:
                    score_subline_new = min(score_subline)
                    score_line_new.append(score_subline_new)
            list_score_new.append(score_line_new)

        output = transform_data(text_all, list_box_kie, list_score_new)
        return output

    def preprocess_image(_image):
        _image = alpha_to_color(_image, alpha_color)
        if inv:
            _image = cv2.bitwise_not(_image)
        if bin:
            _image = binarize_img(_image)
        return _image

    if det and rec:
        ocr_res = []
        for idx, img in enumerate(imgs):
            img = preprocess_image(img)
            dt_boxes, rec_res, _ = self.__call__(img, cls, slice)
            if not dt_boxes and not rec_res:
                ocr_res.append(None)
                continue
            tmp_res = [[box.tolist(), res] for box, res in zip(dt_boxes, rec_res)]
            ocr_res.append(tmp_res)
        return ocr_res
    elif det and not rec:
        ocr_res = []
        for idx, img in enumerate(imgs):
            img = preprocess_image(img)
            dt_boxes, elapse = self.text_detector(img)
            if dt_boxes.size == 0:
                ocr_res.append(None)
                continue
            tmp_res = [box.tolist() for box in dt_boxes]
            ocr_res.append(tmp_res)
        return ocr_res
    else:
        ocr_res = []
        cls_res = []
        for idx, img in enumerate(imgs):
            if not isinstance(img, list):
                img = preprocess_image(img)
                img = [img]
            if self.use_angle_cls and cls:
                img, cls_res_tmp, elapse = self.text_classifier(img)
                if not rec:
                    cls_res.append(cls_res_tmp)
            rec_res, elapse = self.text_recognizer(img)
            ocr_res.append(rec_res)
        if not rec:
            return cls_res
        return ocr_res

And edit the _load_ocr_info function in the label_ops.py file:
def _load_ocr_info(self, data):
if self.inf_er_mode:
# ocr_result = self.ocr_engine.ocr(data["image"], cls=False)[0]
ocr_result = self.ocr_engine.ocr_v2(data["image"], json_path="/home/minh/CVS/Key_Information_Extraction/200-mau/1a22222931c84aa7b39af02ee93e53b9.json")[0]
print(ocr_result)
ocr_info = []
for res in ocr_result:
ocr_info.append(
{
"transcription": res[1][0],
"bbox": self.trans_poly_to_bbox(res[0]),
"points": res[0],
}
)
print(ocr_info)
return ocr_info
else:
info = data["label"]
# read text info
info_dict = json.loads(info)
return info_dict
But when I run the code I get the following error. Do you know what error is happening to me and how to solve it. Thank you
Error:
[2024-06-06 08:37:12,786] [ ERROR] - Downloading from https://bj.bcebos.com/paddlenlp/models/community/.output/ser_vi_layoutxlm_giaykhaisinh_zh/best_accuracy/model_state.pdparams failed with code 404!
Traceback (most recent call last):
File "/home/teamai_cvs/anaconda3/envs/KIE/lib/python3.9/site-packages/paddlenlp/transformers/model_utils.py", line 549, in from_pretrained
resolved_resource_files[file_id] = get_path_from_url_with_filelock(file_path, cache_dir)
File "/home/teamai_cvs/anaconda3/envs/KIE/lib/python3.9/site-packages/paddlenlp/utils/downloader.py", line 167, in get_path_from_url_with_filelock
result = get_path_from_url(url=url, root_dir=root_dir, md5sum=md5sum, check_exist=check_exist)
File "/home/teamai_cvs/anaconda3/envs/KIE/lib/python3.9/site-packages/paddlenlp/utils/downloader.py", line 132, in get_path_from_url
fullpath = _download(url, root_dir, md5sum)
File "/home/teamai_cvs/anaconda3/envs/KIE/lib/python3.9/site-packages/paddlenlp/utils/downloader.py", line 193, in _download
raise RuntimeError("Downloading from {} failed with code " "{}!".format(url, req.status_code))
RuntimeError: Downloading from https://bj.bcebos.com/paddlenlp/models/community/.output/ser_vi_layoutxlm_giaykhaisinh_zh/best_accuracy/model_state.pdparams failed with code 404!

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/teamai_cvs/CVS/Key_Information_Extraction/PaddleOCR_GKS_v2_re/tools/infer_kie_token_ser.py", line 130, in
ser_engine = SerPredictor(config)
File "/home/teamai_cvs/CVS/Key_Information_Extraction/PaddleOCR_GKS_v2_re/tools/infer_kie_token_ser.py", line 70, in init
self.model = build_model(config["Architecture"])
File "/home/teamai_cvs/CVS/Key_Information_Extraction/PaddleOCR_GKS_v2_re/ppocr/modeling/architectures/init.py", line 30, in build_model
arch = BaseModel(config)
File "/home/teamai_cvs/CVS/Key_Information_Extraction/PaddleOCR_GKS_v2_re/ppocr/modeling/architectures/base_model.py", line 55, in init
self.backbone = build_backbone(config["Backbone"], model_type)
File "/home/teamai_cvs/CVS/Key_Information_Extraction/PaddleOCR_GKS_v2_re/ppocr/modeling/backbones/init.py", line 129, in build_backbone
module_class = eval(module_name)(**config)
File "/home/teamai_cvs/CVS/Key_Information_Extraction/PaddleOCR_GKS_v2_re/ppocr/modeling/backbones/vqa_layoutlm.py", line 153, in init
super(LayoutXLMForSer, self).init(
File "/home/teamai_cvs/CVS/Key_Information_Extraction/PaddleOCR_GKS_v2_re/ppocr/modeling/backbones/vqa_layoutlm.py", line 65, in init
self.model = model_class.from_pretrained(checkpoints)
File "/home/teamai_cvs/anaconda3/envs/KIE/lib/python3.9/site-packages/paddlenlp/transformers/model_utils.py", line 552, in from_pretrained
raise RuntimeError(
RuntimeError: Can't load weights for '.output/ser_vi_layoutxlm_giaykhaisinh_zh/best_accuracy'.
Please make sure that '.output/ser_vi_layoutxlm_giaykhaisinh_zh/best_accuracy' is:

• a correct model-identifier of built-in pretrained models,
• or a correct model-identifier of community-contributed pretrained models,
• or the correct path to a directory containing relevant modeling files(model_weights and model_config).

[Bodarobi]

[email protected]

[Bodarobi]

---

[GreatV]

@minhduc01168 Please verify that the pre-trained model path is correct: https://bj.bcebos.com/paddlenlp/models/community/.output/ser_vi_layoutxlm_giaykhaisinh_zh/best_accuracy/model_state.pdparams, and the current URL is incorrect.

[minhduc01168]

Thank you very much it solved my problem