|
| 1 | +# coding=utf-8 |
| 2 | +""" |
| 3 | + @project: maxkb |
| 4 | + @Author：虎 |
| 5 | + @file： md_parse_qa_handle.py |
| 6 | + @date：2024/5/21 14:59 |
| 7 | + @desc: |
| 8 | +""" |
| 9 | +import re |
| 10 | +import traceback |
| 11 | + |
| 12 | +from charset_normalizer import detect |
| 13 | + |
| 14 | +from common.handle.base_parse_qa_handle import BaseParseQAHandle, get_title_row_index_dict, get_row_value |
| 15 | +from common.utils.logger import maxkb_logger |
| 16 | + |
| 17 | + |
| 18 | +class MarkdownParseQAHandle(BaseParseQAHandle): |
| 19 | + def support(self, file, get_buffer): |
| 20 | + file_name: str = file.name.lower() |
| 21 | + if file_name.endswith(".md") or file_name.endswith(".markdown"): |
| 22 | + return True |
| 23 | + return False |
| 24 | + |
| 25 | + def parse_markdown_table(self, content): |
| 26 | + """解析 Markdown 表格,返回表格数据列表""" |
| 27 | + tables = [] |
| 28 | + lines = content.split('\n') |
| 29 | + i = 0 |
| 30 | + |
| 31 | + while i < len(lines): |
| 32 | + line = lines[i].strip() |
| 33 | + # 检测表格开始(包含 | 符号) |
| 34 | + if '|' in line and line.startswith('|'): |
| 35 | + table_data = [] |
| 36 | + # 读取表头 |
| 37 | + header = [cell.strip() for cell in line.split('|')[1:-1]] |
| 38 | + table_data.append(header) |
| 39 | + i += 1 |
| 40 | + |
| 41 | + # 跳过分隔行 (例如: | --- | --- |) |
| 42 | + if i < len(lines) and re.match(r'\s*\|[\s\-:]+\|\s*', lines[i]): |
| 43 | + i += 1 |
| 44 | + |
| 45 | + # 读取数据行 |
| 46 | + while i < len(lines): |
| 47 | + line = lines[i].strip() |
| 48 | + if not line.startswith('|'): |
| 49 | + break |
| 50 | + row = [cell.strip() for cell in line.split('|')[1:-1]] |
| 51 | + if len(row) > 0: |
| 52 | + table_data.append(row) |
| 53 | + i += 1 |
| 54 | + |
| 55 | + if len(table_data) > 1: # 至少有表头和一行数据 |
| 56 | + tables.append(table_data) |
| 57 | + else: |
| 58 | + i += 1 |
| 59 | + |
| 60 | + return tables |
| 61 | + |
| 62 | + def handle(self, file, get_buffer, save_image): |
| 63 | + buffer = get_buffer(file) |
| 64 | + try: |
| 65 | + # 检测编码并读取文件内容 |
| 66 | + encoding = detect(buffer)['encoding'] |
| 67 | + content = buffer.decode(encoding if encoding else 'utf-8') |
| 68 | + |
| 69 | + # 解析 Markdown 表格 |
| 70 | + tables = self.parse_markdown_table(content) |
| 71 | + |
| 72 | + if not tables: |
| 73 | + return [{'name': file.name, 'paragraphs': []}] |
| 74 | + |
| 75 | + paragraph_list = [] |
| 76 | + |
| 77 | + # 处理每个表格 |
| 78 | + for table in tables: |
| 79 | + if len(table) < 2: |
| 80 | + continue |
| 81 | + |
| 82 | + title_row_list = table[0] |
| 83 | + title_row_index_dict = get_title_row_index_dict(title_row_list) |
| 84 | + |
| 85 | + # 处理表格的每一行数据 |
| 86 | + for row in table[1:]: |
| 87 | + content = get_row_value(row, title_row_index_dict, 'content') |
| 88 | + if content is None: |
| 89 | + continue |
| 90 | + |
| 91 | + problem = get_row_value(row, title_row_index_dict, 'problem_list') |
| 92 | + problem = str(problem) if problem is not None else '' |
| 93 | + problem_list = [{'content': p[0:255]} for p in problem.split('\n') if len(p.strip()) > 0] |
| 94 | + |
| 95 | + title = get_row_value(row, title_row_index_dict, 'title') |
| 96 | + title = str(title) if title is not None else '' |
| 97 | + |
| 98 | + paragraph_list.append({ |
| 99 | + 'title': title[0:255], |
| 100 | + 'content': content[0:102400], |
| 101 | + 'problem_list': problem_list |
| 102 | + }) |
| 103 | + |
| 104 | + return [{'name': file.name, 'paragraphs': paragraph_list}] |
| 105 | + |
| 106 | + except Exception as e: |
| 107 | + maxkb_logger.error(f"Error processing Markdown file {file.name}: {e}, {traceback.format_exc()}") |
| 108 | + return [{'name': file.name, 'paragraphs': []}] |
0 commit comments