Create README.md

Author: BOBrown

.gitignore

@@ -0,0 +1,7 @@
+dist/
+build/
+**.egg-info/
+**__pycache__/
+ckpts/
+**version.py
+

README.md

@@ -0,0 +1,77 @@
+<div align="center">
+<h1>StructEqTable-Deploy: A High-efficiency Open-source Toolkit for Table-to-Latex Transformation</h1>
+
+
+[[ Related Paper ]](https://arxiv.org/abs/2406.11633) [[ Website ]](https://unimodal4reasoning.github.io/DocGenome_page/) [[ Dataset (Google Drive)]](https://drive.google.com/drive/folders/1OIhnuQdIjuSSDc_QL2nP4NwugVDgtItD) [[ Dataset (Hugging Face) ]](https://huggingface.co/datasets/U4R/DocGenome/tree/main)
+
+[[Models 🤗(Hugging Face)]](https://huggingface.co/U4R/StructTable-base/tree/main)
+
+
+</div>
+
+Welcome to the official repository of StructEqTable-Deploy, a solution that converts images of Table into LaTeX, powered by scalable data from [DocGenome benchmark](https://unimodal4reasoning.github.io/DocGenome_page/).
+
+
+## Abstract
+Table is an effective way to represent structured data in scientific publications, financial statements, invoices, web pages, and many other scenarios. Extracting tabular data from a visual table image and performing the downstream reasoning tasks according to the extracted data is challenging, mainly due to that tables often present complicated column and row headers with spanning cell operation. To address these challenges, we present TableX, a large-scale multi-modal table benchmark extracted from [DocGenome benchmark](https://unimodal4reasoning.github.io/DocGenome_page/) for table pre-training, comprising more than 2 million high-quality Image-LaTeX pair data covering 156 disciplinary classes. Besides, benefiting from such large-scale data, we train an end-to-end model, StructEqTable, which provides the capability to precisely obtain the corresponding LaTeX description from a visual table image and perform multiple table-related reasoning tasks, including structural extraction and question answering, broadening its application scope and potential.
+
+
+## TODO
+
+- [x] Release inference code and checkpoints of StructEqTable.
+- [ ] Support Chinese version of StructEqTable.
+- [ ] Improve the inference speed of StructEqTable.
+
+
+### Installation
+
+``` bash 
+conda create -n structeqtable python=3.9
+
+conda activate structeqtable
+
+pip install "git+https://github.com/UniModal4Reasoning/StructEqTable-Deploy.git"
+
+```
+
+## Demo
+- run the demo.py
+```shell script
+cd demo
+python demo.py \
+--image_path demo/demo.png \
+--ckpt_path ${CKPT_PATH}
+```
+
+- Visualization Results
+    - The input data are sampled from SciHub domain.
+
+![](demo/demo_1.png)
+
+![](demo/demo_2.png)
+
+
+## Acknowledgements
+- [UniMERNet](https://github.com/opendatalab/UniMERNet). A Universal Network for Real-World Mathematical Expression Recognition.
+- [DocGenome](https://github.com/UniModal4Reasoning/DocGenome). An Open Large-scale Scientific Document Benchmark for Training and Testing Multi-modal Large Models.
+- [ChartVLM](https://github.com/UniModal4Reasoning/ChartVLM). A Versatile Benchmark and Foundation Model for Complicated Chart Reasoning.
+- [Donut](https://huggingface.co/naver-clova-ix/donut-base). The UniMERNet's Transformer Encoder-Decoder are referenced from Donut.
+- [Nougat](https://github.com/facebookresearch/nougat). The tokenizer uses Nougat.
+
+
+## License
+[Apache License 2.0](LICENSE)
+
+## Citation
+If you find our models / code / papers useful in your research, please consider giving ⭐ and citations 📝, thx :)  
+```bibtex
+@article{xia2024docgenome,
+  title={DocGenome: An Open Large-scale Scientific Document Benchmark for Training and Testing Multi-modal Large Language Models},
+  author={Xia, Renqiu and Mao, Song and Yan, Xiangchao and Zhou, Hongbin and Zhang, Bo and Peng, Haoyang and Pi, Jiahao and Fu, Daocheng and Wu, Wenjie and Ye, Hancheng and others},
+  journal={arXiv preprint arXiv:2406.11633},
+  year={2024}
+}
+```
+
+## Contact Us
+If you encounter any issues or have questions, please feel free to contact us via [email protected].

demo/demo.png

@@ -0,0 +1,31 @@
+import torch
+import argparse
+
+from PIL import Image
+from struct_eqtable import build_model
+
+
+def parse_config():
+    parser = argparse.ArgumentParser(description='arg parser')
+    parser.add_argument('--image_path', type=str, default='demo.png', help='data path for table image')
+    parser.add_argument('--ckpt_path', type=str, default='', help='ckpt path for table model')
+    args = parser.parse_args()
+    return args
+
+def main():
+    args = parse_config()
+
+    # build model
+    model = build_model(args.ckpt_path, max_new_tokens=4096, max_time=120)
+
+    # model inference
+    raw_image = Image.open(args.image_path)
+    with torch.no_grad():
+        output = model(raw_image)
+
+    # show output latex code of table
+    for i, latex_code in enumerate(output):
+        print(f"Table {i}:\n{latex_code}")
+
+if __name__ == '__main__':
+    main()

demo/demo.py

@@ -0,0 +1,2 @@
+torch
+transformers

demo/demo_1.png

@@ -0,0 +1,25 @@
+from setuptools import find_packages, setup
+
+
+def write_version_to_file(version, target_file):
+    with open(target_file, 'w') as f:
+        print('__version__ = "%s"' % version, file=f)
+
+if __name__ == '__main__':
+    version = '0.1.0'
+    write_version_to_file(version, 'struct_eqtable/version.py')
+
+    setup(
+        name='struct_eqtable',
+        version=version,
+        description='A High-efficiency Open-source Toolkit for Table-to-Latex Transformation',
+        install_requires=[
+            'torch',
+            'transformers',
+        ],
+
+        author='Hongbin Zhou, Xiangchao Yan, Bo Zhang',
+        author_email='[email protected]',
+        license='Apache License 2.0',
+        packages=find_packages(exclude=['demo']),
+    )

demo/demo_2.png

@@ -0,0 +1,7 @@
+from .model import StructTable
+
+
+def build_model(model_ckpt, **kwargs):
+    
+    model = StructTable(model_ckpt, **kwargs)
+    return model

requirements.txt

@@ -0,0 +1,41 @@
+import torch
+
+from torch import nn
+from transformers import AutoModelForVision2Seq, AutoProcessor
+
+
+class StructTable(nn.Module):
+    def __init__(self, model_path, max_new_tokens=2048, max_time=60):
+        super().__init__()
+        self.model_path = model_path
+        self.max_new_tokens = max_new_tokens
+        self.max_generate_time = max_time
+
+        # init model and image processor from ckpt path
+        self.init_image_processor(model_path)
+        self.init_model(model_path)
+    
+    def init_model(self, model_path):
+        self.model = AutoModelForVision2Seq.from_pretrained(model_path)
+        self.model.eval()
+    
+    def init_image_processor(self, image_processor_path):
+        self.data_processor = AutoProcessor.from_pretrained(image_processor_path)
+
+    def forward(self, image):
+        # process image to tokens
+        image_tokens = self.data_processor.image_processor(
+            images=image,
+            return_tensors='pt',
+        )
+
+        # generate text from image tokens
+        model_output = self.model.generate(
+            flattened_patches=image_tokens['flattened_patches'],
+            attention_mask=image_tokens['attention_mask'], 
+            max_new_tokens=self.max_new_tokens,
+            max_time=self.max_generate_time
+        )
+        latex_codes = self.data_processor.batch_decode(model_output, skip_special_tokens=True)
+
+        return latex_codes