Merge branch 'main' of https://github.com/open-sciencelab/GraphGen into baseline

Author: ChenZiHong-Gavin

.env.example

@@ -14,6 +14,14 @@ TRAINEE_MODEL=gpt-4o-mini
 TRAINEE_BASE_URL=
 TRAINEE_API_KEY=
 
+# azure_openai_api
+# SYNTHESIZER_BACKEND=azure_openai_api
+# The following is the same as your "Deployment name" in Azure
+# SYNTHESIZER_MODEL=<your-deployment-name>
+# SYNTHESIZER_BASE_URL=https://<your-resource-name>.openai.azure.com/openai/deployments/<your-deployment-name>/chat/completions
+# SYNTHESIZER_API_KEY=
+# SYNTHESIZER_API_VERSION=<api-version>
+
 # # ollama_api
 # SYNTHESIZER_BACKEND=ollama_api
 # SYNTHESIZER_MODEL=gemma3

.github/contributing.md

@@ -0,0 +1,34 @@
+## Contribution Guide
+Here are the steps to contribute to this project:
+
+1. Star this repository.
+2. Fork this repository.
+   
+   Type the following command on Git bash console:
+   ```bash
+   git clone https://github.com/open-sciencelab/GraphGen.git
+   ```
+
+3. Create a new branch
+   
+    Now before making changes to the files, go to your terminal under the repo you just cloned, and type the following:
+    
+    ```bash
+    git checkout -b add-my-name
+    ```
+  
+    By running the above command, you just created a new branch called add-my-name and checked it out, what this does is that it creates a new branch with the commit history of the master branch or the branch that you were on previously.
+
+4. Make your changes and push your code.
+
+   ```
+   git add .
+   git commit -m "xxx"
+   git push
+   ```
+
+   This will create a new commit with the changes you made.
+
+5. Now create a pull request and add the title.
+
+     Sit back and relax while your pull request is being reviewed and merged.

.pylintrc

@@ -100,7 +100,7 @@ source-roots=
 
 # When enabled, pylint would attempt to guess common misconfiguration and emit
 # user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
+# suggestion-mode=yes
 
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
@@ -452,6 +452,7 @@ disable=raw-checker-failed,
         R0917,  # Too many positional arguments (6/5) (too-many-positional-arguments)
         C0103,
         E0401,
+        W0718, # Catching too general exception Exception (broad-except)
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

README.md

@@ -16,7 +16,6 @@
 
 [![Hugging Face](https://img.shields.io/badge/Demo-on%20HF-blue?logo=huggingface&logoColor=yellow)](https://huggingface.co/spaces/chenzihong/GraphGen)
 [![Model Scope](https://img.shields.io/badge/%F0%9F%A4%96%20Demo-on%20MS-green)](https://modelscope.cn/studios/chenzihong/GraphGen)
-[![OpenXLab](https://img.shields.io/badge/Demo-on%20OpenXLab-blue?logo=openxlab&logoColor=yellow)](https://g-app-center-120612-6433-jpdvmvp.openxlab.space)
 
 
 GraphGen: Enhancing Supervised Fine-Tuning for LLMs with Knowledge-Driven Synthetic Data Generation
@@ -63,13 +62,14 @@ After data generation, you can use [LLaMA-Factory](https://github.com/hiyouga/LL
 
 ## 📌 Latest Updates
 
+- **2025.12.1**: Added search support for [NCBI](https://www.ncbi.nlm.nih.gov/) and [RNAcentral](https://rnacentral.org/) databases, enabling extraction of DNA and RNA data from these bioinformatics databases.
 - **2025.10.30**: We support several new LLM clients and inference backends including [Ollama_client](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/api/ollama_client.py), [http_client](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/api/http_client.py), [HuggingFace Transformers](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/local/hf_wrapper.py) and [SGLang](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/local/sglang_wrapper.py).
 - **2025.10.23**: We support VQA(Visual Question Answering) data generation now. Run script: `bash scripts/generate/generate_vqa.sh`.
-- **2025.10.21**: We support PDF as input format for data generation now via [MinerU](https://github.com/opendatalab/MinerU).
 
 <details>
 <summary>History</summary>
 
+- **2025.10.21**: We support PDF as input format for data generation now via [MinerU](https://github.com/opendatalab/MinerU).
 - **2025.09.29**: We auto-update gradio demo on [Hugging Face](https://huggingface.co/spaces/chenzihong/GraphGen) and [ModelScope](https://modelscope.cn/studios/chenzihong/GraphGen).
 - **2025.08.14**: We have added support for community detection in knowledge graphs using the Leiden algorithm, enabling the synthesis of Chain-of-Thought (CoT) data.
 - **2025.07.31**: We have added Google, Bing, Wikipedia, and UniProt as search back-ends.
@@ -83,9 +83,10 @@ After data generation, you can use [LLaMA-Factory](https://github.com/hiyouga/LL
 We support various LLM inference servers, API servers, inference clients, input file formats, data modalities, output data formats, and output data types.
 Users can flexibly configure according to the needs of synthetic data.
 
-| Inference Server                             | Api Server                                                                     | Inference Client                                           | Input File Format                  | Data Modal    | Data Format                  | Data Type                                       |
-|----------------------------------------------|--------------------------------------------------------------------------------|------------------------------------------------------------|------------------------------------|---------------|------------------------------|-------------------------------------------------|
-| [![hf-icon]HF][hf]<br>[![sg-icon]SGLang][sg] | [![sif-icon]Silicon][sif]<br>[![oai-icon]OpenAI][oai]<br>[![az-icon]Azure][az] | HTTP<br>[![ol-icon]Ollama][ol]<br>[![oai-icon]OpenAI][oai] | CSV<br>JSON<br>JSONL<br>PDF<br>TXT | TEXT<br>IMAGE | Alpaca<br>ChatML<br>Sharegpt | Aggregated<br>Atomic<br>CoT<br>Multi-hop<br>VQA |
+
+| Inference Server                             | Api Server                                                                     | Inference Client                                           | Data Source                                                                                                                                                                                                                                                                           | Data Modal    | Data Type                                       |
+|----------------------------------------------|--------------------------------------------------------------------------------|------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------|
+| [![hf-icon]HF][hf]<br>[![sg-icon]SGLang][sg] | [![sif-icon]Silicon][sif]<br>[![oai-icon]OpenAI][oai]<br>[![az-icon]Azure][az] | HTTP<br>[![ol-icon]Ollama][ol]<br>[![oai-icon]OpenAI][oai] | Files(CSV, JSON, PDF, TXT, etc.)<br>Databases([![uniprot-icon]UniProt][uniprot], [![ncbi-icon]NCBI][ncbi], [![rnacentral-icon]RNAcentral][rnacentral])<br>Search Engines([![bing-icon]Bing][bing], [![google-icon]Google][google])<br>Knowledge Graphs([![wiki-icon]Wikipedia][wiki]) | TEXT<br>IMAGE | Aggregated<br>Atomic<br>CoT<br>Multi-hop<br>VQA |
 
 <!-- links -->
 [hf]: https://huggingface.co/docs/transformers/index
@@ -94,6 +95,13 @@ Users can flexibly configure according to the needs of synthetic data.
 [oai]: https://openai.com
 [az]: https://azure.microsoft.com/en-us/services/cognitive-services/openai-service/
 [ol]: https://ollama.com
+[uniprot]: https://www.uniprot.org/
+[ncbi]: https://www.ncbi.nlm.nih.gov/
+[rnacentral]: https://rnacentral.org/
+[wiki]: https://www.wikipedia.org/
+[bing]: https://www.bing.com/
+[google]: https://www.google.com
+
 
 <!-- icons -->
 [hf-icon]: https://www.google.com/s2/favicons?domain=https://huggingface.co
@@ -103,11 +111,17 @@ Users can flexibly configure according to the needs of synthetic data.
 [az-icon]: https://www.google.com/s2/favicons?domain=https://azure.microsoft.com
 [ol-icon]: https://www.google.com/s2/favicons?domain=https://ollama.com
 
+[uniprot-icon]: https://www.google.com/s2/favicons?domain=https://www.uniprot.org
+[ncbi-icon]: https://www.google.com/s2/favicons?domain=https://www.ncbi.nlm.nih.gov/
+[rnacentral-icon]: https://www.google.com/s2/favicons?domain=https://rnacentral.org/
+[wiki-icon]: https://www.google.com/s2/favicons?domain=https://www.wikipedia.org/
+[bing-icon]: https://www.google.com/s2/favicons?domain=https://www.bing.com/
+[google-icon]: https://www.google.com/s2/favicons?domain=https://www.google.com
 
 
 ## 🚀 Quick Start
 
-Experience GraphGen through [Web](https://g-app-center-120612-6433-jpdvmvp.openxlab.space) or [Backup Web Entrance](https://openxlab.org.cn/apps/detail/chenzihonga/GraphGen)
+Experience GraphGen Demo through [Huggingface](https://huggingface.co/spaces/chenzihong/GraphGen) or [Modelscope](https://modelscope.cn/studios/chenzihong/GraphGen).
 
 For any questions, please check [FAQ](https://github.com/open-sciencelab/GraphGen/issues/10), open new [issue](https://github.com/open-sciencelab/GraphGen/issues) or join our [wechat group](https://cdn.vansin.top/internlm/dou.jpg) and ask.
 
@@ -263,4 +277,4 @@ This project is licensed under the [Apache License 2.0](LICENSE).
 
 ## 📅 Star History
 
-[![Star History Chart](https://api.star-history.com/svg?repos=open-sciencelab/GraphGen&type=Date)](https://www.star-history.com/#open-sciencelab/GraphGen&Date)
+[![Star History Chart](https://api.star-history.com/svg?repos=Intern-Science/GraphGen&type=Date)](https://www.star-history.com/#open-sciencelab/GraphGen&Date)

README_zh.md

@@ -16,7 +16,6 @@
 
 [![Hugging Face](https://img.shields.io/badge/Demo-on%20HF-blue?logo=huggingface&logoColor=yellow)](https://huggingface.co/spaces/chenzihong/GraphGen)
 [![Model Scope](https://img.shields.io/badge/%F0%9F%A4%96%20Demo-on%20MS-green)](https://modelscope.cn/studios/chenzihong/GraphGen)
-[![OpenXLab](https://img.shields.io/badge/Demo-on%20OpenXLab-blue?logo=openxlab&logoColor=yellow)](https://g-app-center-120612-6433-jpdvmvp.openxlab.space)
 
 GraphGen: Enhancing Supervised Fine-Tuning for LLMs with Knowledge-Driven Synthetic Data Generation
 
@@ -63,13 +62,14 @@ GraphGen 首先根据源文本构建细粒度的知识图谱，然后利用期
 在数据生成后，您可以使用[LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory) 和 [xtuner](https://github.com/InternLM/xtuner)对大语言模型进行微调。
 
 ## 📌 最新更新
-- **2025.10.30** 我们支持多种新的 LLM 客户端和推理后端，包括 [Ollama_client]([Ollama_client](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/api/ollama_client.py), [http_client](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/api/http_client.py), [HuggingFace Transformers](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/local/hf_wrapper.py) 和 [SGLang](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/local/sglang_wrapper.py).
+- **2025.12.1**：新增对 [NCBI](https://www.ncbi.nlm.nih.gov/) 和 [RNAcentral](https://rnacentral.org/) 数据库的检索支持，现在可以从这些生物信息学数据库中提取DNA和RNA数据。
+- **2025.10.30**：我们支持多种新的 LLM 客户端和推理后端，包括 [Ollama_client]([Ollama_client](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/api/ollama_client.py), [http_client](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/api/http_client.py), [HuggingFace Transformers](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/local/hf_wrapper.py) 和 [SGLang](https://github.com/open-sciencelab/GraphGen/blob/main/graphgen/models/llm/local/sglang_wrapper.py)。
 - **2025.10.23**：我们现在支持视觉问答（VQA）数据生成。运行脚本：`bash scripts/generate/generate_vqa.sh`。
-- **2025.10.21**：我们现在通过 [MinerU](https://github.com/opendatalab/MinerU) 支持 PDF 作为数据生成的输入格式。
 
 <details>
 <summary>历史更新</summary>
 
+- **2025.10.21**：我们现在通过 [MinerU](https://github.com/opendatalab/MinerU) 支持 PDF 作为数据生成的输入格式。
 - **2025.09.29**：我们在 [Hugging Face](https://huggingface.co/spaces/chenzihong/GraphGen) 和 [ModelScope](https://modelscope.cn/studios/chenzihong/GraphGen) 上自动更新 Gradio 应用。
 - **2025.08.14**：支持利用 Leiden 社区发现算法对知识图谱进行社区划分，合成 CoT 数据。
 - **2025.07.31**：新增 Google、Bing、Wikipedia 和 UniProt 作为搜索后端，帮助填补数据缺口。  
@@ -82,9 +82,9 @@ GraphGen 首先根据源文本构建细粒度的知识图谱，然后利用期
 我们支持多种 LLM 推理服务器、API 服务器、推理客户端、输入文件格式、数据模态、输出数据格式和输出数据类型。
 可以根据合成数据的需求进行灵活配置。
 
-| 推理服务器                                        | API 服务器                                                                        | 推理客户端                                                      | 输入文件格式                             | 数据模态         | 输出数据格式                       | 输出数据类型                                          |
-|----------------------------------------------|--------------------------------------------------------------------------------|------------------------------------------------------------|------------------------------------|--------------|------------------------------|-------------------------------------------------|
-| [![hf-icon]HF][hf]<br>[![sg-icon]SGLang][sg] | [![sif-icon]Silicon][sif]<br>[![oai-icon]OpenAI][oai]<br>[![az-icon]Azure][az] | HTTP<br>[![ol-icon]Ollama][ol]<br>[![oai-icon]OpenAI][oai] | CSV<br>JSON<br>JSONL<br>PDF<br>TXT | TEXT<br>TEXT | Alpaca<br>ChatML<br>Sharegpt | Aggregated<br>Atomic<br>CoT<br>Multi-hop<br>VQA |
+| 推理服务器                                        | API 服务器                                                                        | 推理客户端                                                      | 输入文件格式                                                                                                                                                                                                                                                   | 数据模态          | 输出数据类型                                          |
+|----------------------------------------------|--------------------------------------------------------------------------------|------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|-------------------------------------------------|
+| [![hf-icon]HF][hf]<br>[![sg-icon]SGLang][sg] | [![sif-icon]Silicon][sif]<br>[![oai-icon]OpenAI][oai]<br>[![az-icon]Azure][az] | HTTP<br>[![ol-icon]Ollama][ol]<br>[![oai-icon]OpenAI][oai] | 文件(CSV, JSON, JSONL, PDF, TXT等)<br>数据库([![uniprot-icon]UniProt][uniprot], [![ncbi-icon]NCBI][ncbi], [![rnacentral-icon]RNAcentral][rnacentral])<br>搜索引擎([![bing-icon]Bing][bing], [![google-icon]Google][google])<br>知识图谱([![wiki-icon]Wikipedia][wiki]) | TEXT<br>IMAGE | Aggregated<br>Atomic<br>CoT<br>Multi-hop<br>VQA |
 
 <!-- links -->
 [hf]: https://huggingface.co/docs/transformers/index
@@ -93,6 +93,13 @@ GraphGen 首先根据源文本构建细粒度的知识图谱，然后利用期
 [oai]: https://openai.com
 [az]: https://azure.microsoft.com/en-us/services/cognitive-services/openai-service/
 [ol]: https://ollama.com
+[uniprot]: https://www.uniprot.org/
+[ncbi]: https://www.ncbi.nlm.nih.gov/
+[rnacentral]: https://rnacentral.org/
+[wiki]: https://www.wikipedia.org/
+[bing]: https://www.bing.com/
+[google]: https://www.google.com
+
 
 <!-- icons -->
 [hf-icon]: https://www.google.com/s2/favicons?domain=https://huggingface.co
@@ -102,10 +109,17 @@ GraphGen 首先根据源文本构建细粒度的知识图谱，然后利用期
 [az-icon]: https://www.google.com/s2/favicons?domain=https://azure.microsoft.com
 [ol-icon]: https://www.google.com/s2/favicons?domain=https://ollama.com
 
+[uniprot-icon]: https://www.google.com/s2/favicons?domain=https://www.uniprot.org
+[ncbi-icon]: https://www.google.com/s2/favicons?domain=https://www.ncbi.nlm.nih.gov/
+[rnacentral-icon]: https://www.google.com/s2/favicons?domain=https://rnacentral.org/
+[wiki-icon]: https://www.google.com/s2/favicons?domain=https://www.wikipedia.org/
+[bing-icon]: https://www.google.com/s2/favicons?domain=https://www.bing.com/
+[google-icon]: https://www.google.com/s2/favicons?domain=https://www.google.com
+
 
 ## 🚀 快速开始
 
-通过 [Web](https://g-app-center-120612-6433-jpdvmvp.openxlab.space) 或 [备用 Web 入口](https://openxlab.org.cn/apps/detail/chenzihonga/GraphGen) 体验 GraphGen。
+通过 [Huggingface](https://huggingface.co/spaces/chenzihong/GraphGen) 或 [Modelscope](https://modelscope.cn/studios/chenzihong/GraphGen) 体验 GraphGen。
 
 如有任何问题，请查看 [FAQ](https://github.com/open-sciencelab/GraphGen/issues/10)、提交新的 [issue](https://github.com/open-sciencelab/GraphGen/issues) 或加入我们的[微信群](https://cdn.vansin.top/internlm/dou.jpg)咨询。
 
@@ -259,5 +273,5 @@ GraphGen 首先根据源文本构建细粒度的知识图谱，然后利用期
 
 ## 📅 星标历史
 
-[![Star History Chart](https://api.star-history.com/svg?repos=open-sciencelab/GraphGen&type=Date)](https://www.star-history.com/#open-sciencelab/GraphGen&Date)
+[![Star History Chart](https://api.star-history.com/svg?repos=Intern-Science/GraphGen&type=Date)](https://www.star-history.com/#open-sciencelab/GraphGen&Date)
 

baselines/Genie/genie.py

@@ -120,7 +120,7 @@ async def process_chunk(content: str):
     load_dotenv()
 
     llm_client = OpenAIClient(
-        model_name=os.getenv("SYNTHESIZER_MODEL"),
+        model=os.getenv("SYNTHESIZER_MODEL"),
         api_key=os.getenv("SYNTHESIZER_API_KEY"),
         base_url=os.getenv("SYNTHESIZER_BASE_URL"),
     )

baselines/LongForm/longform.py

@@ -86,7 +86,7 @@ async def process_chunk(content: str):
     load_dotenv()
 
     llm_client = OpenAIClient(
-        model_name=os.getenv("SYNTHESIZER_MODEL"),
+        model=os.getenv("SYNTHESIZER_MODEL"),
         api_key=os.getenv("SYNTHESIZER_API_KEY"),
         base_url=os.getenv("SYNTHESIZER_BASE_URL"),
     )

baselines/SELF-QA/self-qa.py

@@ -154,7 +154,7 @@ async def process_chunk(content: str):
     load_dotenv()
 
     llm_client = OpenAIClient(
-        model_name=os.getenv("SYNTHESIZER_MODEL"),
+        model=os.getenv("SYNTHESIZER_MODEL"),
         api_key=os.getenv("SYNTHESIZER_API_KEY"),
         base_url=os.getenv("SYNTHESIZER_BASE_URL"),
     )

baselines/Wrap/wrap.py

@@ -107,7 +107,7 @@ async def process_chunk(content: str):
     load_dotenv()
 
     llm_client = OpenAIClient(
-        model_name=os.getenv("SYNTHESIZER_MODEL"),
+        model=os.getenv("SYNTHESIZER_MODEL"),
         api_key=os.getenv("SYNTHESIZER_API_KEY"),
         base_url=os.getenv("SYNTHESIZER_BASE_URL"),
     )

graphgen/bases/__init__.py

@@ -1,8 +1,10 @@
+from .base_extractor import BaseExtractor
 from .base_generator import BaseGenerator
 from .base_kg_builder import BaseKGBuilder
 from .base_llm_wrapper import BaseLLMWrapper
 from .base_partitioner import BasePartitioner
 from .base_reader import BaseReader
+from .base_searcher import BaseSearcher
 from .base_splitter import BaseSplitter
 from .base_storage import (
     BaseGraphStorage,