Fix pre-commit errors

Author: fraboniface

.pre-commit-config.yaml

@@ -10,4 +10,4 @@ repos:
     hooks:
       - id: check-merge-conflict
       - id: mixed-line-ending
-exclude: 'rag_system/kotaemon/libs/kotaemon/.*|rag_system/kotaemon/libs/ktem/.*|.*\.ipynb$'
+exclude: 'rag_system/old/.*|library/old/.*|policy_analysis/old/.*|.*\.ipynb$'

library/scraping/download_all_pdfs.py

@@ -15,7 +15,7 @@
 
 from library.database.download_queue_crud import get_papers_to_scrape, mark_paper_failed, mark_paper_scraped
 from library.database.models import ScrapingQueue
-from library.scraping.download_pdf import download_pdf, start_webdriver
+from library.scraping.download_pdf import download_pdf
 from library.scraping.extract_pdf_content import get_markdown_pymupdf
 from selenium import webdriver
 from tqdm import tqdm

library/scraping/extract_sections_from_raw_text.py

@@ -34,7 +34,7 @@ def process_file(
         with ProcessPoolExecutor(max_workers=num_workers) as executor:
             futures = {
                 executor.submit(process_text, text): (idx, oa_id)
-                for idx, (oa_id, text) in enumerate(zip(df.index, df["text"]))
+                for idx, (oa_id, text) in enumerate(zip(df.index, df["text"], strict=True))
             }
 
             buffer = []

library/src/library/scraping/clean/headers_footers.py

@@ -1,9 +1,5 @@
-import os
 import re
-import string
 
-import numpy as np
-import pandas as pd
 
 
 class TrieNode:

library/src/library/scraping/download_pdf.py

@@ -12,7 +12,6 @@
 import tempfile
 
 from selenium import webdriver
-from selenium.common.exceptions import WebDriverException
 from selenium.webdriver.chrome.service import Service
 from webdriver_manager.chrome import ChromeDriverManager
 

policy_analysis/dspy_policies_and_taxonomy_extraction/initial_chunk_cleaning/clean_chunks.py

@@ -1,6 +1,4 @@
 import pyarrow.parquet as pq
-import pyarrow as pa
-import pandas as pd
 import re
 import json
 
@@ -127,7 +125,7 @@ def apply_regex_cleaning(text, regex_rules):
     if TEST_RUN:
         df = df.head(SAMPLE_SIZE)
 
-    for idx, text in enumerate(df[target_col]):
+    for text in df[target_col]:
         cleaned_text, deleted_count, triggered_rules = apply_regex_cleaning(text, REGEX_RULES)
 
         result = {

policy_analysis/dspy_policies_and_taxonomy_extraction/pipeline_policy_and_taxonomy_extraction.py

@@ -2,7 +2,6 @@
 import os
 import pickle
 import pyarrow.parquet as pq
-import pandas as pd
 from dotenv import load_dotenv
 from tqdm import tqdm
 import json
@@ -108,7 +107,7 @@ def suppress_output():
                 # ----------------------------------------------------
 
                 for (text, (openalex_id, chunk_idx), p_out, g_out) in zip(
-                    batch_texts, batch_meta, policy_outputs, geo_outputs
+                    batch_texts, batch_meta, policy_outputs, geo_outputs, strict=False
                 ):
                     policy_dict = p_out.toDict() if hasattr(p_out, "toDict") else p_out
                     geo_dict = g_out.toDict() if hasattr(g_out, "toDict") else g_out

policy_analysis/dspy_policies_and_taxonomy_extraction/policy_extraction/policy_dspy_model_creation.py

@@ -4,14 +4,11 @@
 import json
 import os
 from dotenv import load_dotenv
-import pandas as pd
 from sentence_transformers import CrossEncoder
 import torch
 from datetime import datetime
-from dspy.adapters import JSONAdapter
 
 
-import math 
 # Load environment variables
 load_dotenv()
 

policy_analysis/dspy_policies_and_taxonomy_extraction/taxonomy_extraction/taxonomy_dspy_model_creation.py

@@ -1,14 +1,10 @@
 import dspy
-import csv
-from enum import Enum
-from typing import List, Dict, Any, Union
 import os
 
 from dspy.teleprompt import MIPROv2
 
 import json
 from dotenv import load_dotenv
-import pandas as pd
 from datetime import datetime
 
 from taxonomy_definition.geographical_taxonomy import (

policy_analysis/dspy_policies_and_taxonomy_extraction/taxonomy_extraction/utils.py

@@ -1,3 +1,4 @@
+import ast
 import re
 NULL_STRINGS = {
     "null",