ember.py

import os
from langchain_community.document_loaders import TextLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_huggingface import HuggingFaceEmbeddings
from langchain_community.vectorstores import FAISS

print("Loading text file...")
# Load the text file
loader = TextLoader("trotsky_corpus.txt")
documents = loader.load()
print(f"Loaded {len(documents)} document(s).")

print("Splitting documents into chunks...")
# Split into chunks
text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200
)
chunks = text_splitter.split_documents(documents)
print(f"Split into {len(chunks)} chunks.")

print("Initializing embedding model...")
# Generate embeddings
embeddings = HuggingFaceEmbeddings(
    model_name="BAAI/bge-base-en-v1.5",
    show_progress=True,  # <-- This is the fix for the error
    
    # This optimization will make embedding 41k chunks MUCH faster.
    # Adjust 128 based on your system's VRAM/RAM.
    encode_kwargs={'batch_size': 100} 
)

print("Generating embeddings for all chunks. This will take a while...")
# This is the slow part!
vector_store = FAISS.from_documents(chunks, embeddings)
print("Embedding complete.")

print("Saving vector store to disk...")
# Save the vector store to disk
vector_store.save_local("trotsky_vector_store.faiss")
print("Success! Vector store saved as 'trotsky_vector_store.faiss'.")