evaluator.py

import logging
from typing import TYPE_CHECKING

from lightman_ai.ai.utils import get_agent_class_from_agent_name

from eval.classifier import Classifier
from eval.templates import ResultsFileBuilder
from eval.utils import ResultsMetrics

if TYPE_CHECKING:
    from lightman_ai.article.models import Article

logger = logging.getLogger("eval")


def eval(
    *,
    prompt: str,
    score_threshold: int,
    relevant_articles: set[Article],
    non_relevant_articles: set[Article],
    samples: int,
    tag: str | None,
    agent: str,
    model: str | None,
    parallel_workers: int = 1,
) -> None:
    agent_class = get_agent_class_from_agent_name(agent)
    classified_articles = Classifier(
        agent=agent_class(prompt, model=model),
        score=score_threshold,
        relevant_articles=relevant_articles,
        non_relevant_articles=non_relevant_articles,
        samples=samples,
        workers=parallel_workers,
    ).run()

    results_metrics = ResultsMetrics(raw_results=classified_articles)
    results_template = ResultsFileBuilder(
        results_metrics=results_metrics,
        tag=tag,
        agent=agent,
        samples=samples,
        prompt=prompt,
        score=score_threshold,
        logger=logger,
        model=model or agent_class._default_model_name,
    )

    results_template.save()