🛞 Add ResponseRater model which automatically fetches embeddings

Author: ulfaslak

.codespellignore

@@ -0,0 +1 @@
+astroid

README.md

@@ -48,11 +48,11 @@ rater = EmbeddingsRater(reference_data, embeddings_column='embedding_small')
 
 # Convert LLM response embeddings to probability distributions
 llm_responses = np.random.rand(10, 384)
-pdfs = rater.get_response_pdfs('set1', llm_responses)
+pmfs = rater.get_response_pmfs('set1', llm_responses)
 
 # Get overall survey distribution
-survey_pdf = rater.get_survey_response_pdf(pdfs)
-print(f"Survey distribution: {survey_pdf}")
+survey_pmf = rater.get_survey_response_pmf(pmfs)
+print(f"Survey distribution: {survey_pmf}")
 ```
 
 ## Methodology
@@ -66,8 +66,8 @@ The ESR methodology works by:
 ## Core Components
 
 - `EmbeddingsRater`: Main class implementing the ESR methodology
-- `response_embeddings_to_pdf()`: Core function for similarity-to-probability conversion
-- `scale_pdf()`: Temperature scaling function
+- `response_embeddings_to_pmf()`: Core function for similarity-to-probability conversion
+- `scale_pmf()`: Temperature scaling function
 
 ## Citation
 

embeddings_similarity_rating/__init__.py

@@ -10,16 +10,18 @@
 
 from beartype.claw import beartype_this_package
 
-from .compute import response_embeddings_to_pdf, scale_pdf
+from .compute import response_embeddings_to_pmf, scale_pmf
 from .embeddings_rater import EmbeddingsRater
+from .response_rater import ResponseRater
 
 __version__ = "1.0.0"
 __author__ = "Ben F. Maier, Ulf Aslak"
 
 __all__ = [
     "EmbeddingsRater",
-    "response_embeddings_to_pdf",
-    "scale_pdf",
+    "ResponseRater",
+    "response_embeddings_to_pmf",
+    "scale_pmf",
 ]
 
 beartype_this_package()

embeddings_similarity_rating/compute.py

@@ -1,12 +1,12 @@
 """
-Utility functions for computing and manipulating probability density functions (PDFs) and embeddings.
+Utility functions for computing and manipulating probability density functions (PMFs) and embeddings.
 
 This module provides functions for:
 - Converting between different similarity metrics (cosine, KS)
-- Scaling PDFs using temperature parameters
-- Computing statistical moments of PDFs
-- Finding optimal temperature parameters for PDF scaling
-- Converting response embeddings to PDFs
+- Scaling PMFs using temperature parameters
+- Computing statistical moments of PMFs
+- Finding optimal temperature parameters for PMF scaling
+- Converting response embeddings to PMFs
 
 The module is particularly useful for working with Likert scale responses and their
 embeddings, providing tools to analyze and transform the underlying probability
@@ -16,13 +16,13 @@
 import numpy as np
 
 
-def scale_pdf(pdf, temperature, max_temp=np.inf):
+def scale_pmf(pmf, temperature, max_temp=np.inf):
     """
-    Scale a PDF using temperature scaling.
+    Scale a PMF using temperature scaling.
 
     Parameters
     ----------
-    pdf : array_like
+    pmf : array_like
         Input probability density function
     temperature : float
         Temperature parameter for scaling (0 to max_temp)
@@ -32,7 +32,7 @@ def scale_pdf(pdf, temperature, max_temp=np.inf):
     Returns
     -------
     numpy.ndarray
-        Scaled PDF where all values sum to 1
+        Scaled PMF where all values sum to 1
 
     Notes
     -----
@@ -41,38 +41,53 @@ def scale_pdf(pdf, temperature, max_temp=np.inf):
     - Otherwise uses the specified temperature for scaling
     """
     if temperature == 0.0:
-        if np.all(pdf == pdf[0]):
-            return pdf
+        if np.all(pmf == pmf[0]):
+            return pmf
         else:
-            new_pdf = np.zeros_like(pdf)
-            new_pdf[np.argmax(pdf)] = 1.0
-            return new_pdf
+            new_pmf = np.zeros_like(pmf)
+            new_pmf[np.argmax(pmf)] = 1.0
+            return new_pmf
     elif temperature > max_temp:
-        hist = pdf ** (1 / max_temp)
+        hist = pmf ** (1 / max_temp)
     else:
-        hist = pdf ** (1 / temperature)
+        hist = pmf ** (1 / temperature)
     return hist / hist.sum()
 
 
-def response_embeddings_to_pdf(matrix_responses, matrix_likert_sentences):
+def response_embeddings_to_pmf(matrix_responses, matrix_likert_sentences, epsilon=0.0):
     """
-    Convert response embeddings and Likert sentence embeddings to a PDF.
+    Convert response embeddings and Likert sentence embeddings to a PMF.
 
     Parameters
     ----------
     matrix_responses : array_like
         Matrix of response embeddings
     matrix_likert_sentences : array_like
         Matrix of Likert sentence embeddings
+    epsilon : float, optional
+        Small regularization parameter to prevent division by zero and add smoothing.
+        Default is 0.0 (no regularization).
 
     Returns
     -------
     numpy.ndarray
         Probability density function representing the response distribution
+
+    Notes
+    -----
+    This implements the ESR equation:
+    p_{c,i}(r) = [γ(σ_{r,i}, t_c̃) - γ(σ_ℓ,i, t_c̃) + ε δ_ℓ,r] /
+                 [Σ_r γ(σ_{r,i}, t_c̃) - n_points * γ(σ_ℓ,i, t_c̃) + ε]
+    where γ is the cosine similarity function, δ_ℓ,r is the Kronecker delta,
+    and n_points is the number of Likert scale points.
     """
     M_left = matrix_responses
     M_right = matrix_likert_sentences
 
+    # Handle empty input case
+    if M_left.shape[0] == 0:
+        return np.empty((0, M_right.shape[1]))
+
     # Normalize the right matrix (Likert sentences)
     norm_right = np.linalg.norm(M_right, axis=0)
     M_right = M_right / norm_right[None, :]
@@ -81,10 +96,27 @@ def response_embeddings_to_pdf(matrix_responses, matrix_likert_sentences):
     norm_left = np.linalg.norm(M_left, axis=1)
     M_left = M_left / norm_left[:, None]
 
-    # Calculate cosine similarities and convert to PDF
+    # Calculate cosine similarities: γ(σ_{r,i}, t_c̃)
     cos = (1 + M_left.dot(M_right)) / 2
-    cos = cos - cos.min(axis=1)[:, None]
-    sum_per_row = cos.sum(axis=1)
-    pdf = cos / sum_per_row[:, None]
 
-    return pdf
+    # Find minimum similarity per row: γ(σ_ℓ,i, t_c̃)
+    cos_min = cos.min(axis=1)[:, None]
+
+    # Numerator: γ(σ_{r,i}, t_c̃) - γ(σ_ℓ,i, t_c̃) + ε δ_ℓ,r
+    # The ε δ_ℓ,r term adds epsilon only to exactly one minimum similarity position per row
+    numerator = cos - cos_min
+    if epsilon > 0:
+        # Add epsilon to the first position that achieves minimum in each row (Kronecker delta effect)
+        min_indices = np.argmin(cos, axis=1)
+        for i, min_idx in enumerate(min_indices):
+            numerator[i, min_idx] += epsilon
+
+    # Denominator: Σ_r γ(σ_{r,i}, t_c̃) - n_likert_points * γ(σ_ℓ,i, t_c̃) + ε
+    # This is: sum of all similarities - n_likert_points * minimum similarity + epsilon
+    n_likert_points = cos.shape[1]
+    denominator = cos.sum(axis=1)[:, None] - n_likert_points * cos_min + epsilon
+
+    # Calculate final PMF
+    pmf = numerator / denominator
+
+    return pmf

embeddings_similarity_rating/embeddings_rater.py

@@ -4,7 +4,7 @@
 This module provides functionality to:
 - Validate reference sentence data structure
 - Convert LLM response embeddings to probability distributions
-- Calculate survey response PDFs using different reference sets
+- Calculate survey response PMFs using different reference sets
 - Compare responses against mean or specific reference sets
 
 The module is particularly useful for analyzing Likert scale responses from LLMs
@@ -88,10 +88,10 @@ class EmbeddingsRater:
     >>> # Initialize rater
     >>> rater = EmbeddingsRater(df, embeddings_column='embedding_small')
     >>>
-    >>> # Get PDFs for some LLM responses
+    >>> # Get PMFs for some LLM responses
     >>> llm_responses = np.random.rand(5, 384)  # 5 responses, each with 384-dim embedding
-    >>> pdfs = rater.get_response_pdfs('set1', llm_responses)
-    >>> survey_pdf = rater.get_survey_response_pdf(pdfs)
+    >>> pmfs = rater.get_response_pmfs('set1', llm_responses)
+    >>> survey_pmf = rater.get_survey_response_pmf(pmfs)
     """
 
     def __init__(
@@ -124,9 +124,11 @@ def __init__(
             M = np.array(this_set[embeddings_column].to_list()).T
             self.reference_matrices[sentence_set] = M
 
-    def get_response_pdfs(self, reference_set_id, llm_response_matrix, temperature=1.0):
+    def get_response_pmfs(
+        self, reference_set_id, llm_response_matrix, temperature=1.0, epsilon=0.0
+    ):
         """
-        Convert LLM response embeddings to PDFs using specified reference set.
+        Convert LLM response embeddings to PMFs using specified reference set.
 
         Parameters
         ----------
@@ -136,57 +138,60 @@ def get_response_pdfs(self, reference_set_id, llm_response_matrix, temperature=1
             Matrix of LLM response embeddings
             Shape: (n_responses, n_dimensions)
         temperature : float
-            Get scaled pdf With temperature T:
+            Get scaled pmf With temperature T:
             ``p_new[i] ~ p_old[i]^(1/T)``.
+        epsilon : float, optional
+            Small regularization parameter to prevent division by zero and add smoothing.
+            Default is 0.0 (no regularization).
 
         Returns
         -------
         numpy.ndarray
             Probability distributions for each response
         """
         if isinstance(reference_set_id, str) and reference_set_id.lower() == "mean":
-            # Calculate PDFs using mean over all reference sets
-            llm_response_pdfs = np.array(
+            # Calculate PMFs using mean over all reference sets
+            llm_response_pmfs = np.array(
                 [
-                    compute.response_embeddings_to_pdf(llm_response_matrix, M)
+                    compute.response_embeddings_to_pmf(llm_response_matrix, M, epsilon)
                     for M in self.reference_matrices.values()
                 ]
             ).mean(axis=0)
         else:
-            # Calculate PDFs using specific reference set
+            # Calculate PMFs using specific reference set
             M = self.reference_matrices[reference_set_id]
-            llm_response_pdfs = compute.response_embeddings_to_pdf(
-                llm_response_matrix, M
+            llm_response_pmfs = compute.response_embeddings_to_pmf(
+                llm_response_matrix, M, epsilon
             )
 
         if temperature != 1.0:
-            llm_response_pdfs = np.array(
-                [compute.scale_pdf(_pdf, temperature) for _pdf in llm_response_pdfs]
+            llm_response_pmfs = np.array(
+                [compute.scale_pmf(_pmf, temperature) for _pmf in llm_response_pmfs]
             )
 
-        return llm_response_pdfs
+        return llm_response_pmfs
 
-    def get_survey_response_pdf(self, response_pdfs):
+    def get_survey_response_pmf(self, response_pmfs):
         """
-        Calculate the overall survey response PDF by averaging individual response PDFs.
+        Calculate the overall survey response PMF by averaging individual response PMFs.
 
         Parameters
         ----------
-        response_pdfs : numpy.ndarray
-            Matrix of individual response PDFs
+        response_pmfs : numpy.ndarray
+            Matrix of individual response PMFs
 
         Returns
         -------
         numpy.ndarray
-            Average PDF representing the overall survey response
+            Average PMF representing the overall survey response
         """
-        return response_pdfs.mean(axis=0)
+        return response_pmfs.mean(axis=0)
 
-    def get_survey_response_pdf_by_reference_set_id(
-        self, reference_set_id, llm_response_matrix, temperature=1.0
+    def get_survey_response_pmf_by_reference_set_id(
+        self, reference_set_id, llm_response_matrix, temperature=1.0, epsilon=0.0
     ):
         """
-        Get the survey response PDF using a specific reference set.
+        Get the survey response PMF using a specific reference set.
 
         Parameters
         ----------
@@ -196,14 +201,19 @@ def get_survey_response_pdf_by_reference_set_id(
             Matrix of LLM response embeddings
             Shape: (n_responses, n_dimensions)
         temperature : float, default = 1.0
-            Get scaled pdf With temperature T:
+            Get scaled pmf With temperature T:
             ``p_new[i] ~ p_old[i]^(1/T)``.
+        epsilon : float, optional
+            Small regularization parameter to prevent division by zero and add smoothing.
+            Default is 0.0 (no regularization).
 
         Returns
         -------
         numpy.ndarray
-            Average PDF representing the overall survey response
+            Average PMF representing the overall survey response
         """
-        return self.get_survey_response_pdf(
-            self.get_response_pdfs(reference_set_id, llm_response_matrix)
+        return self.get_survey_response_pmf(
+            self.get_response_pmfs(
+                reference_set_id, llm_response_matrix, temperature, epsilon
+            )
         )