|
| 1 | +"""RerankScorer class for re-ranking based on cross-encoder scoring.""" |
| 2 | + |
| 3 | +import json |
| 4 | +from pathlib import Path |
| 5 | +from typing import Any |
| 6 | + |
| 7 | +import numpy as np |
| 8 | +import numpy.typing as npt |
| 9 | +from sentence_transformers import CrossEncoder |
| 10 | +from torch.nn import Sigmoid |
| 11 | +from typing_extensions import Self |
| 12 | + |
| 13 | +from autointent.context import Context |
| 14 | +from autointent.custom_types import WEIGHT_TYPES, LabelType |
| 15 | + |
| 16 | +from .knn import KNNScorer, KNNScorerDumpMetadata |
| 17 | + |
| 18 | + |
| 19 | +class RerankScorerDumpMetadata(KNNScorerDumpMetadata): |
| 20 | + """ |
| 21 | + Metadata for dumping the state of a RerankScorer. |
| 22 | +
|
| 23 | + :ivar cross_encoder_name: Name of the cross-encoder model used. |
| 24 | + :ivar m: Number of top-ranked neighbors to consider, or None to use k. |
| 25 | + :ivar rank_threshold_cutoff: Rank threshold cutoff for re-ranking, or None. |
| 26 | + """ |
| 27 | + |
| 28 | + cross_encoder_name: str |
| 29 | + m: int | None |
| 30 | + rank_threshold_cutoff: int | None |
| 31 | + |
| 32 | + |
| 33 | +class RerankScorer(KNNScorer): |
| 34 | + """ |
| 35 | + Re-ranking scorer using a cross-encoder for intent classification. |
| 36 | +
|
| 37 | + This module uses a cross-encoder to re-rank the nearest neighbors retrieved by a KNN scorer. |
| 38 | +
|
| 39 | + :ivar name: Name of the scorer, defaults to "rerank". |
| 40 | + :ivar _scorer: CrossEncoder instance for re-ranking. |
| 41 | + """ |
| 42 | + |
| 43 | + name = "rerank" |
| 44 | + _scorer: CrossEncoder |
| 45 | + |
| 46 | + def __init__( |
| 47 | + self, |
| 48 | + embedder_name: str, |
| 49 | + k: int, |
| 50 | + weights: WEIGHT_TYPES, |
| 51 | + cross_encoder_name: str, |
| 52 | + m: int | None = None, |
| 53 | + rank_threshold_cutoff: int | None = None, |
| 54 | + db_dir: str | None = None, |
| 55 | + device: str = "cpu", |
| 56 | + batch_size: int = 32, |
| 57 | + max_length: int | None = None, |
| 58 | + ) -> None: |
| 59 | + """ |
| 60 | + Initialize the RerankScorer. |
| 61 | +
|
| 62 | + :param embedder_name: Name of the embedder used for vectorization. |
| 63 | + :param k: Number of closest neighbors to consider during inference. |
| 64 | + :param weights: Weighting strategy: |
| 65 | + - "uniform" (or False): Equal weight for all neighbors. |
| 66 | + - "distance" (or True): Weight inversely proportional to distance. |
| 67 | + - "closest": Only the closest neighbor of each class is weighted. |
| 68 | + :param cross_encoder_name: Name of the cross-encoder model used for re-ranking. |
| 69 | + :param m: Number of top-ranked neighbors to consider, or None to use k. |
| 70 | + :param rank_threshold_cutoff: Rank threshold cutoff for re-ranking, or None. |
| 71 | + :param db_dir: Path to the database directory, or None to use default. |
| 72 | + :param device: Device to run operations on, e.g., "cpu" or "cuda". |
| 73 | + :param batch_size: Batch size for embedding generation, defaults to 32. |
| 74 | + :param max_length: Maximum sequence length for embedding, or None for default. |
| 75 | + """ |
| 76 | + super().__init__( |
| 77 | + embedder_name=embedder_name, |
| 78 | + k=k, |
| 79 | + weights=weights, |
| 80 | + db_dir=db_dir, |
| 81 | + device=device, |
| 82 | + batch_size=batch_size, |
| 83 | + max_length=max_length, |
| 84 | + ) |
| 85 | + |
| 86 | + self.cross_encoder_name = cross_encoder_name |
| 87 | + self.m = k if m is None else m |
| 88 | + self.rank_threshold_cutoff = rank_threshold_cutoff |
| 89 | + |
| 90 | + @classmethod |
| 91 | + def from_context( |
| 92 | + cls, |
| 93 | + context: Context, |
| 94 | + k: int, |
| 95 | + weights: WEIGHT_TYPES, |
| 96 | + cross_encoder_name: str, |
| 97 | + embedder_name: str | None = None, |
| 98 | + m: int | None = None, |
| 99 | + rank_threshold_cutoff: int | None = None, |
| 100 | + ) -> Self: |
| 101 | + """ |
| 102 | + Create a RerankScorer instance from a given context. |
| 103 | +
|
| 104 | + :param context: Context object containing optimization information and vector index client. |
| 105 | + :param k: Number of closest neighbors to consider during inference. |
| 106 | + :param weights: Weighting strategy. |
| 107 | + :param cross_encoder_name: Name of the cross-encoder model used for re-ranking. |
| 108 | + :param embedder_name: Name of the embedder used for vectorization, or None to use the best existing embedder. |
| 109 | + :param m: Number of top-ranked neighbors to consider, or None to use k. |
| 110 | + :param rank_threshold_cutoff: Rank threshold cutoff for re-ranking, or None. |
| 111 | + :return: An instance of RerankScorer. |
| 112 | + """ |
| 113 | + if embedder_name is None: |
| 114 | + embedder_name = context.optimization_info.get_best_embedder() |
| 115 | + prebuilt_index = True |
| 116 | + else: |
| 117 | + prebuilt_index = context.vector_index_client.exists(embedder_name) |
| 118 | + |
| 119 | + instance = cls( |
| 120 | + embedder_name=embedder_name, |
| 121 | + k=k, |
| 122 | + weights=weights, |
| 123 | + cross_encoder_name=cross_encoder_name, |
| 124 | + m=m, |
| 125 | + rank_threshold_cutoff=rank_threshold_cutoff, |
| 126 | + db_dir=str(context.get_db_dir()), |
| 127 | + device=context.get_device(), |
| 128 | + batch_size=context.get_batch_size(), |
| 129 | + max_length=context.get_max_length(), |
| 130 | + ) |
| 131 | + # TODO: needs re-thinking.... |
| 132 | + instance.prebuilt_index = prebuilt_index |
| 133 | + return instance |
| 134 | + |
| 135 | + def fit(self, utterances: list[str], labels: list[LabelType]) -> None: |
| 136 | + """ |
| 137 | + Fit the RerankScorer with utterances and labels. |
| 138 | +
|
| 139 | + :param utterances: List of utterances to fit the scorer. |
| 140 | + :param labels: List of labels corresponding to the utterances. |
| 141 | + """ |
| 142 | + self._scorer = CrossEncoder(self.cross_encoder_name, device=self.device, max_length=self.max_length) # type: ignore[arg-type] |
| 143 | + |
| 144 | + super().fit(utterances, labels) |
| 145 | + |
| 146 | + def _store_state_to_metadata(self) -> RerankScorerDumpMetadata: |
| 147 | + """ |
| 148 | + Store the current state of the RerankScorer to metadata. |
| 149 | +
|
| 150 | + :return: Metadata containing the current state of the RerankScorer. |
| 151 | + """ |
| 152 | + return RerankScorerDumpMetadata( |
| 153 | + **super()._store_state_to_metadata(), |
| 154 | + m=self.m, |
| 155 | + cross_encoder_name=self.cross_encoder_name, |
| 156 | + rank_threshold_cutoff=self.rank_threshold_cutoff, |
| 157 | + ) |
| 158 | + |
| 159 | + def load(self, path: str) -> None: |
| 160 | + """ |
| 161 | + Load the RerankScorer from a given path. |
| 162 | +
|
| 163 | + :param path: Path to the directory containing the dumped metadata. |
| 164 | + """ |
| 165 | + dump_dir = Path(path) |
| 166 | + |
| 167 | + with (dump_dir / self.metadata_dict_name).open() as file: |
| 168 | + self.metadata: RerankScorerDumpMetadata = json.load(file) |
| 169 | + |
| 170 | + self._restore_state_from_metadata(self.metadata) |
| 171 | + |
| 172 | + def _restore_state_from_metadata(self, metadata: RerankScorerDumpMetadata) -> None: |
| 173 | + """ |
| 174 | + Restore the state of the RerankScorer from metadata. |
| 175 | +
|
| 176 | + :param metadata: Metadata containing the state of the RerankScorer. |
| 177 | + """ |
| 178 | + super()._restore_state_from_metadata(metadata) |
| 179 | + |
| 180 | + self.m = metadata["m"] if metadata["m"] else self.k |
| 181 | + self.cross_encoder_name = metadata["cross_encoder_name"] |
| 182 | + self.rank_threshold_cutoff = metadata["rank_threshold_cutoff"] |
| 183 | + self._scorer = CrossEncoder(self.cross_encoder_name, device=self.device, max_length=self.max_length) # type: ignore[arg-type] |
| 184 | + |
| 185 | + def _predict(self, utterances: list[str]) -> tuple[npt.NDArray[Any], list[list[str]]]: |
| 186 | + """ |
| 187 | + Predict the scores and neighbors for given utterances. |
| 188 | +
|
| 189 | + :param utterances: List of utterances to predict scores for. |
| 190 | + :return: A tuple containing the scores and neighbors. |
| 191 | + """ |
| 192 | + knn_labels, knn_distances, knn_neighbors = self._get_neighbours(utterances) |
| 193 | + |
| 194 | + labels: list[list[LabelType]] = [] |
| 195 | + distances: list[list[float]] = [] |
| 196 | + neighbours: list[list[str]] = [] |
| 197 | + |
| 198 | + for query, query_labels, query_distances, query_docs in zip( |
| 199 | + utterances, knn_labels, knn_distances, knn_neighbors, strict=True |
| 200 | + ): |
| 201 | + cur_ranks = self._scorer.rank( |
| 202 | + query, query_docs, top_k=self.m, batch_size=self.batch_size, activation_fct=Sigmoid() |
| 203 | + ) |
| 204 | + |
| 205 | + for dst, src in zip( |
| 206 | + [labels, distances, neighbours], [query_labels, query_distances, query_docs], strict=True |
| 207 | + ): |
| 208 | + dst.append([src[rank["corpus_id"]] for rank in cur_ranks]) # type: ignore[attr-defined, index] |
| 209 | + |
| 210 | + scores = self._count_scores(np.array(labels), np.array(distances)) |
| 211 | + return scores, neighbours |
0 commit comments