@@ -96,60 +96,59 @@ from azure.ai.evaluation import DocumentRetrievalEvaluator
96
96
retrieval_ground_truth = [
97
97
{
98
98
" document_id" : " 1" ,
99
- " query_relevance_judgement " : 4
99
+ " query_relevance_label " : 4
100
100
},
101
101
{
102
102
" document_id" : " 2" ,
103
- " query_relevance_judgement " : 2
103
+ " query_relevance_label " : 2
104
104
},
105
105
{
106
106
" document_id" : " 3" ,
107
- " query_relevance_judgement " : 3
107
+ " query_relevance_label " : 3
108
108
},
109
109
{
110
110
" document_id" : " 4" ,
111
- " query_relevance_judgement " : 1
111
+ " query_relevance_label " : 1
112
112
},
113
113
{
114
114
" document_id" : " 5" ,
115
- " query_relevance_judgement " : 0
115
+ " query_relevance_label " : 0
116
116
},
117
117
]
118
118
119
+ # these reterieval scores
119
120
retrieved_documents = [
120
121
{
121
122
" document_id" : " 2" ,
122
- " query_relevance_judgement " : 45.1
123
+ " relevance_score " : 45.1
123
124
},
124
125
{
125
126
" document_id" : " 6" ,
126
- " query_relevance_judgement " : 35.8
127
+ " relevance_score " : 35.8
127
128
},
128
129
{
129
130
" document_id" : " 3" ,
130
- " query_relevance_judgement " : 29.2
131
+ " relevance_score " : 29.2
131
132
},
132
133
{
133
134
" document_id" : " 5" ,
134
- " query_relevance_judgement " : 25.4
135
+ " relevance_score " : 25.4
135
136
},
136
137
{
137
138
" document_id" : " 7" ,
138
- " query_relevance_judgement " : 18.8
139
+ " relevance_score " : 18.8
139
140
},
140
141
]
141
142
142
- default_threshold = {
143
- " ndcg@3" : 0.5 ,
144
- " xdcg@3" : 0.5 ,
145
- " fidelity" : 0.5 ,
146
- " top1_relevance" : 50 ,
147
- " top3_max_relevance" : 50 ,
148
- " total_retrieved_documents" : 50 ,
149
- " total_ground_truth_documents" : 50 ,
150
- }
151
-
152
- document_retrieval_evaluator = DocumentRetrievalEvaluator(threshold = default_threshold)
143
+ document_retrieval_evaluator = DocumentRetrievalEvaluator(
144
+ ndcg_threshold = 0.5 ,
145
+ xdcg_threshold = 50.0 ,
146
+ fidelity_threshold = 0.5 ,
147
+ top1_relevance_threshold = 50.0 ,
148
+ top3_max_relevance_threshold = 50.0 ,
149
+ total_retrieved_documents_threshold = 50 ,
150
+ total_ground_truth_documents_threshold = 50
151
+ )
153
152
document_retrieval_evaluator(retrieval_ground_truth = retrieval_ground_truth, retrieved_documents = retrieved_documents)
154
153
```
155
154
0 commit comments