Refactor scoring calculations to use logarithmic scaling

- Updated the combined score calculation in the topic API and topic extraction logic to use logarithmic scaling for occurrence count.
- Adjusted SQL queries in topic extraction and diverse topics functions to reflect the new scoring method, enhancing the relevance of trending and diverse topics.

Author: werther41

app/api/topics/route.ts

@@ -91,7 +91,7 @@ export async function GET(request: NextRequest) {
         combinedScore:
           "combinedScore" in topic
             ? topic.combinedScore
-            : topic.occurrence_count * topic.avg_tfidf_score,
+            : Math.log(topic.occurrence_count + 1) * topic.avg_tfidf_score,
       })),
       metadata: {
         timeWindow,

lib/topic-extraction.ts

@@ -319,7 +319,7 @@ export async function getTrendingTopics(options?: {
     }
 
     query += `
-      ORDER BY (occurrence_count * avg_tfidf_score) DESC
+      ORDER BY (LOG(occurrence_count + 1) * avg_tfidf_score) DESC
       LIMIT ?
     `
     params.push(limit)

lib/topic-search.ts

@@ -270,7 +270,7 @@ export async function getDiverseTopics(options?: {
       params.push(limit * 3) // Get more for randomization
     } else {
       query += `
-        ORDER BY (occurrence_count * avg_tfidf_score) DESC
+        ORDER BY (LOG(occurrence_count + 1) * avg_tfidf_score) DESC
         LIMIT ?
       `
       params.push(limit * 2) // Get more to filter for diversity
@@ -287,7 +287,7 @@ export async function getDiverseTopics(options?: {
         occurrenceCount,
         avgTfidfScore,
         lastSeenAt: row.last_seen_at as string,
-        combinedScore: occurrenceCount * avgTfidfScore,
+        combinedScore: Math.log(occurrenceCount + 1) * avgTfidfScore,
       }
     })