Skip to content

Commit aa35fa3

Browse files
MartinBraquetMartinBraquet
committed
Pre compute compatibility scores for faster profile lookup
1 parent f97d244 commit aa35fa3

File tree

17 files changed

+390
-258
lines changed

17 files changed

+390
-258
lines changed

‎backend/api/src/compatible-profiles.ts‎

Lines changed: 14 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,62 +1,29 @@
1-
import {groupBy, sortBy} from 'lodash'
2-
import {APIError, type APIHandler} from 'api/helpers/endpoint'
3-
import {getCompatibilityScore, hasAnsweredQuestions} from 'common/profiles/compatibility-score'
4-
import {getCompatibilityAnswers, getGenderCompatibleProfiles, getProfile,} from 'shared/profiles/supabase'
5-
import {log} from 'shared/utils'
1+
import {type APIHandler} from 'api/helpers/endpoint'
2+
import {createSupabaseDirectClient} from "shared/supabase/init";
63

74
export const getCompatibleProfilesHandler: APIHandler<'compatible-profiles'> = async (props) => {
8-
return getCompatibleProfiles(props.userId, false)
5+
return getCompatibleProfiles(props.userId)
96
}
107

118
export const getCompatibleProfiles = async (
129
userId: string,
13-
includeProfilesWithoutPromptAnswers: boolean = true,
1410
) => {
15-
const profile = await getProfile(userId)
16-
17-
log('got profile', {
18-
id: profile?.id,
19-
userId: profile?.user_id,
20-
username: profile?.user?.username,
21-
})
22-
23-
if (!profile) throw new APIError(404, 'Profile not found')
24-
25-
let profiles = await getGenderCompatibleProfiles(profile)
26-
27-
const profileAnswers = await getCompatibilityAnswers([
28-
userId,
29-
...profiles.map((l) => l.user_id),
30-
])
31-
log('got profile answers ' + profileAnswers.length)
32-
33-
const answersByUserId = groupBy(profileAnswers, 'creator_id')
34-
if (!includeProfilesWithoutPromptAnswers) {
35-
profiles = profiles.filter((l) => hasAnsweredQuestions(answersByUserId[l.user_id]))
36-
if (!hasAnsweredQuestions(answersByUserId[profile.user_id])) profiles = []
37-
}
38-
const profileCompatibilityScores = Object.fromEntries(
39-
profiles.map(
40-
(l) =>
41-
[
42-
l.user_id,
43-
getCompatibilityScore(
44-
answersByUserId[profile.user_id] ?? [],
45-
answersByUserId[l.user_id] ?? []
46-
),
47-
] as const
48-
)
11+
const pg = createSupabaseDirectClient()
12+
const scores = await pg.map(
13+
`select *
14+
from compatibility_scores
15+
where score is not null
16+
and (user_id_1 = $1 or user_id_2 = $1)`,
17+
[userId],
18+
(r) => [r.user_id_1 == userId ? r.user_id_2 : r.user_id_1, {score: r.score}] as const
4919
)
5020

51-
const sortedCompatibleProfiles = sortBy(
52-
profiles,
53-
(l) => profileCompatibilityScores[l.user_id].score
54-
).reverse()
21+
const profileCompatibilityScores = Object.fromEntries(scores)
22+
23+
// console.log('scores', profileCompatibilityScores)
5524

5625
return {
5726
status: 'success',
58-
profile,
59-
compatibleProfiles: sortedCompatibleProfiles,
6027
profileCompatibilityScores,
6128
}
6229
}

‎backend/api/src/delete-compatibility-answer.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {APIHandler} from 'api/helpers/endpoint'
22
import {createSupabaseDirectClient} from 'shared/supabase/init'
33
import {APIError} from 'common/api/utils'
4+
import {recomputeCompatibilityScoresForUser} from 'shared/compatibility/compute-scores'
45

56
export const deleteCompatibilityAnswer: APIHandler<'delete-compatibility-answer'> = async (
67
{id}, auth) => {
@@ -27,4 +28,14 @@ export const deleteCompatibilityAnswer: APIHandler<'delete-compatibility-answer'
2728
AND creator_id = $2`,
2829
[id, auth.uid]
2930
)
31+
32+
const continuation = async () => {
33+
// Recompute precomputed compatibility scores for this user
34+
await recomputeCompatibilityScoresForUser(auth.uid, pg)
35+
}
36+
37+
return {
38+
status: 'success',
39+
continue: continuation,
40+
}
3041
}

‎backend/api/src/get-profiles.ts‎

Lines changed: 39 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
import {type APIHandler} from 'api/helpers/endpoint'
22
import {convertRow} from 'shared/profiles/supabase'
3-
import {createSupabaseDirectClient} from 'shared/supabase/init'
3+
import {createSupabaseDirectClient, pgp} from 'shared/supabase/init'
44
import {from, join, leftJoin, limit, orderBy, renderSql, select, where,} from 'shared/supabase/sql-builder'
5-
import {getCompatibleProfiles} from 'api/compatible-profiles'
6-
import {intersection} from 'lodash'
7-
import {MAX_INT, MIN_BIO_LENGTH, MIN_INT} from "common/constants";
5+
import {MIN_BIO_LENGTH} from "common/constants";
6+
import {compact} from "lodash";
87

98
export type profileQueryType = {
109
limit?: number | undefined,
@@ -40,7 +39,7 @@ export type profileQueryType = {
4039
lastModificationWithin?: string | undefined,
4140
}
4241

43-
const userActivityColumns = ['last_online_time']
42+
// const userActivityColumns = ['last_online_time']
4443

4544

4645
export const loadProfiles = async (props: profileQueryType) => {
@@ -84,86 +83,35 @@ export const loadProfiles = async (props: profileQueryType) => {
8483
const keywords = name ? name.split(",").map(q => q.trim()).filter(Boolean) : []
8584
// console.debug('keywords:', keywords)
8685

87-
// TODO: do this in SQL for better performance. Precompute compatibility scores:
88-
// - Have a table compatibility_scores(user_id_1, user_id_2, score) that updates whenever answers from either user change.
89-
// - Query this table directly with "ORDER BY score DESC LIMIT {limit}".
90-
if (orderByParam === 'compatibility_score') {
91-
if (!compatibleWithUserId) {
92-
console.error('Incompatible with user ID')
93-
throw Error('Incompatible with user ID')
94-
}
95-
96-
const {compatibleProfiles} = await getCompatibleProfiles(compatibleWithUserId)
97-
let profiles = compatibleProfiles.filter(
98-
(l) =>
99-
(!name || l.user.name.toLowerCase().includes(name.toLowerCase())) &&
100-
(!genders || genders.includes(l.gender ?? '')) &&
101-
(!education_levels || education_levels.includes(l.education_level ?? '')) &&
102-
(!mbti || mbti.includes(l.mbti ?? '')) &&
103-
(!pref_gender || intersection(pref_gender, l.pref_gender).length) &&
104-
(!pref_age_min || (l.age ?? MAX_INT) >= pref_age_min) &&
105-
(!pref_age_max || (l.age ?? MIN_INT) <= pref_age_max) &&
106-
(!drinks_min || (l.drinks_per_month ?? MAX_INT) >= drinks_min) &&
107-
(!drinks_max || (l.drinks_per_month ?? MIN_INT) <= drinks_max) &&
108-
(!pref_relation_styles ||
109-
intersection(pref_relation_styles, l.pref_relation_styles).length) &&
110-
(!pref_romantic_styles ||
111-
intersection(pref_romantic_styles, l.pref_romantic_styles).length) &&
112-
(!diet ||
113-
intersection(diet, l.diet).length) &&
114-
(!political_beliefs ||
115-
intersection(political_beliefs, l.political_beliefs).length) &&
116-
(!relationship_status ||
117-
intersection(relationship_status, l.relationship_status).length) &&
118-
(!languages ||
119-
intersection(languages, l.languages).length) &&
120-
(!religion ||
121-
intersection(religion, l.religion).length) &&
122-
(!wants_kids_strength ||
123-
wants_kids_strength == -1 ||
124-
!l.wants_kids_strength ||
125-
l.wants_kids_strength == -1 ||
126-
(wants_kids_strength >= 2
127-
? l.wants_kids_strength >= wants_kids_strength
128-
: l.wants_kids_strength <= wants_kids_strength)) &&
129-
(has_kids == undefined ||
130-
has_kids == -1 ||
131-
(has_kids == 0 && !l.has_kids) ||
132-
(l.has_kids && l.has_kids > 0)) &&
133-
(is_smoker === undefined || l.is_smoker === is_smoker) &&
134-
(!l.disabled) &&
135-
(l.id.toString() != skipId) &&
136-
(!geodbCityIds ||
137-
(l.geodb_city_id && geodbCityIds.includes(l.geodb_city_id))) &&
138-
(!filterLocation || (
139-
l.city_latitude && l.city_longitude &&
140-
Math.abs(l.city_latitude - lat) < radius / 69.0 &&
141-
Math.abs(l.city_longitude - lon) < radius / (69.0 * Math.cos(lat * Math.PI / 180)) &&
142-
Math.pow(l.city_latitude - lat, 2) + Math.pow((l.city_longitude - lon) * Math.cos(lat * Math.PI / 180), 2) < Math.pow(radius / 69.0, 2)
143-
)) &&
144-
(shortBio || (l.bio_length ?? 0) >= MIN_BIO_LENGTH)
145-
)
146-
147-
const count = profiles.length
148-
149-
const cursor = after
150-
? profiles.findIndex((l) => l.id.toString() === after) + 1
151-
: 0
152-
console.debug(cursor)
153-
154-
if (limitParam) profiles = profiles.slice(cursor, cursor + limitParam)
155-
156-
return {profiles, count}
86+
if (orderByParam === 'compatibility_score' && !compatibleWithUserId) {
87+
console.error('Incompatible with user ID')
88+
throw Error('Incompatible with user ID')
15789
}
15890

159-
const tablePrefix = userActivityColumns.includes(orderByParam) ? 'user_activity' : 'profiles'
91+
const tablePrefix = orderByParam === 'compatibility_score'
92+
? 'compatibility_scores'
93+
: orderByParam === 'last_online_time'
94+
? 'user_activity'
95+
: 'profiles'
96+
16097
const userActivityJoin = 'user_activity on user_activity.user_id = profiles.user_id'
98+
const compatibilityScoreJoin = pgp.as.format(`compatibility_scores cs on (cs.user_id_1 = LEAST(profiles.user_id, $(compatibleWithUserId)) and cs.user_id_2 = GREATEST(profiles.user_id, $(compatibleWithUserId)))`, {compatibleWithUserId})
99+
100+
const _orderBy = orderByParam === 'compatibility_score' ? 'cs.score' : `${tablePrefix}.${orderByParam}`
101+
const afterFilter = renderSql(
102+
select(_orderBy),
103+
from('profiles'),
104+
orderByParam === 'last_online_time' && leftJoin(userActivityJoin),
105+
orderByParam === 'compatibility_score' && compatibleWithUserId && join(compatibilityScoreJoin),
106+
where('profiles.id = $(after)', {after}),
107+
)
161108

162-
const tableSelection = [
109+
const tableSelection = compact([
163110
from('profiles'),
164111
join('users on users.id = profiles.user_id'),
165-
leftJoin(userActivityJoin),
166-
]
112+
orderByParam === 'last_online_time' && leftJoin(userActivityJoin),
113+
orderByParam === 'compatibility_score' && compatibleWithUserId && join(compatibilityScoreJoin),
114+
])
167115

168116
const filters = [
169117
where('looking_for_matches = true'),
@@ -281,29 +229,27 @@ export const loadProfiles = async (props: profileQueryType) => {
281229
lastModificationWithin && where(`last_modification_time >= NOW() - INTERVAL $(lastModificationWithin)`, {lastModificationWithin}),
282230
]
283231

232+
let selectCols = 'profiles.*, name, username, users.data as user'
233+
if (orderByParam === 'compatibility_score') {
234+
selectCols += ', cs.score as compatibility_score'
235+
} else if (orderByParam === 'last_online_time') {
236+
selectCols += ', user_activity.last_online_time'
237+
}
238+
284239
const query = renderSql(
285-
select('profiles.*, name, username, users.data as user, user_activity.last_online_time'),
240+
select(selectCols),
286241
...tableSelection,
287242
...filters,
288-
289-
orderBy(`${tablePrefix}.${orderByParam} DESC`),
290-
291-
after && where(
292-
`${tablePrefix}.${orderByParam} < (
293-
SELECT ${tablePrefix}.${orderByParam}
294-
FROM profiles
295-
LEFT JOIN ${userActivityJoin}
296-
WHERE profiles.id = $(after)
297-
)`, {after}),
298-
243+
orderBy(`${_orderBy} DESC`),
244+
after && where(`${_orderBy} < (${afterFilter})`),
299245
limitParam && limit(limitParam),
300246
)
301247

302-
// console.debug('query:', query)
248+
console.debug('query:', query)
303249

304250
const profiles = await pg.map(query, [], convertRow)
305251

306-
console.debug('profiles:', profiles)
252+
// console.debug('profiles:', profiles)
307253

308254
const countQuery = renderSql(
309255
select(`count(*) as count`),

‎backend/api/src/set-compatibility-answer.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {APIHandler} from './helpers/endpoint'
22
import {createSupabaseDirectClient} from 'shared/supabase/init'
33
import {Row} from 'common/supabase/utils'
4+
import {recomputeCompatibilityScoresForUser} from 'shared/compatibility/compute-scores'
45

56
export const setCompatibilityAnswer: APIHandler<'set-compatibility-answer'> = async (
67
{questionId, multipleChoice, prefChoices, importance, explanation},
@@ -30,5 +31,13 @@ export const setCompatibilityAnswer: APIHandler<'set-compatibility-answer'> = as
3031
],
3132
})
3233

33-
return result
34+
const continuation = async () => {
35+
// Recompute precomputed compatibility scores for this user
36+
await recomputeCompatibilityScoresForUser(auth.uid, pg)
37+
}
38+
39+
return {
40+
result: result,
41+
continue: continuation,
42+
}
3443
}

‎backend/shared/src/compatibility/compute-scores.ts‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
import {SupabaseDirectClient} from 'shared/supabase/init'
2+
import {Row as RowFor} from 'common/supabase/utils'
3+
import {getCompatibilityScore, hasAnsweredQuestions} from 'common/profiles/compatibility-score'
4+
import {getCompatibilityAnswers, getGenderCompatibleProfiles, getProfile} from "shared/profiles/supabase"
5+
import {groupBy} from "lodash"
6+
import {hrtime} from "node:process"
7+
8+
type AnswerRow = RowFor<'compatibility_answers'>
9+
10+
// Canonicalize pair ordering (user_id_1 < user_id_2 lexicographically)
11+
function canonicalPair(a: string, b: string) {
12+
return a < b ? [a, b] as const : [b, a] as const
13+
}
14+
15+
export async function recomputeCompatibilityScoresForUser(
16+
userId: string,
17+
pg: SupabaseDirectClient,
18+
) {
19+
const startTs = hrtime.bigint()
20+
21+
// Load all answers for the target user
22+
const answersSelf = await pg.manyOrNone<AnswerRow>(
23+
'select * from compatibility_answers where creator_id = $1',
24+
[userId]
25+
)
26+
27+
// If the user has no answered questions, set the score to null
28+
if (!hasAnsweredQuestions(answersSelf)) {
29+
await pg.none(
30+
`update compatibility_scores
31+
set score = null
32+
where user_id_1 = $1
33+
or user_id_2 = $1`,
34+
[userId]
35+
)
36+
return
37+
}
38+
39+
const profile = await getProfile(userId, pg)
40+
if (!profile) throw new Error(`Profile not found for user ${userId}`)
41+
let profiles = await getGenderCompatibleProfiles(profile)
42+
const otherUserIds = profiles.map((l) => l.user_id)
43+
const profileAnswers = await getCompatibilityAnswers([userId, ...otherUserIds])
44+
const answersByUser = groupBy(profileAnswers, 'creator_id')
45+
46+
console.log(`Recomputing compatibility scores for user ${userId}, ${otherUserIds.length} other users.`)
47+
48+
const rows = []
49+
50+
for (const otherId of otherUserIds) {
51+
const answersOther = answersByUser[otherId] ?? []
52+
if (!hasAnsweredQuestions(answersOther)) continue
53+
54+
const {score} = getCompatibilityScore(answersSelf, answersOther)
55+
const adaptedScore = score + (Math.random() - 0.5) * 0.001 // Add some noise to avoid ties (for profile sorting / pagination)
56+
57+
const [u1, u2] = canonicalPair(userId, otherId)
58+
rows.push([u1, u2, adaptedScore])
59+
}
60+
61+
if (rows.length === 0) return
62+
63+
const values = rows
64+
.map((_, i) => `($${i * 3 + 1}, $${i * 3 + 2}, $${i * 3 + 3})`)
65+
.join(", ");
66+
67+
const flatParams = rows.flat();
68+
69+
// Upsert scores for each pair
70+
await pg.none(
71+
`
72+
INSERT INTO compatibility_scores (user_id_1, user_id_2, score)
73+
VALUES
74+
${values}
75+
ON CONFLICT (user_id_1, user_id_2)
76+
DO UPDATE SET score = EXCLUDED.score
77+
`,
78+
flatParams
79+
);
80+
81+
//
82+
// for (const otherId of otherUserIds) {
83+
// const answersOther = answersByUser[otherId] ?? []
84+
// if (!hasAnsweredQuestions(answersOther)) continue
85+
//
86+
// const {score} = getCompatibilityScore(answersSelf, answersOther)
87+
// const [u1, u2] = canonicalPair(userId, otherId)
88+
// await pg.none(
89+
// `insert into compatibility_scores (user_id_1, user_id_2, score)
90+
// values ($1, $2, $3)
91+
// on conflict (user_id_1, user_id_2)
92+
// do update set score = excluded.score`,
93+
// [u1, u2, adaptedScore]
94+
// )
95+
// }
96+
97+
const dt = Number(hrtime.bigint() - startTs) / 1e9
98+
console.log(`Done recomputing compatibility scores for user ${userId} (${dt.toFixed(1)}s).`)
99+
}

0 commit comments

Comments
 (0)