Skip to content

Commit e957ed7

Browse files
silkentrancesilkentrance
committed
fix #4479: add fuzzy keyword search to title
1 parent e82f3ca commit e957ed7

File tree

11 files changed

+167
-13
lines changed

11 files changed

+167
-13
lines changed

‎models/fixtures/access.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,3 +171,9 @@
171171
user_id: 40
172172
repo_id: 61
173173
mode: 4
174+
175+
-
176+
id: 30
177+
user_id: 40
178+
repo_id: 62
179+
mode: 4

‎models/fixtures/issue.yml‎

Lines changed: 105 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@
346346
poster_id: 39
347347
original_author_id: 0
348348
name: repo60 pull1
349-
content: content for the 1st issue
349+
content: content for the 1st pr
350350
milestone_id: 0
351351
priority: 0
352352
is_closed: false
@@ -360,10 +360,112 @@
360360
id: 22
361361
repo_id: 61
362362
index: 1
363-
poster_id: 40
363+
poster_id: 39
364364
original_author_id: 0
365365
name: repo61 pull1
366-
content: content for the 1st issue
366+
content: content for the 1st pr
367+
milestone_id: 0
368+
priority: 0
369+
is_closed: false
370+
is_pull: true
371+
num_comments: 0
372+
created_unix: 1707270422
373+
updated_unix: 1707270422
374+
is_locked: false
375+
376+
-
377+
id: 23
378+
repo_id: 62
379+
index: 1
380+
poster_id: 40
381+
original_author_id: 0
382+
name: Ökononmie der UI
383+
content: die ökonomie der UI muß dringend verbessert werden
384+
milestone_id: 0
385+
priority: 0
386+
is_closed: false
387+
is_pull: false
388+
num_comments: 0
389+
created_unix: 1707270422
390+
updated_unix: 1707270422
391+
is_locked: false
392+
393+
-
394+
id: 24
395+
repo_id: 62
396+
index: 2
397+
poster_id: 40
398+
original_author_id: 0
399+
name: 갃갃갃
400+
content: 啊啊啊
401+
milestone_id: 0
402+
priority: 0
403+
is_closed: false
404+
is_pull: false
405+
num_comments: 0
406+
created_unix: 1707270422
407+
updated_unix: 1707270422
408+
is_locked: false
409+
410+
-
411+
id: 25
412+
repo_id: 62
413+
index: 3
414+
poster_id: 40
415+
original_author_id: 0
416+
name: 啊啊啊
417+
content: 갃갃갃
418+
milestone_id: 0
419+
priority: 0
420+
is_closed: false
421+
is_pull: false
422+
num_comments: 0
423+
created_unix: 1707270422
424+
updated_unix: 1707270422
425+
is_locked: false
426+
427+
-
428+
id: 26
429+
repo_id: 62
430+
index: 4
431+
poster_id: 40
432+
original_author_id: 0
433+
name: repo62 pull1 - Ökononmie der UI
434+
content: die ökonomie der UI muß dringend verbessert werden
435+
milestone_id: 0
436+
priority: 0
437+
is_closed: false
438+
is_pull: true
439+
num_comments: 0
440+
created_unix: 1707270422
441+
updated_unix: 1707270422
442+
is_locked: false
443+
444+
-
445+
id: 27
446+
repo_id: 62
447+
index: 5
448+
poster_id: 40
449+
original_author_id: 0
450+
name: repo62 pull2 - 갃갃갃
451+
content: 啊啊啊
452+
milestone_id: 0
453+
priority: 0
454+
is_closed: false
455+
is_pull: true
456+
num_comments: 0
457+
created_unix: 1707270422
458+
updated_unix: 1707270422
459+
is_locked: false
460+
461+
-
462+
id: 28
463+
repo_id: 62
464+
index: 6
465+
poster_id: 40
466+
original_author_id: 0
467+
name: repo62 pull3 - 啊啊啊
468+
content: 갃갃갃
367469
milestone_id: 0
368470
priority: 0
369471
is_closed: false

‎models/fixtures/issue_index.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,7 @@
3333
-
3434
group_id: 51
3535
max_index: 1
36+
37+
-
38+
group_id: 62
39+
max_index: 6

‎models/fixtures/repository.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1768,3 +1768,34 @@
17681768
size: 0
17691769
is_fsck_enabled: true
17701770
close_issues_via_commit_in_any_branch: false
1771+
1772+
-
1773+
id: 62
1774+
owner_id: 41
1775+
owner_name: org41
1776+
lower_name: repo62
1777+
name: repo62
1778+
default_branch: main
1779+
num_watches: 0
1780+
num_stars: 0
1781+
num_forks: 0
1782+
num_issues: 3
1783+
num_closed_issues: 0
1784+
num_pulls: 3
1785+
num_closed_pulls: 0
1786+
num_milestones: 0
1787+
num_closed_milestones: 0
1788+
num_projects: 0
1789+
num_closed_projects: 0
1790+
is_private: false
1791+
is_empty: false
1792+
is_archived: false
1793+
is_mirror: false
1794+
status: 0
1795+
is_fork: false
1796+
fork_id: 0
1797+
is_template: false
1798+
template_id: 0
1799+
size: 0
1800+
is_fsck_enabled: true
1801+
close_issues_via_commit_in_any_branch: false

‎models/fixtures/user.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1510,7 +1510,7 @@
15101510
num_followers: 0
15111511
num_following: 0
15121512
num_stars: 0
1513-
num_repos: 1
1513+
num_repos: 2
15141514
num_teams: 2
15151515
num_members: 3
15161516
visibility: 0

‎models/issues/issue_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ func TestCountIssues(t *testing.T) {
379379
assert.NoError(t, unittest.PrepareTestDatabase())
380380
count, err := issues_model.CountIssues(db.DefaultContext, &issues_model.IssuesOptions{})
381381
assert.NoError(t, err)
382-
assert.EqualValues(t, 22, count)
382+
assert.EqualValues(t, 25, count)
383383
}
384384

385385
func TestIssueLoadAttributes(t *testing.T) {

‎models/repo/repo_list_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,12 +138,12 @@ func getTestCases() []struct {
138138
{
139139
name: "AllPublic/PublicRepositoriesOfUserIncludingCollaborative",
140140
opts: &repo_model.SearchRepoOptions{ListOptions: db.ListOptions{Page: 1, PageSize: 10}, OwnerID: 15, AllPublic: true, Template: optional.Some(false)},
141-
count: 33,
141+
count: 34,
142142
},
143143
{
144144
name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborative",
145145
opts: &repo_model.SearchRepoOptions{ListOptions: db.ListOptions{Page: 1, PageSize: 10}, OwnerID: 15, Private: true, AllPublic: true, AllLimited: true, Template: optional.Some(false)},
146-
count: 38,
146+
count: 39,
147147
},
148148
{
149149
name: "AllPublic/PublicAndPrivateRepositoriesOfUserIncludingCollaborativeByName",
@@ -158,7 +158,7 @@ func getTestCases() []struct {
158158
{
159159
name: "AllPublic/PublicRepositoriesOfOrganization",
160160
opts: &repo_model.SearchRepoOptions{ListOptions: db.ListOptions{Page: 1, PageSize: 10}, OwnerID: 17, AllPublic: true, Collaborate: optional.Some(false), Template: optional.Some(false)},
161-
count: 33,
161+
count: 34,
162162
},
163163
{
164164
name: "AllTemplates",

‎modules/indexer/internal/bleve/query.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,14 @@ func MatchPhraseQuery(matchPhrase, field, analyzer string, fuzziness int) *query
2828
return q
2929
}
3030

31+
// FuzzyQuery generates a fuzzy query for the given phrase, field, and fuzziness
32+
func FuzzyQuery(matchPhrase, field string, fuzziness int) *query.FuzzyQuery {
33+
q := bleve.NewFuzzyQuery(matchPhrase)
34+
q.FieldVal = field
35+
q.Fuzziness = fuzziness
36+
return q
37+
}
38+
3139
// BoolFieldQuery generates a bool field query for the given value and field
3240
func BoolFieldQuery(value bool, field string) *query.BoolFieldQuery {
3341
q := bleve.NewBoolFieldQuery(value)

‎modules/indexer/internal/bleve/util.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,14 @@ func openIndexer(path string, latestVersion int) (bleve.Index, int, error) {
5050

5151
func GuessFuzzinessByKeyword(s string) int {
5252
// according to https://github.com/blevesearch/bleve/issues/1563, the supported max fuzziness is 2
53-
// magic number 4 was chosen to determine the levenshtein distance per each character of a keyword
54-
// BUT, when using CJK (eg: `갃갃갃` `啊啊啊`), it mismatches a lot.
53+
// BUT, when using CJK (eg: `갃갃갃` `啊啊啊`), it mismatches a lot
54+
// which we need to live with, as we need to support not just ASCII
55+
// in case of code points >= 128 we will increase the fuzziness to 2
56+
// the standard is 1
5557
for _, r := range s {
5658
if r >= 128 {
57-
return 0
59+
return 2
5860
}
5961
}
60-
return min(2, len(s)/4)
62+
return 1
6163
}

‎modules/indexer/issues/bleve/bleve.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
162162
}
163163

164164
queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
165-
inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer, fuzziness),
165+
inner_bleve.FuzzyQuery(options.Keyword, "title", fuzziness),
166166
inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer, fuzziness),
167167
inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer, fuzziness),
168168
}...))

0 commit comments

Comments
 (0)