Skip to content

Commit 58e47f2

Browse files
author
Da.Lei
committed
ut for sensitive checker
1 parent f4cce88 commit 58e47f2

File tree

7 files changed

+294
-11
lines changed

7 files changed

+294
-11
lines changed

‎.mockery.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ dir: "_mocks/{{.PackagePath}}"
44
# https://github.com/vektra/mockery/commit/13fc60740a1eab3fc705fcc131a4b9a6eacfd0fe#diff-4bc8f03915f15ecf2cb9965592fcf933bb06986003613b5633a8dea7962c31a9R40
55
resolve-type-alias: false
66
packages:
7+
io:
8+
config:
9+
interfaces:
10+
Reader:
711
opencsg.com/csghub-server/component:
812
config:
913
interfaces:

‎_mocks/io/mock_Reader.go‎

Lines changed: 88 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎moderation/checker/file_checker_test.go‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package checker
2+
3+
import (
4+
"strings"
5+
"testing"
6+
7+
"opencsg.com/csghub-server/common/types"
8+
)
9+
10+
func TestGetFileChecker(t *testing.T) {
11+
// Test case 1: Test with a folder
12+
fileType1 := "folder"
13+
filePath1 := "/path/to/folder"
14+
lfsRelativePath1 := ""
15+
expected1 := &FolderChecker{}
16+
result1 := GetFileChecker(fileType1, filePath1, lfsRelativePath1)
17+
if _, ok := result1.(*FolderChecker); !ok {
18+
t.Errorf("Expected %T, got %T", expected1, result1)
19+
}
20+
21+
// Test case 2: Test with a LFS file
22+
fileType2 := "file"
23+
filePath2 := "/path/to/lfs/file"
24+
lfsRelativePath2 := "lfs/path/to/file"
25+
expected2 := &LfsFileChecker{}
26+
result2 := GetFileChecker(fileType2, filePath2, lfsRelativePath2)
27+
if _, ok := result2.(*LfsFileChecker); !ok {
28+
t.Errorf("Expected %T, got %T", expected2, result2)
29+
}
30+
31+
// Test case 3: Test with an unknown file type
32+
fileType3 := "file"
33+
filePath3 := "/path/to/unknown/file"
34+
lfsRelativePath3 := ""
35+
expected3 := &UnkownFileChecker{}
36+
result3 := GetFileChecker(fileType3, filePath3, lfsRelativePath3)
37+
if _, ok := result3.(*UnkownFileChecker); !ok {
38+
t.Errorf("Expected %T, got %T", expected3, result3)
39+
}
40+
41+
// Test case 4: Test with an image file
42+
fileType4 := "file"
43+
filePath4 := "/path/to/image.png"
44+
lfsRelativePath4 := ""
45+
expected4 := &ImageFileChecker{}
46+
result4 := GetFileChecker(fileType4, filePath4, lfsRelativePath4)
47+
if _, ok := result4.(*ImageFileChecker); !ok {
48+
t.Errorf("Expected %T, got %T", expected4, result4)
49+
}
50+
51+
// Test case 5: Test with a text file
52+
fileType5 := "file"
53+
filePath5 := "/path/to/text.md"
54+
lfsRelativePath5 := ""
55+
expected5 := &TextFileChecker{}
56+
result5 := GetFileChecker(fileType5, filePath5, lfsRelativePath5)
57+
if _, ok := result5.(*TextFileChecker); !ok {
58+
t.Errorf("Expected %T, got %T", expected5, result5)
59+
}
60+
}
61+
62+
func TestImageFileChecker_Run(t *testing.T) {
63+
checker := &ImageFileChecker{}
64+
reader := strings.NewReader("image content")
65+
expectedStatus := types.SensitiveCheckSkip
66+
expectedMessage := "skip image file"
67+
status, message := checker.Run(reader)
68+
if status != expectedStatus || message != expectedMessage {
69+
t.Errorf("Expected (%v, %v), got (%v, %v)", expectedStatus, expectedMessage, status, message)
70+
}
71+
}
72+
73+
func TestLfsFileChecker_Run(t *testing.T) {
74+
checker := &LfsFileChecker{}
75+
reader := strings.NewReader("lfs content")
76+
expectedStatus := types.SensitiveCheckSkip
77+
expectedMessage := "skip lfs file"
78+
status, message := checker.Run(reader)
79+
if status != expectedStatus || message != expectedMessage {
80+
t.Errorf("Expected (%v, %v), got (%v, %v)", expectedStatus, expectedMessage, status, message)
81+
}
82+
}
83+
84+
func TestFolderChecker_Run(t *testing.T) {
85+
checker := &FolderChecker{}
86+
reader := strings.NewReader("folder content")
87+
expectedStatus := types.SensitiveCheckSkip
88+
expectedMessage := "skip folder"
89+
status, message := checker.Run(reader)
90+
if status != expectedStatus || message != expectedMessage {
91+
t.Errorf("Expected (%v, %v), got (%v, %v)", expectedStatus, expectedMessage, status, message)
92+
}
93+
}

‎moderation/checker/local_sensitive_word_checker.go‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,10 @@ func (d *DFA) BuildDFA(words []string) {
3737

3838
// ContainsSensitiveWord checks if the input text contains any sensitive words
3939
func (d *DFA) ContainsSensitiveWord(text string) bool {
40+
runes := []rune(text)
4041
current := d.root
41-
for _, char := range text {
42+
for i := 0; i < len(runes); i++ {
43+
char := runes[i]
4244
if isIgnoredCharacter(char) {
4345
continue
4446
}
@@ -49,6 +51,12 @@ func (d *DFA) ContainsSensitiveWord(text string) bool {
4951
}
5052
} else {
5153
current = d.root // Reset to the root state
54+
if i > 0 {
55+
// Check if the current character is the start of a new sensitive word
56+
if next, exists := current.transitions[char]; exists {
57+
current = next
58+
}
59+
}
5260
}
5361
}
5462
return false

‎moderation/checker/local_sensitive_word_checker_test.go‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,28 +4,21 @@ import (
44
"testing"
55

66
"github.com/stretchr/testify/assert"
7-
"opencsg.com/csghub-server/common/config"
87
)
98

109
func TestContainsSensitiveWord(t *testing.T) {
11-
cfg := &config.Config{}
12-
cfg.SensitiveCheck.Enable = true
13-
cfg.SensitiveCheck.AccessKeyID = "accessKeyID"
14-
cfg.SensitiveCheck.AccessKeySecret = "accessKeySecret"
15-
cfg.SensitiveCheck.Endpoint = "endpoint"
16-
cfg.SensitiveCheck.Region = "region"
17-
Init(cfg)
18-
1910
d := NewDFA()
2011
d.BuildDFA(getSensitiveWordList(`5pWP5oSf6K+NLHNlbnNpdGl2ZXdvcmQ=`))
2112
assert.True(t, d.ContainsSensitiveWord("敏感词"))
2213
assert.True(t, d.ContainsSensitiveWord("敏*感词"))
2314
assert.True(t, d.ContainsSensitiveWord("敏 感词"))
15+
assert.True(t, d.ContainsSensitiveWord("敏感敏 感词"))
2416
assert.True(t, d.ContainsSensitiveWord("敏\u3000感词"))
25-
assert.True(t, d.ContainsSensitiveWord("敏感词123"))
17+
assert.True(t, d.ContainsSensitiveWord("xxxx敏感词123"))
2618
assert.False(t, d.ContainsSensitiveWord("敏感1234词"))
2719

2820
assert.True(t, d.ContainsSensitiveWord("sensitive word"))
21+
assert.True(t, d.ContainsSensitiveWord("xxs sensitive word"))
2922
assert.True(t, d.ContainsSensitiveWord("sensitive word123"))
3023
assert.False(t, d.ContainsSensitiveWord("sensitive ord"))
3124
}

‎moderation/checker/text_file_checker_test.go‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package checker
2+
3+
import (
4+
"bytes"
5+
"errors"
6+
"testing"
7+
8+
"github.com/stretchr/testify/mock"
9+
mockio "opencsg.com/csghub-server/_mocks/io"
10+
mocksens "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/sensitive"
11+
"opencsg.com/csghub-server/builder/sensitive"
12+
"opencsg.com/csghub-server/common/types"
13+
)
14+
15+
func TestTextFileChecker_Run(t *testing.T) {
16+
17+
t.Run("contains sensitive words", func(t *testing.T) {
18+
localWordChecker = NewDFA()
19+
localWordChecker.BuildDFA(getSensitiveWordList(`5pWP5oSf6K+NLHNlbnNpdGl2ZXdvcmQ=`))
20+
checker := NewTextFileChecker()
21+
22+
reader1 := bytes.NewReader([]byte("This text contains sensitive word."))
23+
expectedStatus1 := types.SensitiveCheckFail
24+
expectedMessage1 := "contains sensitive word"
25+
status1, message1 := checker.Run(reader1)
26+
if status1 != expectedStatus1 || message1 != expectedMessage1 {
27+
t.Errorf("Test case 1 failed: Expected (%v, %v), Got (%v, %v)", expectedStatus1, expectedMessage1, status1, message1)
28+
}
29+
})
30+
t.Run("no sensitive words", func(t *testing.T) {
31+
localWordChecker = NewDFA()
32+
localWordChecker.BuildDFA(getSensitiveWordList(`5pWP5oSf6K+NLHNlbnNpdGl2ZXdvcmQ=`))
33+
mockContentChecker := mocksens.NewMockSensitiveChecker(t)
34+
contentChecker = mockContentChecker
35+
mockContentChecker.EXPECT().PassTextCheck(mock.Anything, mock.Anything, mock.Anything).Return(&sensitive.CheckResult{
36+
IsSensitive: false,
37+
Reason: "",
38+
}, nil)
39+
checker := NewTextFileChecker()
40+
41+
reader2 := bytes.NewReader([]byte("This is a regular text file."))
42+
expectedStatus2 := types.SensitiveCheckPass
43+
expectedMessage2 := ""
44+
status2, message2 := checker.Run(reader2)
45+
if status2 != expectedStatus2 || message2 != expectedMessage2 {
46+
t.Errorf("Test case 2 failed: Expected (%v, %v), Got (%v, %v)", expectedStatus2, expectedMessage2, status2, message2)
47+
}
48+
})
49+
50+
t.Run("failed to read file content", func(t *testing.T) {
51+
checker := NewTextFileChecker()
52+
53+
reader := mockio.NewMockReader(t)
54+
reader.EXPECT().Read(mock.Anything).Return(0, errors.New("failed to read file content"))
55+
expectedStatus3 := types.SensitiveCheckException
56+
expectedMessage3 := "failed to read file content"
57+
status3, message3 := checker.Run(reader)
58+
if status3 != expectedStatus3 || message3 != expectedMessage3 {
59+
t.Errorf("Test case 3 failed: Expected (%v, %v), Got (%v, %v)", expectedStatus3, expectedMessage3, status3, message3)
60+
}
61+
})
62+
}

‎moderation/checker/unknown_file_checker_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package checker
2+
3+
import (
4+
"errors"
5+
"testing"
6+
7+
"github.com/stretchr/testify/mock"
8+
"github.com/stretchr/testify/require"
9+
mockio "opencsg.com/csghub-server/_mocks/io"
10+
"opencsg.com/csghub-server/common/types"
11+
)
12+
13+
func TestUnkownFileChecker_Run(t *testing.T) {
14+
t.Run("fail to read file", func(t *testing.T) {
15+
reader := mockio.NewMockReader(t)
16+
reader.EXPECT().Read(mock.Anything).Return(0, errors.New("unknown exception"))
17+
c := &UnkownFileChecker{}
18+
status, msg := c.Run(reader)
19+
require.Equal(t, types.SensitiveCheckException, status)
20+
require.Equal(t, "failed to read file contents", msg)
21+
})
22+
23+
t.Run("skip audio file", func(t *testing.T) {
24+
reader := mockio.NewMockReader(t)
25+
reader.EXPECT().Read(mock.Anything).RunAndReturn(func(b []byte) (int, error) {
26+
header := []byte{0x46, 0x4F, 0x52, 0x4D, 00, 00, 00, 00, 0x41, 0x49, 0x46, 0x46}
27+
copy(b, header)
28+
return len(header), nil
29+
30+
})
31+
c := &UnkownFileChecker{}
32+
status, _ := c.Run(reader)
33+
require.Equal(t, types.SensitiveCheckSkip, status)
34+
})
35+
}

0 commit comments

Comments
 (0)