add sensitive

Author: yuwnloyblog

.DS_Store

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS `sensitivewords` (
+  `id` int unsigned NOT NULL AUTO_INCREMENT,
+  `app_key` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
+  `word` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL,
+  `word_type` tinyint(1) NOT NULL DEFAULT '1' COMMENT '12',
+  `created_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3),
+  `updated_time` datetime(3) DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uniq_word` (`app_key`,`word`),
+  KEY `idx_appkey` (`app_key`,`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

commons/dbcommons/sqls/20251215.sql

@@ -28,6 +28,7 @@ var (
 	IMErrorCode_APP_LOGIN_ERR_PASS      IMErrorCode = 17013
 	IMErrorCode_APP_PHONE_EXISTED       IMErrorCode = 17014
 	IMErrorCode_APP_EMAIL_EXIST         IMErrorCode = 17015
+	IMErrorCode_APP_Sensitive           IMErrorCode = 17016
 
 	//friends
 	IMErrorCode_APP_FRIEND_DEFAULT         IMErrorCode = 17100

commons/errs/errorcode.go

@@ -0,0 +1,126 @@
+package sensitive
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"regexp"
+	"time"
+
+	"github.com/juggleim/jugglechat-server/commons/sensitive/trie"
+)
+
+// Filter 敏感词过滤器
+type Filter struct {
+	trie  *trie.Trie
+	noise *regexp.Regexp
+}
+
+// New 返回一个敏感词过滤器
+func NewFilter() *Filter {
+	return &Filter{
+		trie:  trie.NewTrie(),
+		noise: regexp.MustCompile(`[|\s&%$@*！!#^~_—｜'";.。，,?<>《》：:]+`),
+	}
+}
+
+// AddWord 添加敏感词
+func (filter *Filter) AddWord(words ...string) {
+	filter.trie.Add(words...)
+}
+
+// DelWord 删除敏感词
+func (filter *Filter) DelWord(words ...string) {
+	filter.trie.Del(words...)
+}
+
+// Filter 过滤敏感词
+func (filter *Filter) Filter(text string) string {
+	return filter.trie.Filter(text)
+}
+
+// Replace 和谐敏感词
+func (filter *Filter) Replace(text string, repl rune) string {
+	return filter.trie.Replace(text, repl)
+}
+
+// FindIn 检测敏感词
+func (filter *Filter) FindIn(text string) (bool, string) {
+	text = filter.RemoveNoise(text)
+	return filter.trie.FindIn(text)
+}
+
+// FindAll 找到所有匹配词
+func (filter *Filter) FindAll(text string) []string {
+	return filter.trie.FindAll(text)
+}
+
+// Validate 检测字符串是否合法
+func (filter *Filter) Validate(text string) (bool, string) {
+	text = filter.RemoveNoise(text)
+	return filter.trie.Validate(text)
+}
+
+// RemoveNoise 去除空格等噪音，噪音可以使用 UpdateNoisePattern 更新
+func (filter *Filter) RemoveNoise(text string) string {
+	return filter.noise.ReplaceAllString(text, "")
+}
+
+// UpdateNoisePattern 更新去噪模式
+func (filter *Filter) UpdateNoisePattern(pattern string) {
+	filter.noise = regexp.MustCompile(pattern)
+}
+
+// LoadWordDict 加载本地敏感词字典
+func (filter *Filter) LoadWordDict(path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer func(f *os.File) {
+		err := f.Close()
+		if err != nil {
+			fmt.Println(err.Error())
+		}
+	}(f)
+
+	return filter.Load(f)
+}
+
+// LoadNetWordDict 加载网络敏感词字典
+func (filter *Filter) LoadNetWordDict(url string) error {
+	c := http.Client{
+		Timeout: 5 * time.Second,
+	}
+	rsp, err := c.Get(url)
+	if err != nil {
+		return err
+	}
+	defer func(Body io.ReadCloser) {
+		err := Body.Close()
+		if err != nil {
+			fmt.Println(err.Error())
+		}
+	}(rsp.Body)
+
+	return filter.Load(rsp.Body)
+}
+
+// Load common method to add words
+func (filter *Filter) Load(rd io.Reader) error {
+	buf := bufio.NewReader(rd)
+	for {
+		line, _, err := buf.ReadLine()
+		if err != nil {
+			if err != io.EOF {
+				return err
+			}
+			break
+		}
+		filter.trie.Add(string(line))
+	}
+
+	return nil
+}

commons/sensitive/filter.go

@@ -0,0 +1,127 @@
+package sensitive
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/juggleim/jugglechat-server/commons/caches"
+	"github.com/juggleim/jugglechat-server/storages"
+	"github.com/juggleim/jugglechat-server/storages/models"
+)
+
+var (
+	filterCache *caches.LruCache
+	filterLocks *sync.RWMutex
+)
+
+func init() {
+	filterCache = caches.NewLruCacheWithAddReadTimeout("filter_cache", 10000, nil, 8*time.Minute, 10*time.Minute)
+	filterLocks = &sync.RWMutex{}
+
+	filterCache.SetValueCreator(func(key interface{}) interface{} {
+		s := NewSensitiveService()
+		start := time.Now()
+		loadAppWords(s, key.(string))
+		fmt.Println("load app words cost:", time.Since(start))
+		return s
+	})
+}
+
+func GetAppSensitiveFilter(appKey string) *SensitiveService {
+	filterLocks.Lock()
+	defer filterLocks.Unlock()
+
+	v, ok := filterCache.GetByCreator(appKey, nil)
+	if !ok {
+		return nil
+	}
+	return v.(*SensitiveService)
+}
+
+func loadAppWords(service *SensitiveService, appKey string) (err error) {
+	var (
+		startId  int64 = 0
+		pageSize int64 = 1000
+	)
+	storage := storages.NewSensitiveWordStorage()
+	for {
+		list, err := storage.QrySensitiveWords(appKey, pageSize, startId)
+		if err != nil {
+			fmt.Println(err.Error())
+			return err
+		}
+		for _, item := range list {
+			if startId < item.ID {
+				startId = item.ID
+			}
+		}
+		service.AddWord(list...)
+		if len(list) < int(pageSize) {
+			break
+		}
+	}
+	return nil
+}
+
+type SensitiveService struct {
+	replaceFilter *Filter
+	denyFilter    *Filter
+	loadLock      *sync.RWMutex
+}
+
+func NewSensitiveService() *SensitiveService {
+	return &SensitiveService{
+		replaceFilter: NewFilter(),
+		denyFilter:    NewFilter(),
+		loadLock:      &sync.RWMutex{},
+	}
+}
+
+func (s *SensitiveService) ReplaceSensitiveWords(text string) (isDeny bool, replacedText string) {
+	s.loadLock.RLock()
+	defer s.loadLock.RUnlock()
+
+	if s.denyFilter != nil {
+		var ok bool
+		ok, _ = s.denyFilter.FindIn(text)
+		if ok {
+			isDeny = true
+			return
+		}
+	}
+	if s.replaceFilter != nil {
+		replacedText = s.replaceFilter.Replace(text, '*')
+	}
+
+	return
+}
+
+func (s *SensitiveService) AddWord(words ...*models.SensitiveWord) {
+	s.loadLock.Lock()
+	defer s.loadLock.Unlock()
+	if s.replaceFilter == nil {
+		s.replaceFilter = NewFilter()
+	}
+	if s.denyFilter == nil {
+		s.denyFilter = NewFilter()
+	}
+	for _, word := range words {
+		if word.WordType == models.SensitiveWordType_deny_word {
+			s.denyFilter.AddWord(word.Word)
+		} else {
+			s.replaceFilter.AddWord(word.Word)
+		}
+	}
+}
+
+func (s *SensitiveService) DelWord(words ...string) {
+	s.loadLock.Lock()
+	defer s.loadLock.Unlock()
+	if s.denyFilter != nil {
+		s.denyFilter.DelWord(words...)
+	}
+	if s.replaceFilter != nil {
+		s.replaceFilter.DelWord(words...)
+	}
+}