Skip to content

Commit 699cdf7

Browse files
author
黑布林
committed
update stopwords
1 parent d9158e4 commit 699cdf7

File tree

1 file changed

+1
-21
lines changed

1 file changed

+1
-21
lines changed

src/memos/memories/textual/tree_text_memory/retrieve/retrieve_utils.py

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,6 @@ def find_project_root(marker=".git"):
9393
return Path(".")
9494

9595

96-
PROJECT_ROOT = find_project_root()
97-
DEFAULT_STOPWORD_FILE = (
98-
PROJECT_ROOT / "examples" / "data" / "config" / "stopwords.txt"
99-
) # cause time delay
100-
101-
10296
class StopwordManager:
10397
_stopwords = None
10498

@@ -109,13 +103,7 @@ def _load_stopwords(cls):
109103
return cls._stopwords
110104

111105
stopwords = set()
112-
try:
113-
with open(DEFAULT_STOPWORD_FILE, encoding="utf-8") as f:
114-
stopwords = {line.strip() for line in f if line.strip()}
115-
logger.info("Stopwords loaded successfully.")
116-
except Exception as e:
117-
logger.warning(f"Error loading stopwords: {e}, using default stopwords.")
118-
stopwords = cls._load_default_stopwords()
106+
stopwords = cls._load_default_stopwords()
119107

120108
cls._stopwords = stopwords
121109
return stopwords
@@ -370,14 +358,6 @@ def is_stopword(cls, word):
370358
cls._load_stopwords()
371359
return word in cls._stopwords
372360

373-
@classmethod
374-
def reload_stopwords(cls, file_path=None):
375-
cls._stopwords = None
376-
if file_path:
377-
global DEFAULT_STOPWORD_FILE
378-
DEFAULT_STOPWORD_FILE = file_path
379-
cls._load_stopwords()
380-
381361

382362
class FastTokenizer:
383363
def __init__(self, use_jieba=True, use_stopwords=True):

0 commit comments

Comments
 (0)