Skip to content

Commit 7614506

Browse files
钱忱钱忱
钱忱
authored and
钱忱
committed
Merge branch 'main' of github.com:OpenBMB/ChatDev
2 parents 7afa42b + f7d3743 commit 7614506

File tree

2 files changed

+166
-1
lines changed

2 files changed

+166
-1
lines changed

‎ecl/ece.py‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
2+
import os
3+
import json
4+
import re
5+
import numpy as np
6+
import argparse
7+
point = 0.95
8+
eliminate_threshold = 0.95
9+
10+
11+
def retrieve_eliminate(Path_directory,UsedMemory_directory,Evolved_directory):
12+
experiences_use = []
13+
content = []
14+
content1 = []
15+
experiences_total = []
16+
usetime_total = []
17+
exp_dict = {}
18+
eliminated_exp = []
19+
20+
directories = [os.path.join(Path_directory, d) for d in os.listdir(Path_directory) if os.path.isdir(os.path.join(Path_directory, d))]
21+
for subdir in directories:
22+
directory = subdir
23+
logdir = [filename for filename in os.listdir(directory) if filename.endswith(".log")]
24+
logdir = os.path.join(directory, logdir[0])
25+
content1 = open(logdir, "r", encoding='UTF-8').read()
26+
27+
pattern1 = re.compile(r'the source code MIDs is (.*?),', re.S)
28+
experiences_sourceMIDs = re.findall(pattern1, content1)
29+
pattern2 = re.compile(r'the target code MIDs is (.*?)\n',re.S)
30+
experiences_targetMIDs = re.findall(pattern2, content1)
31+
pattern3 = re.compile(r'And the (.*?) similarity is',re.S)
32+
experiences_type = re.findall(pattern3,content1)
33+
for i in range(0,len(experiences_sourceMIDs)):
34+
sourceMID = experiences_sourceMIDs[i]
35+
targetMID = experiences_targetMIDs[i]
36+
type = experiences_type[i]
37+
experiences_use.append((sourceMID,targetMID,type))
38+
39+
with open(UsedMemory_directory) as file:
40+
content1 = json.load(file)
41+
new_content = []
42+
for memorypiece in content1:
43+
experiences = memorypiece.get("experiences")
44+
if experiences != None:
45+
experiences_total.extend(experiences)
46+
for experience in experiences:
47+
experience["use_time"] = 0
48+
for experience in experiences_use:
49+
for experience_t in experiences_total:
50+
if experience[0] == experience_t["sourceMID"] and experience[1] == experience_t["targetMID"]:
51+
experience_t["use_time"] += 1
52+
for i,experience_t in enumerate(experiences_total):
53+
usetime_total.append(experience_t["use_time"])
54+
exp_dict[i] = experience_t["use_time"]
55+
file.close()
56+
57+
usetime_sort = sorted(usetime_total)[::-1]
58+
total = np.sum(usetime_sort)
59+
for i in range(len(usetime_sort)):
60+
if np.sum(usetime_sort[:i])/total >= point:
61+
# print("α：",i)
62+
alpha= i
63+
break
64+
index=0
65+
for k in sorted(exp_dict,key=exp_dict.__getitem__,reverse=True):
66+
if index <= alpha:
67+
eliminated_exp.append(experiences_total[k])
68+
index += 1
69+
else:
70+
break
71+
72+
for memorypiece in content1:
73+
experiences = memorypiece.get("experiences")
74+
retrieve_eliminated_experienceList = []
75+
if experiences != None:
76+
for experience in experiences:
77+
if experience in eliminated_exp:
78+
retrieve_eliminated_experienceList.append(experience)
79+
80+
memorypiece["experiences"] = retrieve_eliminated_experienceList
81+
new_content.append(memorypiece)
82+
83+
with open(Evolved_directory, 'w') as file:
84+
json.dump(new_content, file)
85+
86+
87+
# Quality score gain Elimination
88+
def gain_eliminate(NewMemory_directory,Evolved_directory):
89+
content2 = []
90+
with open(NewMemory_directory) as file:
91+
content2 = json.load(file)
92+
new_content2 = []
93+
for memorypiece in content2:
94+
experiences = memorypiece.get("experiences")
95+
gain_eliminated_experienceList = []
96+
97+
if experiences != None:
98+
# print("origin:", len(experiences))
99+
for experience in experiences:
100+
valueGain = experience.get("valueGain")
101+
# print(valueGain)
102+
if valueGain >= eliminate_threshold:
103+
gain_eliminated_experienceList.append(experience)
104+
# print(len(experiences))
105+
memorypiece["experiences"] = gain_eliminated_experienceList
106+
new_content2.append(memorypiece)
107+
else:
108+
new_content2.append(memorypiece)
109+
file.close()
110+
111+
with open(Evolved_directory, 'r') as file:
112+
new_content = json.load(file)
113+
114+
new_content = new_content + new_content2
115+
116+
with open(Evolved_directory, 'w') as file:
117+
json.dump(new_content, file)
118+
119+
120+
121+
def recount_experience(Evolved_directory):
122+
with open(Evolved_directory, 'r') as file:
123+
content = json.load(file)
124+
125+
with open(Evolved_directory, 'w') as file:
126+
i = 0
127+
for memorypiece in content:
128+
memorypiece["total"] = i
129+
i += 1
130+
json.dump(content, file)
131+
132+
def main():
133+
parser = argparse.ArgumentParser(description="Process memory with some directories.")
134+
parser.add_argument("Path_directory", type = str, help="The directory of software")
135+
parser.add_argument("UsedMemory_directory", type=str, help="The directory of MemoryCards")
136+
parser.add_argument("NewMemory_directory", type=str, help="The directory of NewMemoryCards")
137+
parser.add_argument("Evolved_directory", type= str, help="The directory for output")
138+
139+
140+
args = parser.parse_args()
141+
retrieve_eliminate(args.Path_directory,args.UsedMemory_directory,args.Evolved_directory)
142+
gain_eliminate(args.NewMemory_directory,args.Evolved_directory)
143+
recount_experience(args.Evolved_directory)
144+
145+
if __name__ == "__main__":
146+
main()

‎wiki.md‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ then start building a software by ``python3 run.py`` and go to [Visualizer Websi
149149
After this process, the experiences have been extracted from the production of software and added to the agents' experience pool in `ecl/memory/MemoryCards.json`.
150150
\
151151
**For example:**
152-
It you want to memorize only one software, you can use:
152+
If you want to memorize only one software, you can use:
153153
```bash
154154
python3 ecl/ecl.py "<Software Path to file>"
155155
```
@@ -187,6 +187,25 @@ After this process, the experiences have been extracted from the production of s
187187

188188
Detailed descriptions and experiment results about this **Experiential Co-Learning** Module lies in our preprint paper at https://arxiv.org/abs/2312.17025.
189189

190+
## Experiential Co-Evolving Guide
191+
- **Using Co-Evolving**: Use the following command to initiate the evolving of experiences, which uses the `ecl/ece.py` to eliminate `ecl/memory/UsedMemory.json` and `ecl/memory/NewMemory.json`. Then it combines the two parts of experiences to form a new experience pool in `ecl/memory/Evolved_directory.json`.
192+
193+
```bash
194+
python3 ecl/ece.py "<Path_directory>" "<UsedMemory_directory>" "<NewMemory_directory>" "<Evolved_directory>"
195+
```
196+
`<Path_directory>`: The path to the directory of software , generated with the memory `UsedMemory_directory`. \
197+
`<UsedMemory_directory>`: The path to the directory of UsedMemory, which was used to generate the software in `Path_directory`. \
198+
`<NewMemory_directory>`: The path to the directory NewMemory, which acquires from the software in `Path_directory` using `ecl/ecl.py`. \
199+
`<Evolved_directory>`: The path to a directory where you want to store the evolved memory.
200+
\
201+
**For example:**
202+
```bash
203+
python3 ecl/ece.py "WareHouse" "ecl/memory/UsedMemory.json" "ecl/memory/NewMemory.json" "ecl/memory/MemoryCards_Evolved.json"
204+
```
205+
> **Notice:** The software directory and memory directory must correspond. The software in the "<Path_directory>" is generated using "<UsedMemory_directory>", and the "<NewMemory_directory>" is acquired from the software in the "<Path_directory>". That's because when we calculate the frequency distribution of the experience, we need to ensure that the software corresponds to the experiences, which to eliminate certain experiences to obtain a subset with relatively high retrieval probability.
206+
207+
Detailed descriptions and experiment results about this Experiential Co-Evolving Module lies in our preprint paper at https://arxiv.org/abs/2405.04219.
208+
190209
## Customization
191210

192211
- You can customize your company in three kinds of granularity:

0 commit comments

Comments
 (0)