Skip to content

Commit 8614615

Browse files
xeon27xeon27
committed
Comment out WIP code
1 parent ff60de4 commit 8614615

File tree

1 file changed

+75
-72
lines changed

1 file changed

+75
-72
lines changed

‎src/run.py‎

Lines changed: 75 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@
22
from omegaconf import DictConfig
33

44
from generate_capabilities import filter_capabilities, generate_capabilities
5-
from generate_tasks import generate_tasks_using_llm
6-
from lbo import generate_new_capability, get_lbo_train_set
5+
6+
# from generate_tasks import generate_tasks_using_llm
7+
# from lbo import generate_new_capability
78
from model import Model
8-
from utils.prompts import TASK_GENERATION_SYSTEM_PROMPT, TASK_GENERATION_USER_PROMPT
9+
10+
11+
# from utils.lbo_utils import get_lbo_train_set
912

1013

1114
def check_cfg(cfg: DictConfig) -> None:
@@ -63,75 +66,75 @@ def main(cfg: DictConfig) -> None:
6366
capabilities = filter_capabilities(capabilities)
6467
print(capabilities)
6568

66-
# Stage 2. Generate tasks and evaluate subject model on initial capabilities
67-
num_lbo_runs = cfg.lbo_cfg.num_lbo_runs
68-
if cfg.lbo_cfg.pipeline_id == "nearest_neighbour":
69-
# For pipeline 1 (pipeline_id=="nearest_neighbour"), the set of
70-
# generated capabilities are split into two sets
71-
train_capabilities, candidate_capabilities = get_lbo_train_set(
72-
input_data=capabilities,
73-
train_frac=cfg.lbo_cfg.train_frac,
74-
min_train_size=cfg.lbo_cfg.min_train_size,
75-
)
76-
if num_lbo_runs > len(candidate_capabilities):
77-
print(
78-
f"Warning: Number of LBO runs ({num_lbo_runs}) exceeds "
79-
+ f"the number of candidate capabilities ({len(candidate_capabilities)}). "
80-
+ f"Setting the number of LBO runs to {len(candidate_capabilities)}."
81-
)
82-
num_lbo_runs = len(candidate_capabilities)
83-
elif cfg.lbo_cfg.pipeline_id == "discover_new":
84-
# For pipeline 2 (pipeline_id=="discover_new"), use all generated capabilities
85-
# for training
86-
train_capabilities = capabilities
87-
candidate_capabilities = None
88-
89-
# Initialize the subject LLM model
90-
subject_llm = Model(cfg.subject_llm.name)
91-
92-
# TODO: Run this asynchronosly
93-
for capability in capabilities:
94-
# Generate tasks for each capability
95-
generate_tasks_using_llm(
96-
capability=capability,
97-
scientist_llm=scientist_llm,
98-
sys_prompt=TASK_GENERATION_SYSTEM_PROMPT,
99-
user_prompt=TASK_GENERATION_USER_PROMPT,
100-
num_tasks=cfg.capabilities_cfg.num_gen_tasks_per_capability,
101-
scientist_llm_gen_cfg=cfg.scientist_llm.gen_cfg,
102-
)
103-
# Evaluate subject LLM on each capability
104-
capability.evaluate([subject_llm])
105-
106-
# Stage 3. Use LBO to generate new capabilities
107-
for lbo_run_id in range(num_lbo_runs):
108-
new_capability = generate_new_capability(
109-
capabilities=train_capabilities,
110-
subject_llm_name=cfg.subject_llm.name,
111-
capabilities_pool=candidate_capabilities,
112-
pipeline_id=cfg.lbo_cfg.pipeline_id,
113-
lbo_run_id=lbo_run_id,
114-
)
115-
# Generate tasks for new capability
116-
generate_tasks_using_llm(
117-
capability=new_capability,
118-
scientist_llm=scientist_llm,
119-
sys_prompt=TASK_GENERATION_SYSTEM_PROMPT,
120-
user_prompt=TASK_GENERATION_USER_PROMPT,
121-
num_tasks=cfg.capabilities_cfg.num_gen_tasks_per_capability,
122-
scientist_llm_gen_cfg=cfg.scientist_llm.gen_cfg,
123-
)
124-
# Evaluate subject LLM on new capability
125-
new_capability.evaluate([subject_llm])
126-
# Add new capability to train capabilities list
127-
train_capabilities.append(new_capability)
128-
# Remove new capability from candidate capabilities
129-
# for pipeline 1
130-
if candidate_capabilities is not None:
131-
candidate_capabilities.remove(new_capability)
132-
133-
new_capabilities = train_capabilities[-num_lbo_runs:]
134-
print(f"New capabilities: {new_capabilities}")
69+
# # Stage 2. Generate tasks and evaluate subject model on initial capabilities
70+
# num_lbo_runs = cfg.lbo_cfg.num_lbo_runs
71+
# if cfg.lbo_cfg.pipeline_id == "nearest_neighbour":
72+
# # For pipeline 1 (pipeline_id=="nearest_neighbour"), the set of
73+
# # generated capabilities are split into two sets
74+
# train_capabilities, candidate_capabilities = get_lbo_train_set(
75+
# input_data=capabilities,
76+
# train_frac=cfg.lbo_cfg.train_frac,
77+
# min_train_size=cfg.lbo_cfg.min_train_size,
78+
# )
79+
# if num_lbo_runs > len(candidate_capabilities):
80+
# print(
81+
# f"Warning: Number of LBO runs ({num_lbo_runs}) exceeds the number of "
82+
# + f"candidate capabilities ({len(candidate_capabilities)}). "
83+
# + f"Setting the number of LBO runs to {len(candidate_capabilities)}."
84+
# )
85+
# num_lbo_runs = len(candidate_capabilities)
86+
# elif cfg.lbo_cfg.pipeline_id == "discover_new":
87+
# # For pipeline 2 (pipeline_id=="discover_new"), use all generated capabilities
88+
# # for training
89+
# train_capabilities = capabilities
90+
# candidate_capabilities = None
91+
92+
# # Initialize the subject LLM model
93+
# subject_llm = Model(cfg.subject_llm.name)
94+
95+
# # TODO: Run this asynchronosly
96+
# for capability in capabilities:
97+
# # Generate tasks for each capability
98+
# generate_tasks_using_llm(
99+
# capability=capability,
100+
# scientist_llm=scientist_llm,
101+
# sys_prompt=TASK_GENERATION_SYSTEM_PROMPT,
102+
# user_prompt=TASK_GENERATION_USER_PROMPT,
103+
# num_tasks=cfg.capabilities_cfg.num_gen_tasks_per_capability,
104+
# scientist_llm_gen_cfg=cfg.scientist_llm.gen_cfg,
105+
# )
106+
# # Evaluate subject LLM on each capability
107+
# capability.evaluate([subject_llm])
108+
109+
# # Stage 3. Use LBO to generate new capabilities
110+
# for lbo_run_id in range(num_lbo_runs):
111+
# new_capability = generate_new_capability(
112+
# capabilities=train_capabilities,
113+
# subject_llm_name=cfg.subject_llm.name,
114+
# capabilities_pool=candidate_capabilities,
115+
# pipeline_id=cfg.lbo_cfg.pipeline_id,
116+
# lbo_run_id=lbo_run_id,
117+
# )
118+
# # Generate tasks for new capability
119+
# generate_tasks_using_llm(
120+
# capability=new_capability,
121+
# scientist_llm=scientist_llm,
122+
# sys_prompt=TASK_GENERATION_SYSTEM_PROMPT,
123+
# user_prompt=TASK_GENERATION_USER_PROMPT,
124+
# num_tasks=cfg.capabilities_cfg.num_gen_tasks_per_capability,
125+
# scientist_llm_gen_cfg=cfg.scientist_llm.gen_cfg,
126+
# )
127+
# # Evaluate subject LLM on new capability
128+
# new_capability.evaluate([subject_llm])
129+
# # Add new capability to train capabilities list
130+
# train_capabilities.append(new_capability)
131+
# # Remove new capability from candidate capabilities
132+
# # for pipeline 1
133+
# if candidate_capabilities is not None:
134+
# candidate_capabilities.remove(new_capability)
135+
136+
# new_capabilities = train_capabilities[-num_lbo_runs:]
137+
# print(f"New capabilities: {new_capabilities}")
135138

136139

137140
if __name__ == "__main__":

0 commit comments

Comments
 (0)