fix gsm8k bug & add personal-finance seed config (#34)

* refactor visualization.

* fixed relative path.

* moved functions to utilities, fixed relative paths.

* removed extra print in run_capability_generation.py. updated README.

* exposing functions.

* fixing imports.

* added default value for keys.

* simplify GitHub Actions and fix Codecov issues

* ci: fix workflows - remove retry wrappers and fix poetry install

* ci: fix poetry for GitHub Actions.

* ci: make security audit non-blocking, test: fix pytest warnings and make coverage non-blocking

* Remove redundant docs workflow files

* Personal finance basics config added. Seed generation code updated.

* add constants needed for personal finance.

Author: kohankhaki

src/cfg/capabilities/capability_cfgs/personal_finance_basic.yaml

@@ -0,0 +1,10 @@
+personal_finance_basic:
+  name: personal_finance_basic
+  description: This capability contains synthetic multi-step personal finance problems that cover budgeting, compounding, debt strategies, insurance trade-offs, expected-value analysis, and more, requiring detailed numerical reasoning and using advanced finance concepts.
+  domain: personal_finance
+  data_args:
+    source: kohankhaki/personal_finance_question_answer_complex
+    split: train
+    num_repr_tasks: 3
+  instructions: >-
+    f"""Solve the following personal finance problem step by step. The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the final numerical answer to the problem.\n\nProblem: {problem}\n\nShow every calculation, explain any rounding, and put the final answer on its own line as 'ANSWER: $ANSWER'."""

src/cfg/capabilities/personal_finance.yaml

@@ -0,0 +1,3 @@
+defaults:
+  - capability_cfgs:
+    - personal_finance_basic

src/cfg/run_cfg.yaml

@@ -1,5 +1,5 @@
 scientist_llm:
-  name: o4-mini
+  name: o3-mini
   provider: openai
   generation_cfg:
     capability_generation:
@@ -59,85 +59,51 @@ prompt_cfg:
   sys_msg: Complete the given task to the best of your ability.
 
 capabilities_cfg:
-  capabilities_dir: /fs01/projects/aieng/public/ace/artifacts
-  results_dir: gs://ace-artifacts
-  inspect_evals_dir: /fs01/projects/aieng/public/ace/inspect_evals/src/ace_evals
-  domain: math
-  # Method used to generate capabilities
+  capabilities_dir: /h/fkohankh/automated_capability_evaluation_logs
+  results_dir: /h/fkohankh/automated_capability_evaluation_logs/results
+  inspect_evals_dir: /h/fkohankh/automated_capability_evaluation_logs/inspect_evals
+  domain: personal_finance
   method: "hierarchical"
-  # Number of seed capabilities to use for initial capability generation
-  # Set to -1 to use all seed capabilities
   num_seed_capabilities: 1
-  # Number of initial capabilities to generate using the scientist LLM
-  num_gen_capabilities: 100
-  # Buffer for capability generation
-  num_gen_capabilities_buffer: 0.2
-  # Number of capability areas to generate
-  num_capability_areas: 10
-  # Number of initial capabilities to generate per run
-  num_gen_capabilities_per_run: 5
-  # Number of tasks to generate for each capability
-  num_gen_tasks_per_capability: 100
-  # Buffer for task generation
-  num_gen_tasks_buffer: 0.2
-  # Set this flag to true to use representative tasks
-  # as few shot examples for task generation
-  task_gen_few_shot: true
-  # Set this flag to true to use the specific version
-  # of task generation prompt
+  num_gen_capabilities: 20
+  num_gen_capabilities_buffer: 0.0
+  num_capability_areas: 5
+  num_gen_capabilities_per_run: 1
+  num_gen_tasks_per_capability: 1
+  num_gen_tasks_buffer: 0.0
+  task_gen_few_shot: false
   task_gen_prompt_version: "v1"
-  # Number of tasks to evaluate for each capability
-  # Set to -1 to evaluate all tasks
-  num_eval_tasks_per_capability: -1
-  # Number of retries for each run of capability generation
-  capabilities_gen_retry_attempts: 3
+  num_eval_tasks_per_capability: 2
+  capabilities_gen_retry_attempts: 5
   tasks_gen_retry_attempts: 3
-  # Concurrency for task solving and verification
-  concurrency_task_solver: 50
-  concurrency_task_verifier: 50
-  concurrency_task_eval: 50
-  # Inspect evals config
+  concurrency_task_solver: 2
+  concurrency_task_verifier: 2
+  concurrency_task_eval: 2
   inspect_eval_log_level: "info"
 
 lbo_cfg:
-  # Number of capabilities to generate/select using LBO
-  num_lbo_runs: 20
-  # Type of LBO pipeline to use
-  pipeline_id: "discover_new_lbo_knn" # "no_discovery", "discover_new_llm" or "discover_new_lbo_knn"
-  # Train args for "no_discovery" pipeline
+  num_lbo_runs: 2
+  pipeline_id: "no_discovery"
   train_frac: 0.5
-  num_initial_train: 10
-  # Acquisition function that guides selecting the next query point.
-  # "variance" and "expected_variance_reduction" is supported.
-  acquisition_function: "expected_variance_reduction"
-  # Value of k for "discover_new_lbo_knn" pipeline
-  select_k: 5
-  discover_new_llm_retry_attempts: 3
-  discover_new_lbo_knn_retry_attempts: 5
-  discover_new_retry_attempts: 3
+  num_initial_train: 2
+  acquisition_function: "variance"
 
 embedding_cfg:
-  # The embedding model name used to generate capability embeddings used for filtering.
-  embedding_model: "text-embedding-3-small" # "text-embedding-3-small" or "text-embedding-3-large"
-  embedding_size: 512
-  # The cosine similarity threshold for filtering capabilities based on their embeddings.
-  filtering_similarity_threshold: 0.85
+  embedding_model: "text-embedding-3-small"
+  embedding_size: 256
+  filtering_similarity_threshold: 0.7
 
 dimensionality_reduction_cfg:
-  # dimensionality reduction method generates the low dimensional encodings.
-  reduce_dimensionality_method: "pca" # "t-sne", "cut-embedding" or "pca".
+  reduce_dimensionality_method: "pca"
   reduced_dimensionality_size: 2
-  no_discovery_reduced_dimensionality_method: "t-sne"
+  no_discovery_reduced_dimensionality_method: "pca"
   no_discovery_reduced_dimensionality_size: 2
-  discover_new_reduced_dimensionality_method: "pca"
-  discover_new_reduced_dimensionality_size: 2
 
 exp_cfg:
-  # Set this flag to true to run test experiments during development
   seed: 37
   trial_run: false
-  exp_id:
+  exp_id: "personal_finance_o3-mini_r1"
 
 defaults:
   - _self_
-  - capabilities: math
+  - capabilities: personal_finance

src/create_seed_capabilities.py

@@ -285,7 +285,7 @@ def main(cfg: DictConfig) -> None:
             # Only keep problem and answer
             capability_repr_tasks = [
                 {"problem": s["problem"], "answer": s["answer"]}
-                for s in tasks[: dataset._cfg["data_args"]["num_repr_tasks"]]
+                for s in gsm_tasks[: dataset._cfg["data_args"]["num_repr_tasks"]]
             ]
 
             populate_seed_capability_dir(
@@ -302,6 +302,47 @@ def main(cfg: DictConfig) -> None:
             logger.info(
                 f"Created capability {capability_name} with {len(gsm_tasks)} tasks."
             )
+        elif dataset.name == "personal_finance_basic":
+            capability_name = "personal_finance_basic"
+
+            # Reformat raw data
+            pf_tasks = []
+            for task_id, task in enumerate(dataset._data):
+                task["id"] = str(task_id + 1)
+                task["problem"] = task.pop("problem")
+                task["answer"] = task.pop("solution")
+                task["solution"] = task.pop("reasoning")
+                task.pop("area")
+                task.pop("capability")
+                pf_tasks.append(task)
+
+            # Prepare instructions
+            capability_instructions = dataset.instructions.format(
+                problem='{t["problem"]}'
+            )
+
+            # Create representative tasks
+            capability_repr_tasks = [
+                {"problem": t["problem"], "answer": t["answer"]}
+                for t in pf_tasks[: dataset._cfg["data_args"]["num_repr_tasks"]]
+            ]
+
+            populate_seed_capability_dir(
+                base_dir=seed_capability_dir,
+                capability_name=capability_name,
+                capability_description=dataset.description,
+                capability_domain=dataset.domain,
+                capability_data=pf_tasks,
+                capability_repr_tasks=capability_repr_tasks,
+                capability_instructions=capability_instructions,
+                capability_score_func=constants.PERSONAL_FINANCE_BASIC_SCORE_FUNC.strip(
+                    "\n"
+                ),
+                source_dataset=dataset.name,
+            )
+            logger.info(
+                f"Created capability {capability_name} with {len(pf_tasks)} tasks."
+            )
 
 
 if __name__ == "__main__":

src/utils/constants.py

@@ -4,9 +4,9 @@
 from enum import Enum
 
 
-BASE_ARTIFACTS_DIR = "/fs01/projects/aieng/public/ace/artifacts"
+BASE_ARTIFACTS_DIR = "/h/fkohankh/automated_capability_evaluation_logs"
 GCP_BASE_ARTIFACTS_DIR = "gs://ace-artifacts"
-BASE_INSPECT_EVALS_DIR = "/fs01/projects/aieng/public/ace/inspect_evals/src/ace_evals"
+BASE_INSPECT_EVALS_DIR = "/h/fkohankh/automated_capability_evaluation_logs"
 
 SEED_CAPABILITIES_SCORE_DIR = os.path.join(
     GCP_BASE_ARTIFACTS_DIR, "seed_capabilities_results"
@@ -24,10 +24,12 @@
 MATHEMATICS_SCORE_FUNC = f"""def score(t: dict, submission: str) -> float | None:\n{TAB_W_SPACES}{TAB_W_SPACES}from .utils import parse_submission, evaluate_with_llm_judge\n{TAB_W_SPACES}{TAB_W_SPACES}answer = parse_submission(submission)\n{TAB_W_SPACES}{TAB_W_SPACES}correct = evaluate_with_llm_judge(answer, t["answer"])\n{TAB_W_SPACES}{TAB_W_SPACES}return 1.0 if correct else 0.0"""
 # Score function is based on https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/src/inspect_evals/mathematics/utils.py#L57
 GSM8K_SCORE_FUNC = f"""def score(t: dict, submission: str) -> float | None:\n{TAB_W_SPACES}{TAB_W_SPACES}return 1.0 if submission==t["answer"] else 0.0"""
+PERSONAL_FINANCE_BASIC_SCORE_FUNC = f"""def score(t: dict, submission: str) -> float | None:\n{TAB_W_SPACES}{TAB_W_SPACES}from .utils import parse_submission, evaluate_with_llm_judge\n{TAB_W_SPACES}{TAB_W_SPACES}answer = parse_submission(submission)\n{TAB_W_SPACES}{TAB_W_SPACES}correct = evaluate_with_llm_judge(answer, t["answer"])\n{TAB_W_SPACES}{TAB_W_SPACES}return 1.0 if correct else 0.0"""
 
 DATASET_NAME_MAP = {
     "mathematics": "competition_math",
     "gsm8k": "word_problems",
+    "personal_finance_basic": "personal_finance_basic",
 }