feat: added scripts for experiments

Author: Darinochka

scripts/experiments/generate_csv_scoring.py

@@ -0,0 +1,47 @@
+import os
+import json
+import pandas as pd
+import argparse
+
+def process_logs(base_dir, filter_keyword, output_csv):
+    data = []
+
+    for root, dirs, files in os.walk(base_dir):
+        if filter_keyword in root and 'logs.json' in files:
+            dataset_name = os.path.basename(root)
+            dataset_name = dataset_name.split("_scoring")[0]
+            log_path = os.path.join(root, 'logs.json')
+
+            with open(log_path, 'r') as f:
+                log_data = json.load(f)
+
+            for metric_type, metrics in log_data['metrics'].items():
+                for config in log_data['configs'][metric_type]:
+                    method = config['module_type']
+                    metric_name = config['metric_name']
+                    metric_value = config['metric_value']
+
+                    row = {
+                        'dataset': dataset_name,
+                        'method': method,
+                        'metric': metric_name,
+                        'metric_value': metric_value
+                    }
+
+                    for param, value in config['module_params'].items():
+                        row[param] = value
+
+                    data.append(row)
+
+    df = pd.DataFrame(data)
+    df.to_csv(output_csv, index=False)
+    print(f"CSV file '{output_csv}' successfully created!")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Process logs and generate a CSV file.")
+    parser.add_argument('--base_dir', type=str, required=True, help="Base directory where folders are located")
+    parser.add_argument('--filter_keyword', type=str, required=True, help="Keyword to filter folders (e.g., 'multilabel')")
+    parser.add_argument('--output_csv', type=str, required=True, help="Output CSV file name")
+    args = parser.parse_args()
+
+    process_logs(args.base_dir, args.filter_keyword, args.output_csv)

scripts/experiments/generate_experiments.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+DATA_PATH="experiments/intent_description"
+LOG_PATH="experiments/intent_description/multilabel"
+METRIC="scoring_hit_rate"
+USE_MULTILABEL=true
+CONFIG_SCRIPT_PATH="./update_metric.sh"
+
+for FILE in "$DATA_PATH"/*.json; do
+  FILENAME=$(basename "$FILE" .json)
+  DATASET_NAME=$(echo "$FILENAME" | sed 's/_fix.*//')
+
+  # Determine the appropriate multilabel flag for the metric update script
+  if [ "$USE_MULTILABEL" = true ]; then
+    MULTILABEL_ARG="true"
+  else
+    MULTILABEL_ARG="false"
+  fi
+
+  # Update the metric in the configuration file
+  echo "Updating metric for dataset: $DATASET_NAME"
+  $CONFIG_SCRIPT_PATH "$METRIC" "$MULTILABEL_ARG"
+  if [ $? -ne 0 ]; then
+    echo "Error updating metric for $DATASET_NAME. Exiting."
+    exit 1
+  fi
+
+  rm -rf runs/
+
+  echo "Processing dataset: $DATASET_NAME"
+  autointent data.train_path="$FILE" \
+             logs.dirpath="$LOG_PATH/${DATASET_NAME}_${METRIC}" \
+             seed=42 \
+             vector_index.device=cuda \
+             hydra.job_logging.root.level=INFO \
+             data.force_multilabel="$USE_MULTILABEL"
+
+  if [ $? -ne 0 ]; then
+    echo "Error encountered while processing $FILE. Exiting."
+    exit 1
+  else
+    echo "Successfully processed $FILE"
+  fi
+done
+
+echo "All datasets processed successfully."

scripts/experiments/update_metric.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Check if the required arguments are provided
+if [ -z "$1" ] || [ -z "$2" ]; then
+  echo "Usage: $0 <new_metric> <multilabel:true|false>"
+  exit 1
+fi
+
+NEW_METRIC="$1"
+MULTILABEL="$2"
+
+# Determine the correct configuration file based on the multilabel argument
+if [ "$MULTILABEL" == "true" ]; then
+  CONFIG_PATH="../../autointent/datafiles/default-multilabel-config.yaml"
+elif [ "$MULTILABEL" == "false" ]; then
+  CONFIG_PATH="../../autointent/datafiles/default-multiclass-config.yaml"
+else
+  echo "Invalid value for <multilabel>. Use 'true' or 'false'."
+  exit 1
+fi
+
+# Backup the original configuration file
+cp "$CONFIG_PATH" "${CONFIG_PATH}.bak"
+
+# Update the metric value where node_type=scoring
+yq e "(.nodes[] | select(.node_type == \"scoring\") | .metric) = \"$NEW_METRIC\"" -i "$CONFIG_PATH"
+
+if [ $? -eq 0 ]; then
+  echo "Metric value successfully updated to '$NEW_METRIC' in $CONFIG_PATH where node_type=scoring"
+else
+  echo "Failed to update the metric value in $CONFIG_PATH"
+  exit 1
+fi