refactor: refactor data format to support multi-modal input

Author: ChenZiHong-Gavin

examples/configs/cot_config.yaml

@@ -49,7 +49,7 @@ nodes:
       - quiz
     execution_params:
       replicas: 1
-      batch_size: 16
+      batch_size: 128
 
   - id: partition
     op_name: partition
@@ -71,7 +71,7 @@ nodes:
       - partition
     execution_params:
       replicas: 1
-      batch_size: 16
+      batch_size: 128
     params:
       method: aggregated # atomic, aggregated, multi_hop, cot, vqa
       data_format: ChatML # Alpaca, Sharegpt, ChatML

examples/configs/multi_hop_config.yaml

@@ -8,20 +8,25 @@ nodes:
     dependencies: []
     params:
       input_path:
-        - resources/input_examples/json_demo.json
+        - examples/input_examples/json_demo.json
 
   - id: chunk
     op_name: chunk
     type: map_batch
     dependencies:
       - read
+    execution_params:
+      replicas: 4
     params:
       chunk_size: 1024
       chunk_overlap: 100
 
   - id: build_kg
     op_name: build_kg
     type: map_batch
+    execution_params:
+      replicas: 1
+      batch_size: 128
     dependencies:
       - chunk
 
@@ -40,6 +45,9 @@ nodes:
     type: map_batch
     dependencies:
       - partition
+    execution_params:
+      replicas: 1
+      batch_size: 128
     params:
       method: atomic
       data_format: Alpaca

examples/configs/vqa_config.yaml

@@ -0,0 +1 @@
+# Generate CoT QAs

examples/extract/extract_schema_guided.sh …t_schema_guided/extract_schema_guided.shexamples/extract/extract_schema_guided.sh renamed to examples/extract/extract_schema_guided/extract_schema_guided.sh examples/extract/extract_schema_guided.sh renamed to examples/extract/extract_schema_guided/extract_schema_guided.sh

@@ -0,0 +1,55 @@
+global_params:
+  working_dir: cache
+
+nodes:
+  - id: read
+    op_name: read
+    type: source
+    dependencies: []
+    params:
+      input_path:
+        - examples/input_examples/txt_demo.txt
+
+  - id: chunk
+    op_name: chunk
+    type: map_batch
+    dependencies:
+      - read
+    execution_params:
+      replicas: 4
+    params:
+      chunk_size: 1024
+      chunk_overlap: 100
+
+  - id: build_kg
+    op_name: build_kg
+    type: map_batch
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    dependencies:
+      - chunk
+
+  - id: partition
+    op_name: partition
+    type: aggregate
+    dependencies:
+      - build_kg
+    params:
+      method: leiden
+      method_params:
+        max_size: 20
+        use_lcc: false
+        random_seed: 42
+
+  - id: generate
+    op_name: generate
+    type: map_batch
+    dependencies:
+      - partition
+    execution_params:
+      replicas: 1
+      batch_size: 128
+    params:
+      method: cot
+      data_format: Sharegpt