Added new domains - Translation, Golden and also update the new domains in the band section

Author: sidrocks

experiments/3_coreset_engineering/coreset_engine_v5/config/curriculum.yaml

@@ -108,7 +108,7 @@ difficulty_system:
       name: "Nursery"
       intent: "Surface language acquisition"
       allowed_modalities: ["general_text"]
-      allowed_domains: ["web", "social", "qa"]
+      allowed_domains: ["web", "social", "qa", "education", "language_literacy", "conversation", "translation","golden"]
       constraints:
         tokenizer:
           avg_max: 5000
@@ -123,7 +123,7 @@ difficulty_system:
       name: "Primary"
       intent: "Fluent everyday language"
       allowed_modalities: ["general_text", "clean_exposition"]
-      allowed_domains: ["web", "encyclopedia", "news", "social", "qa"]
+      allowed_domains: ["web","encyclopedia","news","social","qa","education","language_literacy","conversation","translation","golden"]
       constraints:
         tokenizer:
           avg_max: 10000
@@ -138,8 +138,7 @@ difficulty_system:
       name: "HighSchool"
       intent: "Structured knowledge without explicit reasoning"
       allowed_modalities: ["general_text", "structured_knowledge"]
-      allowed_domains:
-        ["encyclopedia", "news", "education", "literature", "web", "qa"]
+      allowed_domains: ["encyclopedia", "news","education", "literature","web","qa", "conversation","translation","golden"]
       constraints:
         tokenizer:
           avg_max: 20000
@@ -154,7 +153,7 @@ difficulty_system:
       name: "Undergraduate"
       intent: "Reasoning emergence"
       allowed_modalities: ["structured_knowledge", "technical_text", "code"]
-      allowed_domains: ["science", "math", "education", "code", "literature"]
+      allowed_domains:  ["science", "math", "education", "code", "literature", "conversation","translation","golden"]
       constraints:
         tokenizer:
           avg_max: 40000
@@ -171,7 +170,7 @@ difficulty_system:
       name: "Graduate"
       intent: "Explicit abstraction and algorithms"
       allowed_modalities: ["technical_text", "math", "code"]
-      allowed_domains: ["science", "math", "code", "instruction"]
+      allowed_domains: ["science", "math", "code", "instruction","golden"]
       constraints:
         tokenizer:
           avg_max: 70000
@@ -187,9 +186,8 @@ difficulty_system:
     B5:
       name: "PhD"
       intent: "Planning and system-level reasoning"
-      allowed_modalities:
-        ["hard_reasoning", "math", "advanced_code", "planning"]
-      allowed_domains: ["instruction", "science", "math", "code"]
+      allowed_modalities: ["hard_reasoning", "math", "advanced_code", "planning"]
+      allowed_domains: ["instruction", "science", "math", "code","golden"]
       constraints:
         tokenizer:
           avg_max: .inf
@@ -391,31 +389,17 @@ domains:
 
   band_domain_policy:
     B0:
-      ["web", "social", "qa", "education", "language_literacy", "conversation"]
+      ["web", "social", "qa", "education", "language_literacy", "conversation", "translation","golden"]
     B1:
-      [
-        "web",
-        "encyclopedia",
-        "news",
-        "social",
-        "qa",
-        "education",
-        "language_literacy",
-        "conversation",
-      ]
+      ["web","encyclopedia","news","social","qa","education","language_literacy","conversation","translation","golden"]
     B2:
-      [
-        "encyclopedia",
-        "news",
-        "education",
-        "literature",
-        "web",
-        "qa",
-        "conversation",
-      ]
-    B3: ["science", "math", "education", "code", "literature", "conversation"]
-    B4: ["science", "math", "code", "instruction"]
-    B5: ["instruction", "science", "math", "code"]
+      ["encyclopedia", "news","education", "literature","web","qa", "conversation","translation","golden"]
+    B3: 
+      ["science", "math", "education", "code", "literature", "conversation","translation","golden"]
+    B4: 
+      ["science", "math", "code", "instruction","golden"]
+    B5:
+      ["instruction", "science", "math", "code","golden"]
 
 # ============================================================
 # EVALUATION SIGNAL INTERFACE

experiments/3_coreset_engineering/coreset_engine_v5/tests/test_optimizations.py

@@ -300,6 +300,7 @@ def chunk_generator():
                     language="en",
                     band=DifficultyBand("B2"),
                     source_doc_id="doc_1",
+                    t1_file_path="doc_2.parquet"
                 )
                 yield (f"chunk_{i:05d}", metadata)
 

experiments/3_coreset_engineering/coreset_engine_v5/tests/test_pipeline.py

@@ -87,6 +87,7 @@ def test_chunk_metadata_creation(self):
             language="en",
             band=DifficultyBand.B3,
             source_doc_id="doc_001",
+            t1_file_path="doc_2.parquet",
         )
         assert metadata.chunk_id == "chunk_001"
         assert metadata.band == DifficultyBand.B3
@@ -332,6 +333,7 @@ def test_selection_using_real_sample(self, tmp_path):
                     band=DifficultyBand(data.get("band", "B0")),
                     source_doc_id=data.get("source_doc_id", ""),
                     source_url=data.get("source_url", None),
+                    t1_file_path="doc_2.parquet",
                 )
                 all_chunks[chunk_id] = meta
 
@@ -417,6 +419,7 @@ def test_selection_using_large_sample(self):
                 band=DifficultyBand(data.get("band", "B0")),
                 source_doc_id=data.get("source_doc_id", ""),
                 source_url=data.get("source_url", None),
+                t1_file_path="doc_2.parquet",
             )
             if "token_ids" in data:
                 setattr(meta, "token_ids", list(data["token_ids"]))

experiments/3_coreset_engineering/coreset_engine_v5/tests/test_protected_slice_domain_eligibility.py

@@ -46,6 +46,7 @@ def test_protected_slice_enforcement_respects_band_allowed_domains(
             band=DifficultyBand.B4,
             source_doc_id="doc",
             source_url=None,
+            t1_file_path="doc_2.parquet",
         ),
         "c002": ChunkMetadata(
             chunk_id="c002",
@@ -57,6 +58,7 @@ def test_protected_slice_enforcement_respects_band_allowed_domains(
             band=DifficultyBand.B4,
             source_doc_id="doc",
             source_url=None,
+            t1_file_path="doc_2.parquet",
         ),
     }
 

experiments/3_coreset_engineering/coreset_engine_v5/tools/generate_curriculum_aligned_sample.py

@@ -166,6 +166,7 @@ def generate(
                 "quality_flags": [],
                 "sensitive_markers": [],
                 "start_offset": 0,
+                "t1_file_path":"doc_2.parquet",
             }
             f.write(json.dumps(row) + "\n")
 

experiments/3_coreset_engineering/coreset_engine_v5/tools/generate_large_sample.py

@@ -36,6 +36,7 @@ def generate(out_path: Path, n: int, seed: int = 42):
                 "sensitive_markers": [],
                 "start_offset": 0,
                 "token_ids": token_ids,
+                "t1_file_path":"doc_2.parquet"
             }
             f.write(json.dumps(chunk) + "\n")
 

experiments/3_coreset_engineering/coreset_engine_v5/tools/merge_selected_indices.py

@@ -37,6 +37,7 @@
     "source_doc_id",
     "source_url",
     "source",
+    "t1_file_path",
 ]
 
 LEGACY_TOKEN_COLUMN = "token_count_estimate"