fix

Author: casteryh

.claude/settings.local.json

@@ -0,0 +1,14 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(source ~/.bashrc)",
+      "Bash(conda activate forge-monarch-0-1-1)",
+      "Bash(pip install:*)",
+      "Bash(https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 pip install:*)",
+      "Bash(python -m pytest:*)",
+      "Bash(https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 git push:*)"
+    ],
+    "deny": [],
+    "ask": []
+  }
+}

apps/grpo/main.py

@@ -6,6 +6,19 @@
 
 # Usage: python -m apps.grpo.main --config apps/grpo/qwen3_1_7b.yaml
 
+# Patch importlib.metadata.distributions before wandb imports it
+# to filter out packages with None metadata
+import importlib.metadata
+_original_distributions = importlib.metadata.distributions
+
+def _patched_distributions():
+    """Filter out distributions with None metadata"""
+    for dist in _original_distributions():
+        if dist.metadata is not None:
+            yield dist
+
+importlib.metadata.distributions = _patched_distributions
+
 import asyncio
 import time
 import uuid
@@ -125,12 +138,9 @@ def simple_grpo_loss(
     ref_logprobs: torch.Tensor,
     advantages: torch.Tensor,
     padding_mask: torch.Tensor,
-    beta: float = 0.1,
 ) -> torch.Tensor:
     logprobs: torch.Tensor = compute_logprobs(logits, response)
-    kl = torch.exp(ref_logprobs - logprobs) - (ref_logprobs - logprobs) - 1
-    per_token_policy_loss = torch.exp(logprobs - logprobs.detach()) * advantages
-    per_token_loss = -(per_token_policy_loss - beta * kl)
+    per_token_loss = torch.exp(logprobs - logprobs.detach()) * advantages.detach()
     loss = (
         ((per_token_loss * padding_mask).sum(dim=1))
         / (padding_mask.sum(dim=1).clamp(min=1.0))

apps/grpo/qwen3_1_7b.yaml

@@ -7,10 +7,10 @@ local_batch_size: 16 # per-device batch size
 max_req_tokens: 1024
 max_res_tokens: 1024
 model: "Qwen/Qwen3-1.7B"
-off_by_n: 1 # Off by one by default
+off_by_n: 0 # Off by one by default
 
 # Main loop configuration
-rollout_threads: 1   # Recommended to set equal to policy.num_replicas
+rollout_threads: 1  # Recommended to set equal to policy.num_replicas
 
 
 # Observability configuration

src/forge/__init__.py

@@ -25,14 +25,15 @@
 # to filter out packages with None metadata
 import importlib.metadata
 
-_original_distributions = importlib.metadata.distributions
-
-
-def _patched_distributions():
-    """Filter out distributions with None metadata"""
-    for dist in _original_distributions():
-        if dist.metadata is not None:
-            yield dist
-
-
-importlib.metadata.distributions = _patched_distributions
+# Guard to ensure this runs only once
+if not hasattr(importlib.metadata, "_distributions_patched"):
+    _original_distributions = importlib.metadata.distributions
+
+    def _patched_distributions():
+        """Filter out distributions with None metadata"""
+        for dist in _original_distributions():
+            if dist.metadata is not None:
+                yield dist
+
+    importlib.metadata.distributions = _patched_distributions
+    importlib.metadata._distributions_patched = True