Set KL divergence coefficient to zero in loss function

Changed beta parameter from 0.1 to 0.0 in simple_grpo_loss to remove
the KL divergence penalty term from the loss.

Author: casteryh

sandbox/grpo_language/main.py

@@ -125,7 +125,7 @@ def simple_grpo_loss(
     ref_logprobs: torch.Tensor,
     advantages: torch.Tensor,
     padding_mask: torch.Tensor,
-    beta: float = 0.1,
+    beta: float = 0.0,
 ) -> torch.Tensor:
     logprobs: torch.Tensor = compute_logprobs(logits, response)
     kl = torch.exp(ref_logprobs - logprobs) - (ref_logprobs - logprobs) - 1