Merge pull request #61 from LLMSQL/60-bug-change-names-of-the-variables

text fixed

Author: DzmitryPihulski

README.md

@@ -92,7 +92,7 @@ results = inference_transformers(
     batch_size=8,
     max_new_tokens=256,
     do_sample=False,
-    model_args={
+    model_kwargs={
         "torch_dtype": "bfloat16",
     }
 )

docs/docs/usage.rst

@@ -33,11 +33,11 @@ Using transformers backend.
         batch_size=8,
         max_new_tokens=256,
         temperature=0.7,
-        model_args={
+        model_kwargs={
             "attn_implementation": "flash_attention_2",
             "torch_dtype": "bfloat16",
         },
-        generate_kwargs={
+        generation_kwargs={
             "do_sample": False,
         },
     )

llmsql/__main__.py

@@ -31,7 +31,7 @@ def main() -> None:
   llmsql inference --method transformers \
       --model-or-model-name-or-path meta-llama/Llama-3-8b-instruct \
       --output-file outputs/llama_preds.jsonl \
-      --model-args '{"attn_implementation": "flash_attention_2", "torch_dtype": "bfloat16"}'
+      --model-kwargs '{"attn_implementation": "flash_attention_2", "torch_dtype": "bfloat16"}'
 
   # 4️⃣ Pass LLM init kwargs (for vLLM)
   llmsql inference --method vllm \
@@ -44,7 +44,7 @@ def main() -> None:
       --model-or-model-name-or-path Qwen/Qwen2.5-1.5B-Instruct \
       --output-file outputs/temp_0.9.jsonl \
       --temperature 0.9 \
-      --generate-kwargs '{"do_sample": true, "top_p": 0.9, "top_k": 40}'
+      --generation-kwargs '{"do_sample": true, "top_p": 0.9, "top_k": 40}'
 """
 
     inf_parser = subparsers.add_parser(
@@ -127,15 +127,17 @@ def main() -> None:
             except json.JSONDecodeError:
                 print("⚠️  Could not parse --llm-kwargs JSON, passing as string.")
 
-        if fn_kwargs.get("model_args") is not None:
+        if fn_kwargs.get("model_kwargs") is not None:
             try:
-                fn_kwargs["model_args"] = json.loads(fn_kwargs["model_args"])
+                fn_kwargs["model_kwargs"] = json.loads(fn_kwargs["model_kwargs"])
             except json.JSONDecodeError:
                 raise
 
-        if fn_kwargs.get("generate_kwargs") is not None:
+        if fn_kwargs.get("generation_kwargs") is not None:
             try:
-                fn_kwargs["generate_kwargs"] = json.loads(fn_kwargs["generate_kwargs"])
+                fn_kwargs["generation_kwargs"] = json.loads(
+                    fn_kwargs["generation_kwargs"]
+                )
             except json.JSONDecodeError:
                 raise
 

llmsql/inference/README.md

@@ -33,21 +33,21 @@ pip install llmsql[vllm]
 from llmsql import inference_transformers
 
 results = inference_transformers(
-    model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
-    output_file="outputs/preds_transformers.jsonl",
-    questions_path="data/questions.jsonl",
-    tables_path="data/tables.jsonl",
-    num_fewshots=5,
-    batch_size=8,
-    max_new_tokens=256,
-    temperature=0.7,
-    model_args={
-        "torch_dtype": "bfloat16",
-    },
-    generate_kwargs={
-        "do_sample": False,
-    },
-)
+        model_or_model_name_or_path="Qwen/Qwen2.5-1.5B-Instruct",
+        output_file="outputs/preds_transformers.jsonl",
+        questions_path="data/questions.jsonl",
+        tables_path="data/tables.jsonl",
+        num_fewshots=5,
+        batch_size=8,
+        max_new_tokens=256,
+        temperature=0.7,
+        model_kwargs={
+            "torch_dtype": "bfloat16",
+        },
+        generation_kwargs={
+            "do_sample": False,
+        },
+    )
 ```
 
 ---

llmsql/inference/inference_transformers.py

@@ -21,10 +21,10 @@
         batch_size=8,
         max_new_tokens=256,
         temperature=0.7,
-        model_args={
+        model_kwargs={
             "torch_dtype": "bfloat16",
         },
-        generate_kwargs={
+        generation_kwargs={
             "do_sample": False,
         },
     )