diff --git a/examples/speculative-decoding.py b/examples/speculative-decoding.py new file mode 100755 index 0000000..7c89a75 --- /dev/null +++ b/examples/speculative-decoding.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +"""Example script demonstrating speculative decoding.""" + +import lmstudio as lms + +main_model_key = "qwen2.5-7b-instruct-1m" +draft_model_key = "qwen2.5-0.5b-instruct" + +model = lms.llm(main_model_key) +result = model.respond( + "What are the prime numbers between 0 and 100?", + config={ + "draftModel": draft_model_key, + } +) + +print(result) +stats = result.stats +print(f"Accepted {stats.accepted_draft_tokens_count}/{stats.predicted_tokens_count} tokens") diff --git a/examples/structured-response.py b/examples/structured-response.py index c753678..445d4e3 100755 --- a/examples/structured-response.py +++ b/examples/structured-response.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""Example script demonstrating an interactive LLM chatbot.""" +"""Example script demonstrating structured responses.""" import json diff --git a/examples/tool-use-multiple.py b/examples/tool-use-multiple.py old mode 100644 new mode 100755