Merge remote-tracking branch 'upstream/main'

Author: CFSNM

examples/kfto-sft-llm/sft.ipynb

@@ -451,9 +451,42 @@
    "source": [
     "## Testing the Pre-Trained Model\n",
     "\n",
-    "If you've configured the workbench with a NVIDIA GPU or AMD accelerator, you can run inference to validate the output generated by the fine-tuned model and compare it to the output of the pre-trained model.\n",
+    "If you've configured the workbench with a NVIDIA GPU or AMD accelerator, you can run inferences to validate the output generated by the fine-tuned model and compare it to the output of the pre-trained model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ac2be9e5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Install / upgrade dependencies\n",
+    "!pip install --upgrade transformers peft"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "52431fb6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import json\n",
+    "import transformers\n",
     "\n",
-    "Check / update the pathes to the pre-trained and fine-tuned model checkpoints prior to executing the cells below. "
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline\n",
+    "from peft import LoraConfig, PeftModel\n",
+    "from IPython.display import display, Markdown"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "955d1383",
+   "metadata": {},
+   "source": [
+    "Check / update the paths to the pre-trained and fine-tuned model checkpoints prior to executing the cells below. "
    ]
   },
   {
@@ -506,15 +539,15 @@
     "messages = [\n",
     "    {\n",
     "        \"role\": \"user\",\n",
-    "        \"content\": \"\"\n",
+    "        \"content\": \"Janet's ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?\",\n",
     "    }\n",
     "]\n",
     "\n",
-    "outputs = pipeline(messages, max_new_tokens=256)\n",
+    "outputs = pipeline(messages, max_new_tokens=256, temperature = 0.01)\n",
     "\n",
     "output = \"\"\n",
     "for turn in outputs:\n",
-    "    for item in turn[\"generated text\"]:\n",
+    "    for item in turn[\"generated_text\"]:\n",
     "        output += f\"# {item['role']}\\n\\n{item['content']}\\n\\n\"\n",
     "\n",
     "display(Markdown(output))"
@@ -540,7 +573,7 @@
    "outputs": [],
    "source": [
     "# Merge the fine-tuned adapters into the base model \n",
-    "finetuned_path = \"/opt/app-root/src/shared/Meta-Llama-3.1-8B-Instruct/checkpoint-30/\"\n",
+    "finetuned_path = \"/opt/app-root/src/shared/Meta-Llama-3.1-8B-Instruct/checkpoint-300/\"\n",
     "model = PeftModel.from_pretrained(base_model, finetuned_path)\n",
     "model = model.merge_and_unload()"
    ]
@@ -572,15 +605,15 @@
     "messages = [\n",
     "    {\n",
     "        \"role\": \"user\",\n",
-    "        \"content\": \"\"\n",
+    "        \"content\": \"Janet's ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?\",\n",
     "    }\n",
     "]\n",
     "\n",
-    "outputs = pipeline(messages, max_new_tokens=256)\n",
+    "outputs = pipeline(messages, max_new_tokens=256, temperature = 0.01)\n",
     "\n",
     "output = \"\"\n",
     "for turn in outputs:\n",
-    "    for item in turn[\"generated text\"]:\n",
+    "    for item in turn[\"generated_text\"]:\n",
     "        output += f\"# {item['role']}\\n\\n{item['content']}\\n\\n\"\n",
     "\n",
     "display(Markdown(output))"