diff --git a/lib/langchain/llm/ollama.rb b/lib/langchain/llm/ollama.rb index a0d902cb5..a775d6db0 100644 --- a/lib/langchain/llm/ollama.rb +++ b/lib/langchain/llm/ollama.rb @@ -287,8 +287,16 @@ def auth_headers end def json_responses_chunk_handler(&block) + incomplete_chunk_line = nil proc do |chunk, _size| chunk.split("\n").each do |chunk_line| + if incomplete_chunk_line + chunk_line = incomplete_chunk_line + chunk_line + incomplete_chunk_line = nil + end + + next incomplete_chunk_line = chunk_line unless chunk_line.end_with?("}") + parsed_chunk = JSON.parse(chunk_line) block.call(parsed_chunk) end diff --git a/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_chat_handles_multiline_json.yml b/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_chat_handles_multiline_json.yml new file mode 100644 index 000000000..28532fc14 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_chat_handles_multiline_json.yml @@ -0,0 +1,35 @@ +--- +http_interactions: + - request: + method: post + uri: http://localhost:11434/api/chat + body: + encoding: UTF-8 + string: '{"messages":[{"role":"user","content":"Hey! How are you?"}],"model":"llama3.2","stream":false,"temperature":0.0}' + headers: + User-Agent: + - Faraday v2.12.0 + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Date: + - Sat, 30 Nov 2024 03:33:52 GMT + Content-Length: + - "491" + body: + encoding: UTF-8 + string: | + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.105113Z","message":{"role":"assistant","content":"I'm just a language model, so I don't have feelings or emotions like humans do. However, I'm functioning properly and ready to help with any questions or tasks you may have! How can I assist you today?"}, + "done_reason":"stop","done":true,"total_duration":839744125,"load_duration":29176166,"prompt_eval_count":31,"prompt_eval_duration":166000000,"eval_count":46,"eval_duration":643000000} + recorded_at: Sat, 30 Nov 2024 03:33:52 GMT +recorded_with: VCR 6.2.0 diff --git a/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_chat_when_passing_a_block_handles_multiline_json.yml b/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_chat_when_passing_a_block_handles_multiline_json.yml new file mode 100644 index 000000000..06ab4752a --- /dev/null +++ b/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_chat_when_passing_a_block_handles_multiline_json.yml @@ -0,0 +1,85 @@ +--- +http_interactions: + - request: + method: post + uri: http://localhost:11434/api/chat + body: + encoding: UTF-8 + string: '{"messages":[{"role":"user","content":"Hey! How are you?"}],"model":"llama3.2","stream":true,"temperature":0.0}' + headers: + User-Agent: + - Faraday v2.12.0 + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/x-ndjson + Date: + - Sat, 30 Nov 2024 03:33:52 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: | + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.17055Z","message":{"role":"assistant","content":"I"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.184932Z","message":{"role":"assistant","content":"'m"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.199352Z","message":{"role":"assistant","content":" just"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.213534Z","message":{"role":"assistant","content":" a"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.227945Z","message":{"role":"assistant","content":" language"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.242426Z","message":{"role":"assistant","content":" model"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.256777Z","message":{"role":"assistant","content":","},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.27125Z","message":{"role":"assistant","content":" so"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.285721Z","message":{"role":"assistant","content":" I"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.300169Z","message":{"role":"assistant","content":" don"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.314465Z","message":{"role":"assistant","content":"'t"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.328988Z","message":{"role":"assistant","content":" have"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.343239Z","message":{"role":"assistant","content":" emotions"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.357577Z","message":{"role":"assistant","content":" or"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.371938Z","message":{"role":"assistant","content":" feelings"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.386247Z","message":{"role":"assistant","content":" like"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.400737Z","message":{"role":"assistant","content":" humans"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.414989Z","message":{"role":"assistant","content":" do"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.429373Z","message":{"role":"assistant","content":"."},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.443679Z","message":{"role":"assistant","content":" However"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.457956Z","message":{"role":"assistant","content":","},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.472256Z","message":{"role":"assistant","content":" I"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.48679Z","message":{"role":"assistant","content":"'m"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.501257Z","message":{"role":"assistant","content":" functioning"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.515596Z","message":{"role":"assistant","content":" properly"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.530082Z","message":{"role":"assistant","content":" and"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.544442Z","message":{"role":"assistant","content":" ready"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.558968Z","message":{"role":"assistant","content":" to"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.573301Z","message":{"role":"assistant","content":" help"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.58771Z","message":{"role":"assistant","content":" with"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.602027Z","message":{"role":"assistant","content":" any"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.616521Z","message":{"role":"assistant","content":" questions"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.631006Z","message":{"role":"assistant","content":" or"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.645308Z","message":{"role":"assistant","content":" tasks"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.659801Z","message":{"role":"assistant","content":" you"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.674227Z","message":{"role":"assistant","content":" may"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.688736Z","message":{"role":"assistant","content":" have"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.703259Z","message":{"role":"assistant","content":"."},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.717558Z","message":{"role":"assistant","content":" How"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.732091Z","message":{"role":"assistant","content":" about"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.746467Z","message":{"role":"assistant","content":" you"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.760939Z","message":{"role":"assistant","content":"?"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.775227Z","message":{"role":"assistant","content":" How"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.789709Z","message":{"role":"assistant","content":"'s"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.804113Z","message":{"role":"assistant","content":" your"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.818659Z","message":{"role":"assistant","content":" day"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.833047Z","message":{"role":"assistant","content":" going"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.847505Z","message":{"role":"assistant","content":" so"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.862049Z","message":{"role":"assistant","content":" far"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.876546Z","message":{"role":"assistant","content":"?"},"done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:52.890984Z","message": + {"role":"assistant","content":""},"done_reason":"stop","done":true,"total_duration":746459542,"load_duration":10482042,"prompt_eval_count":31,"prompt_eval_duration":14000000,"eval_count":51,"eval_duration":721000000} + recorded_at: Sat, 30 Nov 2024 03:33:52 GMT +recorded_with: VCR 6.2.0 diff --git a/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_complete_handles_multiline_json.yml b/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_complete_handles_multiline_json.yml new file mode 100644 index 000000000..856f960d6 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_complete_handles_multiline_json.yml @@ -0,0 +1,35 @@ +--- +http_interactions: + - request: + method: post + uri: http://localhost:11434/api/generate + body: + encoding: UTF-8 + string: '{"prompt":"In one word, life is ","model":"llama3.2","stream":false,"options":{"temperature":0.0}}' + headers: + User-Agent: + - Faraday v2.12.0 + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Date: + - Sat, 30 Nov 2024 03:33:27 GMT + Content-Length: + - "456" + body: + encoding: UTF-8 + string: | + {"model":"llama3.2","created_at":"2024-11-30T03:33:27.627456Z","response":"Complicated.","done":true,"done_reason":"stop", + "context":[128006,9125,128007,271,38766,1303,33025,2696,25,6790,220,2366,18,271,128009,128006,882,128007,271,644,832,3492,11,2324,374,220,128009,128006,78191,128007,271,13864,14040,13],"total_duration":820116458,"load_duration":608240166,"prompt_eval_count":32,"prompt_eval_duration":166000000,"eval_count":4,"eval_duration":43000000} + recorded_at: Sat, 30 Nov 2024 03:33:27 GMT +recorded_with: VCR 6.2.0 diff --git a/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_complete_when_passing_a_block_handles_multiline_json.yml b/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_complete_when_passing_a_block_handles_multiline_json.yml new file mode 100644 index 000000000..4e6b76479 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/Langchain_LLM_Ollama_complete_when_passing_a_block_handles_multiline_json.yml @@ -0,0 +1,38 @@ +--- +http_interactions: + - request: + method: post + uri: http://localhost:11434/api/generate + body: + encoding: UTF-8 + string: '{"prompt":"In one word, life is ","model":"llama3.2","stream":true,"options":{"temperature":0.0}}' + headers: + User-Agent: + - Faraday v2.12.0 + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/x-ndjson + Date: + - Sat, 30 Nov 2024 03:33:56 GMT + Transfer-Encoding: + - chunked + body: + encoding: UTF-8 + string: | + {"model":"llama3.2","created_at":"2024-11-30T03:33:56.023803Z","response":"Comp","done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:56.037834Z","response":"licated","done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:56.052315Z","response":".","done":false} + {"model":"llama3.2","created_at":"2024-11-30T03:33:56.06672Z","response":"","done":true,"done_reason":"stop", + "context":[128006,9125,128007,271,38766,1303,33025,2696,25,6790,220,2366,18,271,128009,128006,882,128007,271,644,832,3492,11,2324,374,220,128009,128006,78191,128007,271,13864,14040,13],"total_duration":119446542,"load_duration":8969375,"prompt_eval_count":32,"prompt_eval_duration":66000000,"eval_count":4,"eval_duration":43000000} + recorded_at: Sat, 30 Nov 2024 03:33:56 GMT +recorded_with: VCR 6.2.0 diff --git a/spec/lib/langchain/llm/ollama_spec.rb b/spec/lib/langchain/llm/ollama_spec.rb index b9945f891..dba4ed9a3 100644 --- a/spec/lib/langchain/llm/ollama_spec.rb +++ b/spec/lib/langchain/llm/ollama_spec.rb @@ -87,6 +87,11 @@ expect(response.completion).to eq("Complicated.") end + it "handles multiline json", :vcr do + expect { response }.not_to raise_error + expect(response.completion).to eq("Complicated.") + end + it "does not use streamed responses", vcr: {cassette_name: "Langchain_LLM_Ollama_complete_returns_a_completion"} do expect(client).to receive(:post).with("api/generate", hash_including(stream: false)).and_call_original response @@ -102,6 +107,11 @@ expect(response.total_tokens).to eq(36) end + it "handles multiline json", :vcr do + expect { response }.not_to raise_error + expect(response.completion).to eq("Complicated.") + end + it "uses streamed responses", vcr: {cassette_name: "Langchain_LLM_Ollama_complete_when_passing_a_block_returns_a_completion"} do expect(client).to receive(:post).with("api/generate", hash_including(stream: true)).and_call_original response @@ -125,6 +135,11 @@ expect(response.chat_completion).to include("I'm just a language model") end + it "handles multiline json", :vcr do + expect { response }.not_to raise_error + expect(response.chat_completion).to include("I'm just a language model") + end + it "does not use streamed responses", vcr: {cassette_name: "Langchain_LLM_Ollama_chat_returns_a_chat_completion"} do expect(client).to receive(:post).with("api/chat", hash_including(stream: false)).and_call_original response @@ -139,6 +154,11 @@ expect(response.chat_completion).to include("I'm just a language model") end + it "handles multiline json", :vcr do + expect { response }.not_to raise_error + expect(response.chat_completion).to include("I'm just a language model") + end + it "uses streamed responses", vcr: {cassette_name: "Langchain_LLM_Ollama_chat_when_passing_a_block_returns_a_chat_completion"} do expect(client).to receive(:post).with("api/chat", hash_including(stream: true)).and_call_original response