FIX: do not strip empty string during stream processing (#911)

Fixes issue in Open AI provider eating newlines and spaces

Author: SamSaffron

lib/completions/open_ai_message_processor.rb

@@ -65,7 +65,8 @@ def process_streamed_message(json)
         @tool.parameters = parsed_args
         rval = @tool
         @tool = nil
-      elsif content.present?
+      elsif !content.to_s.empty?
+        # we don't want to strip empty content like "\n", do not use present?
         rval = content
       end
 

spec/lib/completions/endpoints/open_ai_spec.rb

@@ -519,6 +519,28 @@ def request_body(prompt, stream: false, tool_call: false)
           expect(response).to eq(tool_calls)
         end
 
+        it "properly handles newlines" do
+          response = <<~TEXT.strip
+            data: {"id":"chatcmpl-ASngi346UA9k006bF6GBRV66tEJfQ","object":"chat.completion.chunk","created":1731427548,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"content":":\\n\\n"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+            data: {"id":"chatcmpl-ASngi346UA9k006bF6GBRV66tEJfQ","object":"chat.completion.chunk","created":1731427548,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"content":"```"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+            data: {"id":"chatcmpl-ASngi346UA9k006bF6GBRV66tEJfQ","object":"chat.completion.chunk","created":1731427548,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"content":"ruby"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+            data: {"id":"chatcmpl-ASngi346UA9k006bF6GBRV66tEJfQ","object":"chat.completion.chunk","created":1731427548,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"content":"\\n"},"logprobs":null,"finish_reason":null}],"usage":null}
+
+            data: {"id":"chatcmpl-ASngi346UA9k006bF6GBRV66tEJfQ","object":"chat.completion.chunk","created":1731427548,"model":"gpt-4o-2024-08-06","system_fingerprint":"fp_159d8341cc","choices":[{"index":0,"delta":{"content":"def"},"logprobs":null,"finish_reason":null}],"usage":null}
+         TEXT
+
+          open_ai_mock.stub_raw(response)
+          partials = []
+
+          dialect = compliance.dialect(prompt: compliance.generic_prompt)
+          endpoint.perform_completion!(dialect, user) { |partial| partials << partial }
+
+          expect(partials).to eq([":\n\n", "```", "ruby", "\n", "def"])
+        end
+
         it "uses proper token accounting" do
           response = <<~TEXT.strip
             data: {"id":"chatcmpl-9OZidiHncpBhhNMcqCus9XiJ3TkqR","object":"chat.completion.chunk","created":1715644203,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_729ea513f7","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"usage":null}|