Skip to content

Commit 8049f59

Browse files
stainless-app[bot]stainless-app[bot]
authored
feat: use Pathname alongside raw IO handles for file uploads (#119)
1 parent d90d3ae commit 8049f59

File tree

76 files changed

+665
-227
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+665
-227
lines changed

‎README.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,23 @@ stream.each do |completion|
8686
end
8787
```
8888

89+
## File uploads
90+
91+
Request parameters that correspond to file uploads can be passed as `StringIO`, or a [`Pathname`](https://rubyapi.org/3.1/o/pathname) instance.
92+
93+
```ruby
94+
require "pathname"
95+
96+
# using `Pathname`, the file will be lazily read, without reading everything in to memory
97+
file_object = openai.files.create(file: Pathname("input.jsonl"), purpose: "fine-tune")
98+
99+
file = File.read("input.jsonl")
100+
# using `StringIO`, useful if you already have the data in memory
101+
file_object = openai.files.create(file: StringIO.new(file), purpose: "fine-tune")
102+
103+
puts(file_object.id)
104+
```
105+
89106
### Errors
90107

91108
When the library is unable to connect to the API, or if the API returns a non-success status code (i.e., 4xx or 5xx response), a subclass of `OpenAI::Error` will be thrown:

‎lib/openai.rb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
require_relative "openai/internal/type/converter"
4141
require_relative "openai/internal/type/unknown"
4242
require_relative "openai/internal/type/boolean"
43+
require_relative "openai/internal/type/io_like"
4344
require_relative "openai/internal/type/enum"
4445
require_relative "openai/internal/type/union"
4546
require_relative "openai/internal/type/array_of"

‎lib/openai/internal/transport/pooled_net_requester.rb‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ def calibrate_socket_timeout(conn, deadline)
5454
# @param blk [Proc]
5555
#
5656
# @yieldparam [String]
57-
# @return [Net::HTTPGenericRequest]
57+
# @return [Array(Net::HTTPGenericRequest, Proc)]
5858
def build_request(request, &blk)
5959
method, url, headers, body = request.fetch_values(:method, :url, :headers, :body)
6060
req = Net::HTTPGenericRequest.new(
@@ -75,12 +75,12 @@ def build_request(request, &blk)
7575
in StringIO
7676
req["content-length"] ||= body.size.to_s unless req["transfer-encoding"]
7777
req.body_stream = OpenAI::Internal::Util::ReadIOAdapter.new(body, &blk)
78-
in IO | Enumerator
78+
in Pathname | IO | Enumerator
7979
req["transfer-encoding"] ||= "chunked" unless req["content-length"]
8080
req.body_stream = OpenAI::Internal::Util::ReadIOAdapter.new(body, &blk)
8181
end
8282

83-
req
83+
[req, req.body_stream&.method(:close)]
8484
end
8585
end
8686

@@ -125,11 +125,12 @@ def execute(request)
125125

126126
eof = false
127127
finished = false
128+
closing = nil
128129
enum = Enumerator.new do |y|
129130
with_pool(url, deadline: deadline) do |conn|
130131
next if finished
131132

132-
req = self.class.build_request(request) do
133+
req, closing = self.class.build_request(request) do
133134
self.class.calibrate_socket_timeout(conn, deadline)
134135
end
135136

@@ -165,7 +166,9 @@ def execute(request)
165166
rescue StopIteration
166167
nil
167168
end
169+
ensure
168170
conn.finish if !eof && conn&.started?
171+
closing&.call
169172
end
170173
[Integer(response.code), response, (response.body = body)]
171174
end

‎lib/openai/internal/type/array_of.rb‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,10 +79,20 @@ def coerce(value, state:)
7979
#
8080
# @param value [Array<Object>, Object]
8181
#
82+
# @param state [Hash{Symbol=>Object}] .
83+
#
84+
# @option state [Boolean] :can_retry
85+
#
8286
# @return [Array<Object>, Object]
83-
def dump(value)
87+
def dump(value, state:)
8488
target = item_type
85-
value.is_a?(Array) ? value.map { OpenAI::Internal::Type::Converter.dump(target, _1) } : super
89+
if value.is_a?(Array)
90+
value.map do
91+
OpenAI::Internal::Type::Converter.dump(target, _1, state: state)
92+
end
93+
else
94+
super
95+
end
8696
end
8797

8898
# @api private

‎lib/openai/internal/type/base_model.rb‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -252,8 +252,12 @@ def coerce(value, state:)
252252
#
253253
# @param value [OpenAI::Internal::Type::BaseModel, Object]
254254
#
255+
# @param state [Hash{Symbol=>Object}] .
256+
#
257+
# @option state [Boolean] :can_retry
258+
#
255259
# @return [Hash{Object=>Object}, Object]
256-
def dump(value)
260+
def dump(value, state:)
257261
unless (coerced = OpenAI::Internal::Util.coerce_hash(value)).is_a?(Hash)
258262
return super
259263
end
@@ -264,15 +268,15 @@ def dump(value)
264268
name = key.is_a?(String) ? key.to_sym : key
265269
case (field = known_fields[name])
266270
in nil
267-
acc.store(name, super(val))
271+
acc.store(name, super(val, state: state))
268272
else
269273
api_name, mode, type_fn = field.fetch_values(:api_name, :mode, :type_fn)
270274
case mode
271275
in :coerce
272276
next
273277
else
274278
target = type_fn.call
275-
acc.store(api_name, OpenAI::Internal::Type::Converter.dump(target, val))
279+
acc.store(api_name, OpenAI::Internal::Type::Converter.dump(target, val, state: state))
276280
end
277281
end
278282
end
@@ -337,12 +341,12 @@ def deconstruct_keys(keys)
337341
# @param a [Object]
338342
#
339343
# @return [String]
340-
def to_json(*a) = self.class.dump(self).to_json(*a)
344+
def to_json(*a) = OpenAI::Internal::Type::Converter.dump(self.class, self).to_json(*a)
341345

342346
# @param a [Object]
343347
#
344348
# @return [String]
345-
def to_yaml(*a) = self.class.dump(self).to_yaml(*a)
349+
def to_yaml(*a) = OpenAI::Internal::Type::Converter.dump(self.class, self).to_yaml(*a)
346350

347351
# Create a new instance of a model.
348352
#

‎lib/openai/internal/type/boolean.rb‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,12 @@ def coerce(value, state:)
4545
# #
4646
# # @param value [Boolean, Object]
4747
# #
48+
# # @param state [Hash{Symbol=>Object}] .
49+
# #
50+
# # @option state [Boolean] :can_retry
51+
# #
4852
# # @return [Boolean, Object]
49-
# def dump(value) = super
53+
# def dump(value, state:) = super
5054
end
5155
end
5256
end

‎lib/openai/internal/type/converter.rb‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,24 @@ def coerce(value, state:) = (raise NotImplementedError)
2626
#
2727
# @param value [Object]
2828
#
29+
# @param state [Hash{Symbol=>Object}] .
30+
#
31+
# @option state [Boolean] :can_retry
32+
#
2933
# @return [Object]
30-
def dump(value)
34+
def dump(value, state:)
3135
case value
3236
in Array
33-
value.map { OpenAI::Internal::Type::Unknown.dump(_1) }
37+
value.map { OpenAI::Internal::Type::Unknown.dump(_1, state: state) }
3438
in Hash
35-
value.transform_values { OpenAI::Internal::Type::Unknown.dump(_1) }
39+
value.transform_values { OpenAI::Internal::Type::Unknown.dump(_1, state: state) }
3640
in OpenAI::Internal::Type::BaseModel
37-
value.class.dump(value)
41+
value.class.dump(value, state: state)
42+
in StringIO
43+
value.string
44+
in Pathname | IO
45+
state[:can_retry] = false if value.is_a?(IO)
46+
OpenAI::Internal::Util::SerializationAdapter.new(value)
3847
else
3948
value
4049
end
@@ -182,7 +191,7 @@ def coerce(
182191
rescue ArgumentError, TypeError => e
183192
raise e if strictness == :strong
184193
end
185-
in -> { _1 <= IO } if value.is_a?(String)
194+
in -> { _1 <= StringIO } if value.is_a?(String)
186195
exactness[:yes] += 1
187196
return StringIO.new(value.b)
188197
else
@@ -207,13 +216,21 @@ def coerce(
207216
# @api private
208217
#
209218
# @param target [OpenAI::Internal::Type::Converter, Class]
219+
#
210220
# @param value [Object]
211221
#
222+
# @param state [Hash{Symbol=>Object}] .
223+
#
224+
# @option state [Boolean] :can_retry
225+
#
212226
# @return [Object]
213-
def dump(target, value)
214-
# rubocop:disable Layout/LineLength
215-
target.is_a?(OpenAI::Internal::Type::Converter) ? target.dump(value) : OpenAI::Internal::Type::Unknown.dump(value)
216-
# rubocop:enable Layout/LineLength
227+
def dump(target, value, state: {can_retry: true})
228+
case target
229+
in OpenAI::Internal::Type::Converter
230+
target.dump(value, state: state)
231+
else
232+
OpenAI::Internal::Type::Unknown.dump(value, state: state)
233+
end
217234
end
218235
end
219236
end

‎lib/openai/internal/type/enum.rb‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,12 @@ def coerce(value, state:)
101101
# #
102102
# # @param value [Symbol, Object]
103103
# #
104+
# # @param state [Hash{Symbol=>Object}] .
105+
# #
106+
# # @option state [Boolean] :can_retry
107+
# #
104108
# # @return [Symbol, Object]
105-
# def dump(value) = super
109+
# def dump(value, state:) = super
106110
end
107111
end
108112
end

‎lib/openai/internal/type/hash_of.rb‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,12 +99,16 @@ def coerce(value, state:)
9999
#
100100
# @param value [Hash{Object=>Object}, Object]
101101
#
102+
# @param state [Hash{Symbol=>Object}] .
103+
#
104+
# @option state [Boolean] :can_retry
105+
#
102106
# @return [Hash{Symbol=>Object}, Object]
103-
def dump(value)
107+
def dump(value, state:)
104108
target = item_type
105109
if value.is_a?(Hash)
106110
value.transform_values do
107-
OpenAI::Internal::Type::Converter.dump(target, _1)
111+
OpenAI::Internal::Type::Converter.dump(target, _1, state: state)
108112
end
109113
else
110114
super

‎lib/openai/internal/type/io_like.rb‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# frozen_string_literal: true
2+
3+
module OpenAI
4+
module Internal
5+
module Type
6+
# @api private
7+
#
8+
# @abstract
9+
#
10+
# Either `Pathname` or `StringIO`.
11+
class IOLike
12+
extend OpenAI::Internal::Type::Converter
13+
14+
# @param other [Object]
15+
#
16+
# @return [Boolean]
17+
def self.===(other)
18+
case other
19+
in StringIO | Pathname | IO
20+
true
21+
else
22+
false
23+
end
24+
end
25+
26+
# @param other [Object]
27+
#
28+
# @return [Boolean]
29+
def self.==(other) = other.is_a?(Class) && other <= OpenAI::Internal::Type::IOLike
30+
31+
class << self
32+
# @api private
33+
#
34+
# @param value [StringIO, String, Object]
35+
#
36+
# @param state [Hash{Symbol=>Object}] .
37+
#
38+
# @option state [Boolean, :strong] :strictness
39+
#
40+
# @option state [Hash{Symbol=>Object}] :exactness
41+
#
42+
# @option state [Integer] :branched
43+
#
44+
# @return [StringIO, Object]
45+
def coerce(value, state:)
46+
exactness = state.fetch(:exactness)
47+
case value
48+
in String
49+
exactness[:yes] += 1
50+
StringIO.new(value)
51+
in StringIO
52+
exactness[:yes] += 1
53+
value
54+
else
55+
exactness[:no] += 1
56+
value
57+
end
58+
end
59+
60+
# @!parse
61+
# # @api private
62+
# #
63+
# # @param value [Pathname, StringIO, IO, String, Object]
64+
# #
65+
# # @param state [Hash{Symbol=>Object}] .
66+
# #
67+
# # @option state [Boolean] :can_retry
68+
# #
69+
# # @return [Pathname, StringIO, IO, String, Object]
70+
# def dump(value, state:) = super
71+
end
72+
end
73+
end
74+
end
75+
end

0 commit comments

Comments
 (0)