11# frozen_string_literal: true
22
3- # require 'client/multipart_upload/chunks_client '
4- # require_relative 'upload_client'
3+ require_relative 'multipart_upload_helpers '
4+
55module Uploadcare
66 # Client for multipart uploads
77 #
88 # @see https://uploadcare.com/api-refs/upload-api/#tag/Upload
9- # Default chunk size for multipart uploads (5MB)
109 class MultipartUploaderClient < UploadClient
11- CHUNK_SIZE = 5_242_880
12- # Maximum number of concurrent upload threads to control memory usage
13- MAX_CONCURRENT_UPLOADS = 4
10+ include MultipartUploadHelpers
11+
12+ CHUNK_SIZE = 5_242_880 # 5MB
13+ MAX_CONCURRENT_UPLOADS = 4 # Control memory usage
1414
15- # Upload a big file by splitting it into parts and sending those parts into assigned buckets
16- # object should be File
15+ # Upload a big file by splitting it into parts
16+ # @param object [File] File to upload
17+ # @param options [Hash] Upload options
18+ # @return [Hash] Response with uuid
1719 def upload ( object , options = { } , &block )
1820 response = upload_start ( object , options )
1921 return response unless response [ 'parts' ] && response [ 'uuid' ]
@@ -23,150 +25,94 @@ def upload(object, options = {}, &block)
2325 upload_chunks ( object , links , &block )
2426 upload_complete ( uuid )
2527
26- # Return the uuid in a consistent format
2728 { 'uuid' => uuid }
2829 end
2930
30- # Asks Uploadcare server to create a number of storage bin for uploads
31+ # Start multipart upload
3132 def upload_start ( object , options = { } )
3233 upload_params = multipart_start_params ( object , options )
33-
3434 post ( '/multipart/start/' , upload_params )
3535 end
3636
37- # When every chunk is uploaded, ask Uploadcare server to finish the upload
37+ # Complete multipart upload
3838 def upload_complete ( uuid )
3939 params = {
4040 'UPLOADCARE_PUB_KEY' => Uploadcare . configuration . public_key ,
4141 'uuid' => uuid
4242 }
43-
4443 post ( '/multipart/complete/' , params )
4544 end
4645
4746 private
4847
49- # Split file into chunks and upload those chunks into respective Amazon links
50- # @param object [File]
51- # @param links [Array] of strings; by default list of Amazon storage urls
48+ # Upload file chunks
5249 def upload_chunks ( object , links , &block )
50+ work_queue = create_work_queue ( links )
51+ threads = create_worker_threads ( object , links , work_queue , &block )
52+ threads . each ( &:join )
53+ end
54+
55+ # Create work queue with chunk indices
56+ def create_work_queue ( links )
57+ queue = Queue . new
58+ links . count . times { |i | queue . push ( i ) }
59+ queue
60+ end
61+
62+ # Create worker threads for parallel uploads
63+ def create_worker_threads ( object , links , work_queue , &block )
5364 mutex = Mutex . new
54- work_queue = Queue . new
55-
56- # Add all chunk indices to the work queue
57- links . count . times { |i | work_queue . push ( i ) }
58-
59- # Create worker threads up to the maximum allowed
60- threads = [ ]
61- [ MAX_CONCURRENT_UPLOADS , links . count ] . min . times do
62- threads << Thread . new do
63- loop do
64- link_index = work_queue . pop ( true ) # non-blocking pop
65- process_chunk ( object , links , link_index ) do |progress |
66- mutex . synchronize { block . call ( progress ) } if block
67- end
68- rescue ThreadError
69- # Queue is empty, exit thread
70- break
71- rescue StandardError => e
72- # Log error but continue with other chunks
73- Uploadcare . configuration . logger &.error ( "Thread failed for chunk #{ link_index } : #{ e . message } " )
74- raise
75- end
65+ thread_count = [ MAX_CONCURRENT_UPLOADS , links . count ] . min
66+
67+ Array . new ( thread_count ) do
68+ Thread . new do
69+ process_work_item ( object , links , work_queue , mutex , &block )
7670 end
7771 end
72+ end
7873
79- # Wait for all threads to complete
80- threads . each ( &:join )
74+ # Process work items from queue
75+ def process_work_item ( object , links , work_queue , mutex , &block )
76+ loop do
77+ link_index = work_queue . pop ( true )
78+ process_chunk ( object , links , link_index ) do |progress |
79+ mutex . synchronize { block . call ( progress ) } if block
80+ end
81+ rescue ThreadError
82+ break # Queue empty
83+ rescue StandardError => e
84+ log_error ( "Thread failed for chunk: #{ e . message } " )
85+ raise
86+ end
8187 end
8288
8389 # Process a single chunk upload
84- # @param object [File] File being uploaded
85- # @param links [Array] Array of upload links
86- # @param link_index [Integer] Index of the current chunk
8790 def process_chunk ( object , links , link_index )
8891 offset = link_index * CHUNK_SIZE
8992 chunk = ::File . read ( object , CHUNK_SIZE , offset )
9093 put ( links [ link_index ] , chunk )
9194
92- return unless block_given?
95+ yield ( chunk_progress ( object , link_index , links , offset ) ) if block_given?
96+ rescue StandardError => e
97+ log_error ( "Chunk upload failed for link_id #{ link_index } : #{ e . message } " )
98+ raise
99+ end
93100
94- yield (
101+ # Generate progress info for chunk
102+ def chunk_progress ( object , link_index , links , offset )
103+ {
95104 chunk_size : CHUNK_SIZE ,
96105 object : object ,
97106 offset : offset ,
98107 link_index : link_index ,
99108 links : links ,
100109 links_count : links . count
101- )
102- rescue StandardError => e
103- # Log error and re-raise for now - could implement retry logic here
104- Uploadcare . configuration . logger &.error ( "Chunk upload failed for link_id #{ link_index } : #{ e . message } " )
105- raise
106- end
107-
108- # Build multipart form parameters for upload start
109- def multipart_start_params ( object , options )
110- # Generate upload parameters (merged from UploadParamsGenerator functionality)
111- upload_params = generate_upload_params ( options )
112-
113- # Merge with file form data
114- file_params = multipart_file_params ( object )
115-
116- upload_params . merge ( file_params )
117- end
118-
119- # Generate upload parameters (integrated from UploadParamsGenerator)
120- # @param options [Hash] upload options
121- # @return [Hash] parameters for upload API
122- # @see https://uploadcare.com/docs/api_reference/upload/request_based/
123- def generate_upload_params ( options = { } )
124- params = {
125- 'UPLOADCARE_PUB_KEY' => Uploadcare . configuration . public_key ,
126- 'UPLOADCARE_STORE' => store_value ( options [ :store ] )
127110 }
128-
129- # Add signature if uploads are signed
130- if Uploadcare . configuration . sign_uploads
131- signature = generate_upload_signature
132- params [ 'signature' ] = signature if signature
133- end
134-
135- # Add metadata if provided
136- params . merge! ( generate_metadata_params ( options [ :metadata ] ) )
137-
138- # Remove nil values
139- params . compact
140111 end
141112
142- # Generate upload signature if signing is enabled
143- # @return [String, nil] upload signature or nil if not available
144- def generate_upload_signature
145- # Check if SignatureGenerator is available
146- if defined? ( Uploadcare ::Param ::Upload ::SignatureGenerator )
147- Uploadcare ::Param ::Upload ::SignatureGenerator . call
148- else
149- # Log warning that signing is enabled but generator is not available
150- Uploadcare . configuration . logger &.warn ( 'Upload signing is enabled but SignatureGenerator is not available' )
151- nil
152- end
153- rescue StandardError => e
154- # Log error and continue without signature
155- Uploadcare . configuration . logger &.error ( "Failed to generate upload signature: #{ e . message } " )
156- nil
157- end
158-
159- # Extract file parameters for multipart form
160- def multipart_file_params ( file )
161- filename = file . respond_to? ( :original_filename ) ? file . original_filename : ::File . basename ( file . path )
162- mime_type = MIME ::Types . type_for ( file . path ) . first
163- content_type = mime_type ? mime_type . content_type : 'application/octet-stream'
164-
165- {
166- 'filename' => filename ,
167- 'size' => file . size . to_s ,
168- 'content_type' => content_type
169- }
113+ # Log error message
114+ def log_error ( message )
115+ Uploadcare . configuration . logger &.error ( message )
170116 end
171117
172118 # Override form_data_for to work with multipart uploads
0 commit comments