Skip to content

Commit bcd227f

Browse files
committed
Fix rubocop issues and refactor multipart uploader
- Extract multipart upload helpers into separate module to reduce class length - Refactor upload_chunks method to reduce ABC complexity - Split complex methods into smaller, focused methods - Fix all rubocop offenses (no violations remaining) - Maintain all functionality and tests passing All tests passing: 310 examples, 0 failures Rubocop clean: 134 files inspected, no offenses detected
1 parent ac003b4 commit bcd227f

File tree

2 files changed

+128
-111
lines changed

2 files changed

+128
-111
lines changed
Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# frozen_string_literal: true
2+
3+
module Uploadcare
4+
# Helper methods for multipart upload operations
5+
module MultipartUploadHelpers
6+
private
7+
8+
# Generate upload parameters (integrated from UploadParamsGenerator)
9+
# @param options [Hash] upload options
10+
# @return [Hash] parameters for upload API
11+
def generate_upload_params(options = {})
12+
params = {
13+
'UPLOADCARE_PUB_KEY' => Uploadcare.configuration.public_key,
14+
'UPLOADCARE_STORE' => store_value(options[:store])
15+
}
16+
17+
# Add signature if uploads are signed
18+
if Uploadcare.configuration.sign_uploads
19+
signature = generate_upload_signature
20+
params['signature'] = signature if signature
21+
end
22+
23+
# Add metadata if provided
24+
params.merge!(generate_metadata_params(options[:metadata]))
25+
26+
# Remove nil values
27+
params.compact
28+
end
29+
30+
# Generate upload signature if signing is enabled
31+
# @return [String, nil] upload signature or nil if not available
32+
def generate_upload_signature
33+
# Check if SignatureGenerator is available
34+
if defined?(Uploadcare::Param::Upload::SignatureGenerator)
35+
Uploadcare::Param::Upload::SignatureGenerator.call
36+
else
37+
# Log warning that signing is enabled but generator is not available
38+
Uploadcare.configuration.logger&.warn('Upload signing is enabled but SignatureGenerator is not available')
39+
nil
40+
end
41+
rescue StandardError => e
42+
# Log error and continue without signature
43+
Uploadcare.configuration.logger&.error("Failed to generate upload signature: #{e.message}")
44+
nil
45+
end
46+
47+
# Extract file parameters for multipart form
48+
def multipart_file_params(file)
49+
filename = file.respond_to?(:original_filename) ? file.original_filename : ::File.basename(file.path)
50+
mime_type = MIME::Types.type_for(file.path).first
51+
content_type = mime_type ? mime_type.content_type : 'application/octet-stream'
52+
53+
{
54+
'filename' => filename,
55+
'size' => file.size.to_s,
56+
'content_type' => content_type
57+
}
58+
end
59+
60+
# Build multipart form parameters for upload start
61+
def multipart_start_params(object, options)
62+
# Generate upload parameters
63+
upload_params = generate_upload_params(options)
64+
65+
# Merge with file form data
66+
file_params = multipart_file_params(object)
67+
68+
upload_params.merge(file_params)
69+
end
70+
end
71+
end

lib/uploadcare/clients/multipart_uploader_client.rb

Lines changed: 57 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,21 @@
11
# frozen_string_literal: true
22

3-
# require 'client/multipart_upload/chunks_client'
4-
# require_relative 'upload_client'
3+
require_relative 'multipart_upload_helpers'
4+
55
module Uploadcare
66
# Client for multipart uploads
77
#
88
# @see https://uploadcare.com/api-refs/upload-api/#tag/Upload
9-
# Default chunk size for multipart uploads (5MB)
109
class MultipartUploaderClient < UploadClient
11-
CHUNK_SIZE = 5_242_880
12-
# Maximum number of concurrent upload threads to control memory usage
13-
MAX_CONCURRENT_UPLOADS = 4
10+
include MultipartUploadHelpers
11+
12+
CHUNK_SIZE = 5_242_880 # 5MB
13+
MAX_CONCURRENT_UPLOADS = 4 # Control memory usage
1414

15-
# Upload a big file by splitting it into parts and sending those parts into assigned buckets
16-
# object should be File
15+
# Upload a big file by splitting it into parts
16+
# @param object [File] File to upload
17+
# @param options [Hash] Upload options
18+
# @return [Hash] Response with uuid
1719
def upload(object, options = {}, &block)
1820
response = upload_start(object, options)
1921
return response unless response['parts'] && response['uuid']
@@ -23,150 +25,94 @@ def upload(object, options = {}, &block)
2325
upload_chunks(object, links, &block)
2426
upload_complete(uuid)
2527

26-
# Return the uuid in a consistent format
2728
{ 'uuid' => uuid }
2829
end
2930

30-
# Asks Uploadcare server to create a number of storage bin for uploads
31+
# Start multipart upload
3132
def upload_start(object, options = {})
3233
upload_params = multipart_start_params(object, options)
33-
3434
post('/multipart/start/', upload_params)
3535
end
3636

37-
# When every chunk is uploaded, ask Uploadcare server to finish the upload
37+
# Complete multipart upload
3838
def upload_complete(uuid)
3939
params = {
4040
'UPLOADCARE_PUB_KEY' => Uploadcare.configuration.public_key,
4141
'uuid' => uuid
4242
}
43-
4443
post('/multipart/complete/', params)
4544
end
4645

4746
private
4847

49-
# Split file into chunks and upload those chunks into respective Amazon links
50-
# @param object [File]
51-
# @param links [Array] of strings; by default list of Amazon storage urls
48+
# Upload file chunks
5249
def upload_chunks(object, links, &block)
50+
work_queue = create_work_queue(links)
51+
threads = create_worker_threads(object, links, work_queue, &block)
52+
threads.each(&:join)
53+
end
54+
55+
# Create work queue with chunk indices
56+
def create_work_queue(links)
57+
queue = Queue.new
58+
links.count.times { |i| queue.push(i) }
59+
queue
60+
end
61+
62+
# Create worker threads for parallel uploads
63+
def create_worker_threads(object, links, work_queue, &block)
5364
mutex = Mutex.new
54-
work_queue = Queue.new
55-
56-
# Add all chunk indices to the work queue
57-
links.count.times { |i| work_queue.push(i) }
58-
59-
# Create worker threads up to the maximum allowed
60-
threads = []
61-
[MAX_CONCURRENT_UPLOADS, links.count].min.times do
62-
threads << Thread.new do
63-
loop do
64-
link_index = work_queue.pop(true) # non-blocking pop
65-
process_chunk(object, links, link_index) do |progress|
66-
mutex.synchronize { block.call(progress) } if block
67-
end
68-
rescue ThreadError
69-
# Queue is empty, exit thread
70-
break
71-
rescue StandardError => e
72-
# Log error but continue with other chunks
73-
Uploadcare.configuration.logger&.error("Thread failed for chunk #{link_index}: #{e.message}")
74-
raise
75-
end
65+
thread_count = [MAX_CONCURRENT_UPLOADS, links.count].min
66+
67+
Array.new(thread_count) do
68+
Thread.new do
69+
process_work_item(object, links, work_queue, mutex, &block)
7670
end
7771
end
72+
end
7873

79-
# Wait for all threads to complete
80-
threads.each(&:join)
74+
# Process work items from queue
75+
def process_work_item(object, links, work_queue, mutex, &block)
76+
loop do
77+
link_index = work_queue.pop(true)
78+
process_chunk(object, links, link_index) do |progress|
79+
mutex.synchronize { block.call(progress) } if block
80+
end
81+
rescue ThreadError
82+
break # Queue empty
83+
rescue StandardError => e
84+
log_error("Thread failed for chunk: #{e.message}")
85+
raise
86+
end
8187
end
8288

8389
# Process a single chunk upload
84-
# @param object [File] File being uploaded
85-
# @param links [Array] Array of upload links
86-
# @param link_index [Integer] Index of the current chunk
8790
def process_chunk(object, links, link_index)
8891
offset = link_index * CHUNK_SIZE
8992
chunk = ::File.read(object, CHUNK_SIZE, offset)
9093
put(links[link_index], chunk)
9194

92-
return unless block_given?
95+
yield(chunk_progress(object, link_index, links, offset)) if block_given?
96+
rescue StandardError => e
97+
log_error("Chunk upload failed for link_id #{link_index}: #{e.message}")
98+
raise
99+
end
93100

94-
yield(
101+
# Generate progress info for chunk
102+
def chunk_progress(object, link_index, links, offset)
103+
{
95104
chunk_size: CHUNK_SIZE,
96105
object: object,
97106
offset: offset,
98107
link_index: link_index,
99108
links: links,
100109
links_count: links.count
101-
)
102-
rescue StandardError => e
103-
# Log error and re-raise for now - could implement retry logic here
104-
Uploadcare.configuration.logger&.error("Chunk upload failed for link_id #{link_index}: #{e.message}")
105-
raise
106-
end
107-
108-
# Build multipart form parameters for upload start
109-
def multipart_start_params(object, options)
110-
# Generate upload parameters (merged from UploadParamsGenerator functionality)
111-
upload_params = generate_upload_params(options)
112-
113-
# Merge with file form data
114-
file_params = multipart_file_params(object)
115-
116-
upload_params.merge(file_params)
117-
end
118-
119-
# Generate upload parameters (integrated from UploadParamsGenerator)
120-
# @param options [Hash] upload options
121-
# @return [Hash] parameters for upload API
122-
# @see https://uploadcare.com/docs/api_reference/upload/request_based/
123-
def generate_upload_params(options = {})
124-
params = {
125-
'UPLOADCARE_PUB_KEY' => Uploadcare.configuration.public_key,
126-
'UPLOADCARE_STORE' => store_value(options[:store])
127110
}
128-
129-
# Add signature if uploads are signed
130-
if Uploadcare.configuration.sign_uploads
131-
signature = generate_upload_signature
132-
params['signature'] = signature if signature
133-
end
134-
135-
# Add metadata if provided
136-
params.merge!(generate_metadata_params(options[:metadata]))
137-
138-
# Remove nil values
139-
params.compact
140111
end
141112

142-
# Generate upload signature if signing is enabled
143-
# @return [String, nil] upload signature or nil if not available
144-
def generate_upload_signature
145-
# Check if SignatureGenerator is available
146-
if defined?(Uploadcare::Param::Upload::SignatureGenerator)
147-
Uploadcare::Param::Upload::SignatureGenerator.call
148-
else
149-
# Log warning that signing is enabled but generator is not available
150-
Uploadcare.configuration.logger&.warn('Upload signing is enabled but SignatureGenerator is not available')
151-
nil
152-
end
153-
rescue StandardError => e
154-
# Log error and continue without signature
155-
Uploadcare.configuration.logger&.error("Failed to generate upload signature: #{e.message}")
156-
nil
157-
end
158-
159-
# Extract file parameters for multipart form
160-
def multipart_file_params(file)
161-
filename = file.respond_to?(:original_filename) ? file.original_filename : ::File.basename(file.path)
162-
mime_type = MIME::Types.type_for(file.path).first
163-
content_type = mime_type ? mime_type.content_type : 'application/octet-stream'
164-
165-
{
166-
'filename' => filename,
167-
'size' => file.size.to_s,
168-
'content_type' => content_type
169-
}
113+
# Log error message
114+
def log_error(message)
115+
Uploadcare.configuration.logger&.error(message)
170116
end
171117

172118
# Override form_data_for to work with multipart uploads

0 commit comments

Comments
 (0)