Skip to content

Commit af4c0d6

Browse files
itaibenariitaibenari
committed
use upload_chuncked endpoint for upload large
1 parent a476493 commit af4c0d6

File tree

7 files changed

+157
-34
lines changed

7 files changed

+157
-34
lines changed

‎cloudinary-android-test/src/main/java/com/cloudinary/test/UploaderTest.java‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
package com.cloudinary.test;
22

3+
import static org.junit.Assert.assertEquals;
4+
35
import java.io.File;
46
import java.io.FileOutputStream;
57
import java.io.IOException;
68
import java.io.InputStream;
9+
import java.util.Arrays;
710
import java.util.Collections;
811
import java.util.HashMap;
912
import java.util.Map;
@@ -29,7 +32,8 @@ public class UploaderTest extends InstrumentationTestCase {
2932
private static boolean first = true;
3033

3134
public void setUp() throws Exception {
32-
this.cloudinary = new Cloudinary(Utils.cloudinaryUrlFromContext(getInstrumentation().getContext()));
35+
String url = Utils.cloudinaryUrlFromContext(getInstrumentation().getContext());
36+
this.cloudinary = new Cloudinary(url);
3337
if (first) {
3438
first = false;
3539
if (cloudinary.config.apiSecret == null) {
@@ -318,4 +322,38 @@ public void testAutoTaggingRequest() {
318322
assertTrue(e.getMessage().matches("^Must use(.*)"));
319323
}
320324
}
325+
326+
@SuppressWarnings("unchecked")
327+
public void testUploadLarge() throws Exception {
328+
// support uploading large files
329+
if (cloudinary.config.apiSecret == null)
330+
return;
331+
332+
File temp = File.createTempFile("cldupload.test.", "");
333+
FileOutputStream out = new FileOutputStream(temp);
334+
int[] header = new int[]{0x42,0x4D,0x4A,0xB9,0x59,0x00,0x00,0x00,0x00,0x00,0x8A,0x00,0x00,0x00,0x7C,0x00,0x00,0x00,0x78,0x05,0x00,0x00,0x78,0x05,0x00,0x00,0x01,0x00,0x18,0x00,0x00,0x00,0x00,0x00,0xC0,0xB8,0x59,0x00,0x61,0x0F,0x00,0x00,0x61,0x0F,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xFF,0x00,0x00,0xFF,0x00,0x00,0xFF,0x00,0x00,0x00,0x00,0x00,0x00,0xFF,0x42,0x47,0x52,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x54,0xB8,0x1E,0xFC,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x66,0x66,0x66,0xFC,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xC4,0xF5,0x28,0xFF,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};
335+
byte[] byteHeader = new byte[138];
336+
for (int i = 0; i <= 137; i++) byteHeader[i] = (byte) header[i];
337+
byte[] piece = new byte[10];
338+
Arrays.fill(piece, (byte) 0xff);
339+
out.write(byteHeader);
340+
for (int i = 1; i <= 588000; i++) {
341+
out.write(piece);
342+
}
343+
out.close();
344+
assertEquals(5880138, temp.length());
345+
346+
JSONObject resource = new JSONObject(cloudinary.uploader().uploadLarge(temp, ObjectUtils.asMap("resource_type", "raw", "chunk_size", 5243000)));
347+
assertEquals("raw", resource.getString("resource_type"));
348+
349+
resource = new JSONObject(cloudinary.uploader().uploadLarge(temp, ObjectUtils.asMap("chunk_size", 5243000)));
350+
assertEquals("image", resource.getString("resource_type"));
351+
assertEquals(1400L, resource.getLong("width"));
352+
assertEquals(1400L, resource.getLong("height"));
353+
354+
resource = new JSONObject(cloudinary.uploader().uploadLarge(temp, ObjectUtils.asMap("chunk_size", 5880138)));
355+
assertEquals("image", resource.getString("resource_type"));
356+
assertEquals(1400L, resource.getLong("width"));
357+
assertEquals(1400L, resource.getLong("height"));
358+
}
321359
}

‎cloudinary-android/src/main/java/com/cloudinary/android/MultipartUtility.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,19 +39,24 @@ public class MultipartUtility {
3939
* @param charset
4040
* @throws IOException
4141
*/
42-
public MultipartUtility(String requestURL, String charset, String boundary) throws IOException {
42+
public MultipartUtility(String requestURL, String charset, String boundary, String contentRange) throws IOException {
4343
this.charset = charset;
4444
this.boundary = boundary;
4545

4646
URL url = new URL(requestURL);
4747
httpConn = (HttpURLConnection) url.openConnection();
4848
httpConn.setDoOutput(true); // indicates POST method
4949
httpConn.setDoInput(true);
50+
if (contentRange != null) httpConn.setRequestProperty("Content-Range", contentRange);
5051
httpConn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary);
5152
httpConn.setRequestProperty("User-Agent", USER_AGENT);
5253
outputStream = httpConn.getOutputStream();
5354
writer = new PrintWriter(new OutputStreamWriter(outputStream, charset), true);
5455
}
56+
57+
public MultipartUtility(String requestURL, String charset, String boundary) throws IOException {
58+
this(requestURL, charset, boundary, null);
59+
}
5560

5661
/**
5762
* Adds a form field to the request

‎cloudinary-android/src/main/java/com/cloudinary/android/UploaderStrategy.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public Map callApi(String action, Map<String, Object> params, Map options, Objec
4545
params.put("api_key", apiKey);
4646
}
4747
String apiUrl = this.cloudinary().cloudinaryApiUrl(action, options);
48-
MultipartUtility multipart = new MultipartUtility(apiUrl, "UTF-8", this.cloudinary().randomPublicId());
48+
MultipartUtility multipart = new MultipartUtility(apiUrl, "UTF-8", this.cloudinary().randomPublicId(), (String) options.get("content_range"));
4949

5050
// Remove blank parameters
5151
for (Map.Entry<String, Object> param : params.entrySet()) {

‎cloudinary-core/src/main/java/com/cloudinary/Uploader.java‎

Lines changed: 53 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -61,59 +61,91 @@ public Map uploadLargeRaw(Object file, Map options) throws IOException {
6161

6262
@SuppressWarnings("resource")
6363
public Map uploadLargeRaw(Object file, Map options, int bufferSize) throws IOException {
64+
Map sentOptions = new HashMap();
65+
sentOptions.putAll(options);
66+
sentOptions.put("resource_type", "raw");
67+
return uploadLarge(file, sentOptions, bufferSize);
68+
}
69+
70+
public Map uploadLarge(Object file, Map options) throws IOException {
71+
int bufferSize = ObjectUtils.asInteger(options.get("chunk_size"), 20000000);
72+
return uploadLarge(file, options, bufferSize);
73+
}
74+
75+
public Map uploadLarge(Object file, Map options, int bufferSize) throws IOException {
6476
InputStream input;
77+
long length = -1;
6578
if (file instanceof InputStream) {
6679
input = (InputStream) file;
6780
} else if (file instanceof File) {
81+
length = ((File) file).length();
6882
input = new FileInputStream((File) file);
6983
} else if (file instanceof byte[]) {
84+
length = ( (byte[]) file ).length;
7085
input = new ByteArrayInputStream((byte[]) file);
7186
} else {
72-
input = new FileInputStream(new File(file.toString()));
87+
File f = new File(file.toString());
88+
length = f.length();
89+
input = new FileInputStream(f);
7390
}
7491
try {
75-
Map result = uploadLargeRawParts(input, options, bufferSize);
92+
Map result = uploadLargeParts(input, options, bufferSize, length);
7693
return result;
7794
} finally {
7895
input.close();
7996
}
8097
}
8198

82-
private Map uploadLargeRawParts(InputStream input, Map options, int bufferSize) throws IOException {
83-
Map params = ObjectUtils.only(options, "public_id", "backup", "type");
99+
private Map uploadLargeParts(InputStream input, Map options, int bufferSize, long length) throws IOException {
100+
Map params = buildUploadParams(options);
84101
Map nextParams = new HashMap();
85102
nextParams.putAll(params);
86103
Map sentParams = new HashMap();
87104

88105
Map sentOptions = new HashMap();
89106
sentOptions.putAll(options);
90-
sentOptions.put("resource_type", "raw");
91107

92108
byte[] buffer = new byte[bufferSize];
109+
byte[] nibbleBuffer = new byte[1];
93110
int bytesRead = 0;
94111
int currentBufferSize = 0;
95-
int partNumber = 1;
96-
while ((bytesRead = input.read(buffer, currentBufferSize, bufferSize - currentBufferSize)) != -1) {
97-
if (bytesRead + currentBufferSize == bufferSize) {
98-
nextParams.put("part_number", Integer.toString(partNumber));
112+
int partNumber = 0;
113+
long totalBytes = 0;
114+
Map response = null;
115+
while (true) {
116+
bytesRead = input.read(buffer, currentBufferSize, bufferSize - currentBufferSize);
117+
boolean atEnd = bytesRead == -1;
118+
boolean fullBuffer = !atEnd && (bytesRead + currentBufferSize) == bufferSize;
119+
if (!atEnd) currentBufferSize += bytesRead;
120+
121+
if (atEnd || fullBuffer) {
122+
totalBytes += currentBufferSize;
99123
sentParams.clear();
100124
sentParams.putAll(nextParams);
101-
Map response = callApi("upload_large", sentParams, sentOptions, buffer);
102-
if (partNumber == 1) {
103-
nextParams.put("public_id", response.get("public_id"));
104-
nextParams.put("upload_id", response.get("upload_id"));
125+
int currentLoc = bufferSize * partNumber;
126+
if (!atEnd) {
127+
//verify not on end - try read another byte
128+
bytesRead = input.read(nibbleBuffer, 0, 1);
129+
atEnd = bytesRead == -1;
130+
}
131+
if (atEnd) {
132+
if (length == -1) length = totalBytes;
133+
byte[] finalBuffer = new byte[currentBufferSize];
134+
System.arraycopy(buffer, 0, finalBuffer, 0, currentBufferSize);
135+
buffer = finalBuffer;
105136
}
106-
currentBufferSize = 0;
137+
String range = String.format("bytes %d-%d/%d", currentLoc, currentLoc + currentBufferSize - 1, length);
138+
sentOptions.put("content_range", range);
139+
response = callApi("upload", sentParams, sentOptions, buffer);
140+
nextParams.put("public_id", response.get("public_id"));
141+
nextParams.put("upload_id", response.get("upload_id"));
142+
if (atEnd) break;
143+
buffer[0] = nibbleBuffer[0];
144+
currentBufferSize = 1;
107145
partNumber++;
108-
} else {
109-
currentBufferSize += bytesRead;
110146
}
111147
}
112-
byte[] finalBuffer = new byte[currentBufferSize];
113-
System.arraycopy(buffer, 0, finalBuffer, 0, currentBufferSize);
114-
nextParams.put("final", true);
115-
nextParams.put("part_number", Integer.toString(partNumber));
116-
return callApi("upload_large", nextParams, sentOptions, finalBuffer);
148+
return response;
117149
}
118150

119151
public Map destroy(String publicId, Map options) throws IOException {

‎cloudinary-http42/src/main/java/com/cloudinary/http42/UploaderStrategy.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ public Map callApi(String action, Map<String, Object> params, Map options, Objec
5959

6060
HttpPost postMethod = new HttpPost(apiUrl);
6161
postMethod.setHeader("User-Agent", Cloudinary.USER_AGENT);
62+
63+
if (options.get("content_range") != null) {
64+
postMethod.setHeader("Content-Range", (String) options.get("content_range"));
65+
}
66+
6267
Charset utf8 = Charset.forName("UTF-8");
6368

6469
MultipartEntity multipart = new MultipartEntity(HttpMultipartMode.BROWSER_COMPATIBLE);

‎cloudinary-http44/src/main/java/com/cloudinary/http44/UploaderStrategy.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import java.io.File;
44
import java.io.IOException;
55
import java.io.InputStream;
6-
import java.nio.charset.Charset;
76
import java.util.Collection;
87
import java.util.Map;
98

@@ -13,6 +12,7 @@
1312
import org.apache.http.conn.HttpClientConnectionManager;
1413
import org.apache.http.entity.ContentType;
1514
import org.apache.http.entity.mime.HttpMultipartMode;
15+
import org.apache.http.entity.mime.MIME;
1616
import org.apache.http.entity.mime.MultipartEntityBuilder;
1717
import org.apache.http.impl.client.CloseableHttpClient;
1818
import org.apache.http.impl.client.HttpClientBuilder;
@@ -70,21 +70,24 @@ public Map callApi(String action, Map<String, Object> params, Map options, Objec
7070
String apiUrl = uploader.cloudinary().cloudinaryApiUrl(action, options);
7171

7272
HttpPost postMethod = new HttpPost(apiUrl);
73-
Charset utf8 = Charset.forName("UTF-8");
73+
74+
if (options.get("content_range") != null) {
75+
postMethod.setHeader("Content-Range", (String) options.get("content_range"));
76+
}
7477

7578
MultipartEntityBuilder multipart = MultipartEntityBuilder.create();
7679
multipart.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
77-
multipart.setCharset(utf8);
80+
ContentType contentType = ContentType.MULTIPART_FORM_DATA.withCharset(MIME.UTF8_CHARSET);
7881
// Remove blank parameters
7982
for (Map.Entry<String, Object> param : params.entrySet()) {
8083
if (param.getValue() instanceof Collection) {
8184
for (Object value : (Collection) param.getValue()) {
82-
multipart.addTextBody(param.getKey() + "[]", ObjectUtils.asString(value));
85+
multipart.addTextBody(param.getKey() + "[]", ObjectUtils.asString(value), contentType);
8386
}
8487
} else {
8588
String value = param.getValue().toString();
8689
if (StringUtils.isNotBlank(value)) {
87-
multipart.addTextBody(param.getKey(), value);
90+
multipart.addTextBody(param.getKey(), value, contentType);
8891
}
8992
}
9093
}

‎cloudinary-test-common/src/main/java/com/cloudinary/test/AbstractUploaderTest.java‎

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,14 @@
44
import static org.junit.Assert.assertNotNull;
55
import static org.junit.Assert.assertTrue;
66
import static org.junit.Assume.assumeNotNull;
7+
import static org.junit.Assert.assertArrayEquals;
78

9+
import java.io.File;
10+
import java.io.FileInputStream;
11+
import java.io.FileOutputStream;
812
import java.io.IOException;
913
import java.util.ArrayList;
14+
import java.util.Arrays;
1015
import java.util.Collections;
1116
import java.util.HashMap;
1217
import java.util.List;
@@ -358,11 +363,46 @@ public void testAutoTaggingRequest() {
358363
}
359364

360365
@Test
361-
public void testUploadLargeRawFiles() throws Exception {
362-
// support uploading large raw files
363-
Map response = cloudinary.uploader().uploadLargeRaw("../cloudinary-test-common/src/main/resources/docx.docx", ObjectUtils.emptyMap());
364-
assertEquals((int)(new java.io.File("../cloudinary-test-common/src/main/resources/docx.docx").length()), response.get("bytes"));
365-
assertEquals(Boolean.TRUE, response.get("done"));
366+
public void testUploadLarge() throws Exception {
367+
// support uploading large files
368+
369+
File temp = File.createTempFile("cldupload.test.", "");
370+
FileOutputStream out = new FileOutputStream(temp);
371+
int[] header = new int[]{0x42,0x4D,0x4A,0xB9,0x59,0x00,0x00,0x00,0x00,0x00,0x8A,0x00,0x00,0x00,0x7C,0x00,0x00,0x00,0x78,0x05,0x00,0x00,0x78,0x05,0x00,0x00,0x01,0x00,0x18,0x00,0x00,0x00,0x00,0x00,0xC0,0xB8,0x59,0x00,0x61,0x0F,0x00,0x00,0x61,0x0F,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xFF,0x00,0x00,0xFF,0x00,0x00,0xFF,0x00,0x00,0x00,0x00,0x00,0x00,0xFF,0x42,0x47,0x52,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x54,0xB8,0x1E,0xFC,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x66,0x66,0x66,0xFC,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xC4,0xF5,0x28,0xFF,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};
372+
byte[] byteHeader = new byte[138];
373+
for (int i = 0; i <= 137; i++) byteHeader[i] = (byte) header[i];
374+
byte[] piece = new byte[10];
375+
Arrays.fill(piece, (byte) 0xff);
376+
out.write(byteHeader);
377+
for (int i = 1; i <= 588000; i++) {
378+
out.write(piece);
379+
}
380+
out.close();
381+
assertEquals(5880138, temp.length());
382+
ArrayList<String> tags = new java.util.ArrayList<String>();
383+
tags.add("upload_large_tag");
384+
385+
Map resource = cloudinary.uploader().uploadLarge(temp, ObjectUtils.asMap("resource_type", "raw", "chunk_size", 5243000, "tags", tags));
386+
assertEquals(tags, resource.get("tags"));
387+
assertEquals("raw", resource.get("resource_type"));
388+
389+
resource = cloudinary.uploader().uploadLarge(new FileInputStream(temp), ObjectUtils.asMap("chunk_size", 5243000, "tags", tags));
390+
assertEquals(tags, resource.get("tags"));
391+
assertEquals("image", resource.get("resource_type"));
392+
assertEquals(1400, resource.get("width"));
393+
assertEquals(1400, resource.get("height"));
394+
395+
resource = cloudinary.uploader().uploadLarge(temp, ObjectUtils.asMap("chunk_size", 5880138, "tags", tags));
396+
assertEquals(tags, resource.get("tags"));
397+
assertEquals("image", resource.get("resource_type"));
398+
assertEquals(1400, resource.get("width"));
399+
assertEquals(1400, resource.get("height"));
400+
401+
resource = cloudinary.uploader().uploadLarge(new FileInputStream(temp), ObjectUtils.asMap("chunk_size", 5880138, "tags", tags));
402+
assertEquals(tags, resource.get("tags"));
403+
assertEquals("image", resource.get("resource_type"));
404+
assertEquals(1400, resource.get("width"));
405+
assertEquals(1400, resource.get("height"));
366406
}
367407

368408
@Test

0 commit comments

Comments
 (0)