Skip to content

Commit 1681c41

Browse files
yury-syury-s
authored
feat: large file uploads (#891)
1 parent 9f68605 commit 1681c41

File tree

12 files changed

+304
-12
lines changed

12 files changed

+304
-12
lines changed

‎playwright/src/main/java/com/microsoft/playwright/impl/BrowserContextImpl.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -560,4 +560,11 @@ void didClose() {
560560
}
561561
listeners.notify(EventType.CLOSE, this);
562562
}
563+
564+
WritableStream createTempFile(String name) {
565+
JsonObject params = new JsonObject();
566+
params.addProperty("name", name);
567+
JsonObject json = sendMessage("createTempFile", params).getAsJsonObject();
568+
return connection.getExistingObject(json.getAsJsonObject("writableStream").get("guid").getAsString());
569+
}
563570
}

‎playwright/src/main/java/com/microsoft/playwright/impl/Connection.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,9 @@ private ChannelOwner createRemoteObject(String parentGuid, JsonObject params) {
302302
case "Worker":
303303
result = new WorkerImpl(parent, type, guid, initializer);
304304
break;
305+
case "WritableStream":
306+
result = new WritableStream(parent, type, guid, initializer);
307+
break;
305308
default:
306309
throw new PlaywrightException("Unknown type " + type);
307310
}

‎playwright/src/main/java/com/microsoft/playwright/impl/ElementHandleImpl.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@
3333
import java.util.List;
3434

3535
import static com.microsoft.playwright.impl.Serialization.*;
36-
import static com.microsoft.playwright.impl.Utils.convertType;
36+
import static com.microsoft.playwright.impl.Utils.*;
37+
import static com.microsoft.playwright.impl.Utils.addLargeFileUploadParams;
3738
import static com.microsoft.playwright.options.ScreenshotType.JPEG;
3839
import static com.microsoft.playwright.options.ScreenshotType.PNG;
3940

@@ -299,7 +300,7 @@ public boolean isVisible() {
299300
}
300301

301302
@Override
302-
public Frame ownerFrame() {
303+
public FrameImpl ownerFrame() {
303304
return withLogging("ElementHandle.ownerFrame", () -> {
304305
JsonObject json = sendMessage("ownerFrame").getAsJsonObject();
305306
if (!json.has("frame")) {
@@ -455,7 +456,24 @@ private void selectTextImpl(SelectTextOptions options) {
455456

456457
@Override
457458
public void setInputFiles(Path[] files, SetInputFilesOptions options) {
458-
setInputFiles(Utils.toFilePayloads(files), options);
459+
withLogging("ElementHandle.setInputFiles", () -> setInputFilesImpl(files, options));
460+
}
461+
462+
void setInputFilesImpl(Path[] files, SetInputFilesOptions options) {
463+
FrameImpl frame = ownerFrame();
464+
if (frame == null) {
465+
throw new Error("Cannot set input files to detached element");
466+
}
467+
if (hasLargeFile(files)) {
468+
if (options == null) {
469+
options = new SetInputFilesOptions();
470+
}
471+
JsonObject params = gson().toJsonTree(options).getAsJsonObject();
472+
addLargeFileUploadParams(files, params, frame.page().context());
473+
sendMessage("setInputFilePaths", params);
474+
} else {
475+
setInputFilesImpl(Utils.toFilePayloads(files), options);
476+
}
459477
}
460478

461479
@Override
@@ -469,6 +487,7 @@ public void setInputFiles(FilePayload[] files, SetInputFilesOptions options) {
469487
}
470488

471489
void setInputFilesImpl(FilePayload[] files, SetInputFilesOptions options) {
490+
checkFilePayloadSize(files);
472491
if (options == null) {
473492
options = new SetInputFilesOptions();
474493
}

‎playwright/src/main/java/com/microsoft/playwright/impl/FileChooserImpl.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ public void setFiles(Path files, SetFilesOptions options) {
5858

5959
@Override
6060
public void setFiles(Path[] files, SetFilesOptions options) {
61-
setFiles(Utils.toFilePayloads(files), options);
61+
page.withLogging("FileChooser.setInputFiles",
62+
() -> element.setInputFilesImpl(files, convertType(options, ElementHandle.SetInputFilesOptions.class)));
6263
}
6364

6465
@Override
@@ -69,6 +70,6 @@ public void setFiles(FilePayload files, SetFilesOptions options) {
6970
@Override
7071
public void setFiles(FilePayload[] files, SetFilesOptions options) {
7172
page.withLogging("FileChooser.setInputFiles",
72-
() -> element.setInputFilesImpl(files, convertType(options, ElementHandle.SetInputFilesOptions.class)));
73+
() -> element.setInputFilesImpl(files, convertType(options, ElementHandle.SetInputFilesOptions.class)));
7374
}
7475
}

‎playwright/src/main/java/com/microsoft/playwright/impl/FrameImpl.java‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.microsoft.playwright.options.*;
2424

2525
import java.io.IOException;
26+
import java.io.OutputStream;
2627
import java.nio.charset.StandardCharsets;
2728
import java.nio.file.Files;
2829
import java.nio.file.Path;
@@ -31,7 +32,7 @@
3132
import java.util.function.Predicate;
3233
import java.util.regex.Pattern;
3334

34-
import static com.microsoft.playwright.impl.Utils.convertType;
35+
import static com.microsoft.playwright.impl.Utils.*;
3536
import static com.microsoft.playwright.options.WaitUntilState.*;
3637
import static com.microsoft.playwright.impl.Serialization.*;
3738

@@ -686,21 +687,32 @@ public void setInputFiles(String selector, Path[] files, SetInputFilesOptions op
686687
withLogging("Frame.setInputFiles", () -> setInputFilesImpl(selector, files, options));
687688
}
688689

690+
void setInputFilesImpl(String selector, Path[] files, SetInputFilesOptions options) {
691+
if (hasLargeFile(files)) {
692+
if (options == null) {
693+
options = new SetInputFilesOptions();
694+
}
695+
JsonObject params = gson().toJsonTree(options).getAsJsonObject();
696+
addLargeFileUploadParams(files, params, page.context());
697+
params.addProperty("selector", selector);
698+
sendMessage("setInputFilePaths", params);
699+
} else {
700+
setInputFilesImpl(selector, Utils.toFilePayloads(files), options);
701+
}
702+
}
703+
689704
@Override
690705
public void setInputFiles(String selector, FilePayload files, SetInputFilesOptions options) {
691706
setInputFiles(selector, new FilePayload[]{files}, options);
692707
}
693708

694-
void setInputFilesImpl(String selector, Path[] files, SetInputFilesOptions options) {
695-
setInputFiles(selector, Utils.toFilePayloads(files), options);
696-
}
697-
698709
@Override
699710
public void setInputFiles(String selector, FilePayload[] files, SetInputFilesOptions options) {
700711
withLogging("Frame.setInputFiles", () -> setInputFilesImpl(selector, files, options));
701712
}
702713

703714
void setInputFilesImpl(String selector, FilePayload[] files, SetInputFilesOptions options) {
715+
checkFilePayloadSize(files);
704716
if (options == null) {
705717
options = new SetInputFilesOptions();
706718
}

‎playwright/src/main/java/com/microsoft/playwright/impl/Serialization.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,14 @@ public JsonArray serialize(List<KeyboardModifier> modifiers, Type typeOfSrc, Jso
212212
}
213213
}
214214

215+
static JsonArray toJsonArray(Path[] files) {
216+
JsonArray jsonFiles = new JsonArray();
217+
for (Path p : files) {
218+
jsonFiles.add(p.toString());
219+
}
220+
return jsonFiles;
221+
}
222+
215223
static JsonArray toJsonArray(FilePayload[] files) {
216224
JsonArray jsonFiles = new JsonArray();
217225
for (FilePayload p : files) {

‎playwright/src/main/java/com/microsoft/playwright/impl/Utils.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,25 @@
1616

1717
package com.microsoft.playwright.impl;
1818

19+
import com.google.gson.JsonArray;
20+
import com.google.gson.JsonObject;
1921
import com.microsoft.playwright.PlaywrightException;
2022
import com.microsoft.playwright.options.FilePayload;
2123
import com.microsoft.playwright.options.HttpHeader;
2224

2325
import java.io.FileOutputStream;
2426
import java.io.IOException;
2527
import java.io.InputStream;
28+
import java.io.OutputStream;
2629
import java.lang.reflect.Field;
2730
import java.lang.reflect.Modifier;
2831
import java.nio.file.Files;
2932
import java.nio.file.Path;
3033
import java.util.*;
3134
import java.util.regex.Pattern;
3235

36+
import static com.microsoft.playwright.impl.Serialization.toJsonArray;
37+
3338
class Utils {
3439
static <F, T> T convertType(F f, Class<T> t) {
3540
if (f == null) {
@@ -144,6 +149,49 @@ static String mimeType(Path path) {
144149
return mimeType;
145150
}
146151

152+
static final int maxUplodBufferSize = 50 * 1024 * 1024;
153+
154+
static boolean hasLargeFile(Path[] files) {
155+
for (Path file: files) {
156+
try {
157+
if (Files.size(file)> maxUplodBufferSize) {
158+
return true;
159+
}
160+
} catch (IOException e) {
161+
throw new PlaywrightException("Cannot get file size.", e);
162+
}
163+
}
164+
return false;
165+
}
166+
167+
static void addLargeFileUploadParams(Path[] files, JsonObject params, BrowserContextImpl context) {
168+
if (context.browser().isRemote) {
169+
List<WritableStream> streams = new ArrayList<>();
170+
JsonArray jsonStreams = new JsonArray();
171+
for (Path path : files) {
172+
WritableStream temp = context.createTempFile(path.getFileName().toString());
173+
streams.add(temp);
174+
try (OutputStream out = temp.stream()) {
175+
Files.copy(path, out);
176+
} catch (IOException e) {
177+
throw new PlaywrightException("Failed to copy file to remote server.", e);
178+
}
179+
jsonStreams.add(temp.toProtocol());
180+
}
181+
params.add("streams", jsonStreams);
182+
} else {
183+
params.add("localPaths", toJsonArray(files));
184+
}
185+
}
186+
187+
static void checkFilePayloadSize(FilePayload[] files) {
188+
for (FilePayload file: files) {
189+
if (file.buffer.length > maxUplodBufferSize) {
190+
throw new PlaywrightException("Cannot set buffer larger than 50Mb, please write it to a file and pass its path instead.");
191+
}
192+
}
193+
}
194+
147195
static FilePayload[] toFilePayloads(Path[] files) {
148196
List<FilePayload> payloads = new ArrayList<>();
149197
for (Path file : files) {

‎playwright/src/main/java/com/microsoft/playwright/impl/WritableStream.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package com.microsoft.playwright.impl;
2+
3+
import com.google.gson.JsonObject;
4+
5+
import java.io.IOException;
6+
import java.io.OutputStream;
7+
import java.nio.ByteBuffer;
8+
import java.nio.charset.StandardCharsets;
9+
import java.util.Base64;
10+
11+
class WritableStream extends ChannelOwner {
12+
WritableStream(ChannelOwner parent, String type, String guid, JsonObject initializer) {
13+
super(parent, type, guid, initializer);
14+
}
15+
16+
OutputStream stream() {
17+
return new OutputStream() {
18+
@Override
19+
public void write(int b) throws IOException {
20+
write(new byte[] { (byte) b });
21+
}
22+
23+
@Override
24+
public void write(byte[] b, int off, int len) throws IOException {
25+
JsonObject params = new JsonObject();
26+
ByteBuffer buffer = ByteBuffer.wrap(b, off, len);
27+
ByteBuffer encoded = Base64.getEncoder().encode(buffer);
28+
params.addProperty("binary", new String(encoded.array(), StandardCharsets.UTF_8));
29+
sendMessage("write", params);
30+
}
31+
};
32+
}
33+
34+
JsonObject toProtocol() {
35+
JsonObject json = new JsonObject();
36+
json.addProperty("guid", guid);
37+
return json;
38+
}
39+
}

‎playwright/src/test/java/com/microsoft/playwright/MultipartFormData.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package com.microsoft.playwright;
2+
3+
import com.sun.net.httpserver.HttpExchange;
4+
5+
import java.io.ByteArrayOutputStream;
6+
import java.io.IOException;
7+
import java.io.OutputStream;
8+
import java.nio.charset.StandardCharsets;
9+
import java.util.ArrayList;
10+
import java.util.List;
11+
import java.util.regex.Matcher;
12+
import java.util.regex.Pattern;
13+
14+
public class MultipartFormData {
15+
static MultipartFormData parseRequest(HttpExchange exchange) throws IOException {
16+
ByteArrayOutputStream bodyBytes = new ByteArrayOutputStream();
17+
try (OutputStream output = bodyBytes) {
18+
Utils.copy(exchange.getRequestBody(), output);
19+
}
20+
String body = new String(bodyBytes.toByteArray(), StandardCharsets.UTF_8);
21+
String contentType = exchange.getRequestHeaders().get("content-type").get(0);
22+
Matcher matcher = Pattern.compile("boundary=(.*)$").matcher(contentType);
23+
if (!matcher.find()) {
24+
throw new RuntimeException("Boundary not found!");
25+
}
26+
String boundary = matcher.group(1);
27+
return new MultipartFormData(body, boundary);
28+
}
29+
30+
static class Field {
31+
final String filename;
32+
final String content;
33+
34+
Field(String filename, String content) {
35+
this.filename = filename;
36+
this.content = content;
37+
}
38+
}
39+
40+
final List<Field> fields = new ArrayList<>();
41+
42+
MultipartFormData(String body, String boundary) {
43+
String[] parts = Pattern.compile("--" + boundary + "(--)?\r\n", Pattern.MULTILINE).split(body);
44+
for (String part : parts) {
45+
if (part.trim().length() == 0) {
46+
continue;
47+
}
48+
String[] headersAndContent = Pattern.compile("\r\n\r\n", Pattern.MULTILINE).split(part);
49+
if (headersAndContent.length != 2) {
50+
throw new RuntimeException("Unexpected format: " + part);
51+
}
52+
String headers = headersAndContent[0];
53+
String filename = null;
54+
for (String header: Pattern.compile("\r\n", Pattern.MULTILINE).split(headers)) {
55+
Matcher matcher = Pattern.compile("content-disposition: .*filename=\"([^\"]+)\"", Pattern.CASE_INSENSITIVE).matcher(header);
56+
if (!matcher.find()) {
57+
continue;
58+
}
59+
filename = matcher.group(1);
60+
}
61+
String content = headersAndContent[1];
62+
content = content.substring(0, content.length() - "\r\n".length());
63+
fields.add(new Field(filename, content));
64+
}
65+
}
66+
}

‎playwright/src/test/java/com/microsoft/playwright/TestBrowserTypeConnect.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import java.nio.file.Path;
3131
import java.nio.file.Paths;
3232
import java.util.*;
33+
import java.util.concurrent.CompletableFuture;
34+
import java.util.concurrent.ExecutionException;
3335
import java.util.stream.Collectors;
3436

3537
import static com.microsoft.playwright.Utils.*;
@@ -518,4 +520,44 @@ void shouldFulfillWithGlobalFetchResult() {
518520
assertEquals(200, response.status());
519521
assertEquals("{\"foo\": \"bar\"}\n", response.text());
520522
}
523+
524+
@Test
525+
void shouldUploadLargeFile(@TempDir Path tmpDir) throws IOException, ExecutionException, InterruptedException {
526+
Assumptions.assumeTrue(3 <= (Runtime.getRuntime().maxMemory() >> 30), "Fails if max heap size is < 3Gb");
527+
page.navigate(server.PREFIX + "/input/fileupload.html");
528+
Path uploadFile = tmpDir.resolve("200MB.zip");
529+
String str = String.join("", Collections.nCopies(4 * 1024, "A"));
530+
531+
try (Writer stream = new OutputStreamWriter(Files.newOutputStream(uploadFile))) {
532+
for (int i = 0; i < 50 * 1024; i++) {
533+
stream.write(str);
534+
}
535+
}
536+
Locator input = page.locator("input[type='file']");
537+
JSHandle events = input.evaluateHandle("e => {\n" +
538+
" const events = [];\n" +
539+
" e.addEventListener('input', () => events.push('input'));\n" +
540+
" e.addEventListener('change', () => events.push('change'));\n" +
541+
" return events;\n" +
542+
" }");
543+
input.setInputFiles(uploadFile);
544+
assertEquals("200MB.zip", input.evaluate("e => e.files[0].name"));
545+
assertEquals(asList("input", "change"), events.evaluate("e => e"));
546+
CompletableFuture<MultipartFormData> formData = new CompletableFuture<>();
547+
server.setRoute("/upload", exchange -> {
548+
try {
549+
MultipartFormData multipartFormData = MultipartFormData.parseRequest(exchange);
550+
formData.complete(multipartFormData);
551+
} catch (Exception e) {
552+
e.printStackTrace();
553+
formData.completeExceptionally(e);
554+
}
555+
exchange.sendResponseHeaders(200, -1);
556+
});
557+
page.click("input[type=submit]", new Page.ClickOptions().setTimeout(90_000));
558+
List<MultipartFormData.Field> fields = formData.get().fields;
559+
assertEquals(1, fields.size());
560+
assertEquals("200MB.zip", fields.get(0).filename);
561+
assertEquals(200 * 1024 * 1024, fields.get(0).content.length());
562+
}
521563
}

0 commit comments

Comments
 (0)