16612 download large files

java/src/org/openqa/selenium/grid/data/NodeStatus.java

@@ -248,4 +248,10 @@ public Map<String, Object> toJson() {
 
     return unmodifiableMap(toReturn);
   }
+
+  @Override
+  public String toString() {
+    return String.format(
+        "NodeStatus(availability: %s, slots:%s, uri:%s)", availability, slots.size(), externalUri);
+  }
 }

java/src/org/openqa/selenium/grid/data/Session.java

@@ -144,4 +144,9 @@ public boolean equals(Object that) {
   public int hashCode() {
     return Objects.hash(id, uri);
   }
+
+  @Override
+  public String toString() {
+    return String.format("Session(id:%s, url:%s, started at: %s)", id, uri, getStartTime());
+  }
 }

java/src/org/openqa/selenium/grid/node/BUILD.bazel

@@ -22,5 +22,6 @@ java_library(
         "//java/src/org/openqa/selenium/remote",
         "//java/src/org/openqa/selenium/status",
         artifact("com.google.guava:guava"),
+        artifact("org.jspecify:jspecify"),
     ],
 )

java/src/org/openqa/selenium/grid/node/Node.java

@@ -171,6 +171,9 @@ protected Node(
             get("/session/{sessionId}/se/files")
                 .to(params -> new DownloadFile(this, sessionIdFrom(params)))
                 .with(spanDecorator("node.download_file")),
+            get("/session/{sessionId}/se/files/{fileName}")
+                .to(params -> new DownloadFile(this, sessionIdFrom(params)))
+                .with(spanDecorator("node.download_file")),
             post("/session/{sessionId}/se/files")
                 .to(params -> new DownloadFile(this, sessionIdFrom(params)))
                 .with(spanDecorator("node.download_file")),

java/src/org/openqa/selenium/grid/node/local/LocalNode.java

@@ -19,12 +19,18 @@
 
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static java.nio.file.Files.readAttributes;
+import static java.time.ZoneOffset.UTC;
+import static java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME;
+import static java.util.Arrays.asList;
+import static java.util.Locale.US;
+import static java.util.Objects.requireNonNullElseGet;
 import static org.openqa.selenium.HasDownloads.DownloadedFile;
 import static org.openqa.selenium.concurrent.ExecutorServices.shutdownGracefully;
 import static org.openqa.selenium.grid.data.Availability.DOWN;
 import static org.openqa.selenium.grid.data.Availability.DRAINING;
 import static org.openqa.selenium.grid.data.Availability.UP;
 import static org.openqa.selenium.grid.node.CapabilityResponseEncoder.getEncoder;
+import static org.openqa.selenium.net.Urls.urlDecode;
 import static org.openqa.selenium.remote.CapabilityType.ENABLE_DOWNLOADS;
 import static org.openqa.selenium.remote.HttpSessionId.getSessionId;
 import static org.openqa.selenium.remote.RemoteTags.CAPABILITIES;
@@ -39,6 +45,7 @@
 import com.github.benmanes.caffeine.cache.Ticker;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
+import com.google.common.net.MediaType;
 import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
@@ -47,9 +54,11 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -107,6 +116,7 @@
 import org.openqa.selenium.json.Json;
 import org.openqa.selenium.remote.Browser;
 import org.openqa.selenium.remote.SessionId;
+import org.openqa.selenium.remote.http.Contents;
 import org.openqa.selenium.remote.http.HttpMethod;
 import org.openqa.selenium.remote.http.HttpRequest;
 import org.openqa.selenium.remote.http.HttpResponse;
@@ -123,6 +133,7 @@ public class LocalNode extends Node implements Closeable {
 
   private static final Json JSON = new Json();
   private static final Logger LOG = Logger.getLogger(LocalNode.class.getName());
+  private static final DateTimeFormatter HTTP_DATE_FORMAT = RFC_1123_DATE_TIME.withLocale(US);
 
   private final EventBus bus;
   private final URI externalUri;
@@ -742,9 +753,12 @@ public HttpResponse downloadFile(HttpRequest req, SessionId id) {
         Optional.ofNullable(tempFS.getBaseDir().listFiles()).orElse(new File[] {})[0];
 
     try {
-      if (req.getMethod().equals(HttpMethod.GET)) {
+      if (req.getMethod().equals(HttpMethod.GET) && req.getUri().endsWith("/se/files")) {
         return listDownloadedFiles(downloadsDirectory);
       }
+      if (req.getMethod().equals(HttpMethod.GET)) {
+        return getDownloadedFile(downloadsDirectory, extractFileName(req));
+      }
       if (req.getMethod().equals(HttpMethod.DELETE)) {
         return deleteDownloadedFile(downloadsDirectory);
       }
@@ -754,6 +768,19 @@ public HttpResponse downloadFile(HttpRequest req, SessionId id) {
     }
   }
 
+  private String extractFileName(HttpRequest req) {
+    return extractFileName(req.getUri());
+  }
+
+  String extractFileName(String uri) {
+    String prefix = "/se/files/";
+    int index = uri.lastIndexOf(prefix);
+    if (index < 0) {
+      throw new IllegalArgumentException("Unexpected URL for downloading a file: " + uri);
+    }
+    return urlDecode(uri.substring(index + prefix.length())).replace(' ', '+');
+  }
+
   /** User wants to list files that can be downloaded */
   private HttpResponse listDownloadedFiles(File downloadsDirectory) {
     File[] files = Optional.ofNullable(downloadsDirectory.listFiles()).orElse(new File[] {});
@@ -798,29 +825,63 @@ private HttpResponse getDownloadedFile(HttpRequest req, File downloadsDirectory)
                     new WebDriverException(
                         "Please specify file to download in payload as {\"name\":"
                             + " \"fileToDownload\"}"));
-    File[] allFiles =
-        Optional.ofNullable(downloadsDirectory.listFiles((dir, name) -> name.equals(filename)))
-            .orElse(new File[] {});
-    if (allFiles.length == 0) {
-      throw new WebDriverException(
-          String.format(
-              "Cannot find file [%s] in directory %s.",
-              filename, downloadsDirectory.getAbsolutePath()));
-    }
-    if (allFiles.length != 1) {
-      throw new WebDriverException(
-          String.format("Expected there to be only 1 file. There were: %s.", allFiles.length));
-    }
-    String content = Zip.zip(allFiles[0]);
+    File file = findDownloadedFile(downloadsDirectory, filename);
+    String content = Zip.zip(file);
     Map<String, Object> data =
         Map.of(
             "filename", filename,
-            "file", getFileInfo(allFiles[0]),
+            "file", getFileInfo(file),
             "contents", content);
     Map<String, Map<String, Object>> result = Map.of("value", data);
     return new HttpResponse().setContent(asJson(result));
   }
 
+  private HttpResponse getDownloadedFile(File downloadsDirectory, String fileName)
+      throws IOException {
+    if (fileName.isEmpty()) {
+      throw new WebDriverException("Please specify file to download in URL");
+    }
+    File file = findDownloadedFile(downloadsDirectory, fileName);
+    BasicFileAttributes attributes = readAttributes(file.toPath(), BasicFileAttributes.class);
+    return new HttpResponse()
+        .setHeader("Content-Type", MediaType.OCTET_STREAM.toString())
+        .setHeader("Content-Length", String.valueOf(attributes.size()))
+        .setHeader("Last-Modified", lastModifiedHeader(attributes.lastModifiedTime()))
+        .setContent(Contents.file(file));
+  }
+
+  private String lastModifiedHeader(FileTime fileTime) {
+    return HTTP_DATE_FORMAT.format(fileTime.toInstant().atZone(UTC));
+  }
+
+  private File findDownloadedFile(File downloadsDirectory, String filename)
+      throws WebDriverException {
+    List<File> matchingFiles =
+        asList(
+            requireNonNullElseGet(
+                downloadsDirectory.listFiles((dir, name) -> name.equals(filename)),
+                () -> new File[0]));
+    if (matchingFiles.isEmpty()) {
+      List<File> files = downloadedFiles(downloadsDirectory);
+      throw new WebDriverException(
+          String.format(
+              "Cannot find file [%s] in directory %s. Found %s files: %s.",
+              filename, downloadsDirectory.getAbsolutePath(), files.size(), files));
+    }
+    if (matchingFiles.size() != 1) {
+      throw new WebDriverException(
+          String.format(
+              "Expected there to be only 1 file. Found %s files: %s.",
+              matchingFiles.size(), matchingFiles));
+    }
+    return matchingFiles.get(0);
+  }
+
+  private static List<File> downloadedFiles(File downloadsDirectory) {
+    File[] files = requireNonNullElseGet(downloadsDirectory.listFiles(), () -> new File[0]);
+    return asList(files);
+  }
+
   private HttpResponse deleteDownloadedFile(File downloadsDirectory) {
     File[] files = Optional.ofNullable(downloadsDirectory.listFiles()).orElse(new File[] {});
     for (File file : files) {

java/src/org/openqa/selenium/net/Urls.java

@@ -17,14 +17,16 @@
 
 package org.openqa.selenium.net;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.net.URLDecoder;
 import java.net.URLEncoder;
-import java.nio.charset.StandardCharsets;
 import org.jspecify.annotations.NullMarked;
 import org.openqa.selenium.internal.Require;
 
@@ -43,7 +45,11 @@ private Urls() {
    * @see URLEncoder#encode(java.lang.String, java.lang.String)
    */
   public static String urlEncode(String value) {
-    return URLEncoder.encode(value, StandardCharsets.UTF_8);
+    return URLEncoder.encode(value, UTF_8);
+  }
+
+  public static String urlDecode(String encodedValue) {
+    return URLDecoder.decode(encodedValue, UTF_8);
   }
 
   public static URL fromUri(URI uri) {

java/src/org/openqa/selenium/netty/server/FileBackedOutputStreamContentSupplier.java

@@ -0,0 +1,74 @@
+// Licensed to the Software Freedom Conservancy (SFC) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The SFC licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.openqa.selenium.netty.server;
+
+import com.google.common.io.FileBackedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.charset.Charset;
+import org.openqa.selenium.remote.http.Contents;
+
+class FileBackedOutputStreamContentSupplier implements Contents.Supplier {
+  private final String description;
+  private final FileBackedOutputStream buffer;
+  private final long length;
+
+  FileBackedOutputStreamContentSupplier(
+      String description, FileBackedOutputStream buffer, long length) {
+    this.description = description;
+    this.buffer = buffer;
+    this.length = length;
+  }
+
+  @Override
+  public InputStream get() {
+    try {
+      return buffer.asByteSource().openBufferedStream();
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+
+  @Override
+  public long length() {
+    return length;
+  }
+
+  @Override
+  public void close() throws IOException {
+    buffer.reset();
+  }
+
+  @Override
+  public String toString() {
+    return String.format("Content for %s (%s bytes)", description, length);
+  }
+
+  @Override
+  public String contentAsString(Charset charset) {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    try {
+      buffer.asByteSource().copyTo(out);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return out.toString(charset);
+  }
+}

java/src/org/openqa/selenium/netty/server/RequestConverter.java

@@ -23,7 +23,6 @@
 import static io.netty.handler.codec.http.HttpMethod.OPTIONS;
 import static io.netty.handler.codec.http.HttpMethod.POST;
 
-import com.google.common.io.ByteSource;
 import com.google.common.io.FileBackedOutputStream;
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.ChannelHandlerContext;
@@ -37,11 +36,9 @@
 import io.netty.handler.codec.http.LastHttpContent;
 import io.netty.handler.codec.http.QueryStringDecoder;
 import io.netty.util.ReferenceCountUtil;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Logger;
 import org.openqa.selenium.internal.Debug;
 import org.openqa.selenium.remote.http.Contents;
@@ -56,7 +53,7 @@ class RequestConverter extends SimpleChannelInboundHandler<HttpObject> {
   private static final List<io.netty.handler.codec.http.HttpMethod> SUPPORTED_METHODS =
       Arrays.asList(DELETE, GET, POST, OPTIONS);
   private volatile FileBackedOutputStream buffer;
-  private volatile int length;
+  private final AtomicLong length = new AtomicLong();
   private volatile HttpRequest request;
 
   @Override
@@ -93,7 +90,7 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Ex
           AttributeKey.HTTP_FLAVOR.getKey(), nettyRequest.protocolVersion().majorVersion());
 
       buffer = null;
-      length = -1;
+      length.set(-1);
     }
 
     if (request != null && msg instanceof HttpContent) {
@@ -103,12 +100,12 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Ex
       if (nBytes > 0) {
         if (buffer == null) {
           buffer = new FileBackedOutputStream(3 * 1024 * 1024, true);
-          length = 0;
+          length.set(0);
         }
 
         try {
           buf.readBytes(buffer, nBytes);
-          length += nBytes;
+          length.addAndGet(nBytes);
         } finally {
           buf.release();
         }
@@ -118,30 +115,8 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Ex
         LOG.log(Debug.getDebugLogLevel(), "End of http request: {0}", msg);
 
         if (buffer != null) {
-          ByteSource source = buffer.asByteSource();
-          int len = length;
-
           request.setContent(
-              new Contents.Supplier() {
-                @Override
-                public InputStream get() {
-                  try {
-                    return source.openBufferedStream();
-                  } catch (IOException e) {
-                    throw new UncheckedIOException(e);
-                  }
-                }
-
-                @Override
-                public int length() {
-                  return len;
-                }
-
-                @Override
-                public void close() throws IOException {
-                  buffer.reset();
-                }
-              });
+              new FileBackedOutputStreamContentSupplier(request.toString(), buffer, length.get()));
         } else {
           request.setContent(Contents.empty());
         }