diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java b/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java index 4048f5c104..c40048adc5 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java @@ -1,27 +1,34 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package ucar.nc2.dataset; import static java.net.HttpURLConnection.HTTP_FORBIDDEN; import static java.net.HttpURLConnection.HTTP_NOT_ACCEPTABLE; import static java.net.HttpURLConnection.HTTP_OK; import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; + import com.google.common.annotations.VisibleForTesting; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import javax.annotation.Nullable; -import com.google.common.collect.Multimap; import thredds.client.catalog.ServiceType; -import thredds.inventory.MFile; -import thredds.inventory.MFiles; import ucar.httpservices.HTTPFactory; import ucar.httpservices.HTTPMethod; import ucar.nc2.util.EscapeStrings; import ucar.unidata.util.StringUtil2; import ucar.unidata.util.Urlencoded; -import java.io.*; -import java.util.*; /** * Detection of the protocol from a location string. @@ -31,6 +38,7 @@ * @since 10/20/2015. */ public class DatasetUrl { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DatasetUrl.class); private static final String alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; private static final String slashalpha = "\\/" + alpha; @@ -398,15 +406,23 @@ private static ServiceType disambiguateHttp(String location) throws IOException return null; } + private static void handleCheckIfResponse(HTTPMethod method, int statusCode, ServiceType serviceChecked) + throws IOException { + if (statusCode == HTTP_UNAUTHORIZED) { + throw new IOException("Unauthorized to open dataset " + method.getURI()); + } else if (statusCode == HTTP_FORBIDDEN) { + logger.warn(String.format( + "Forbidden Request to %s - assuming remote server is not a %s server, but could also indicate incorrect credentials.", + method.getURI(), serviceChecked)); + } + } + // cdmremote private static ServiceType checkIfCdmr(String location) throws IOException { try (HTTPMethod method = HTTPFactory.Head(location + "?req=header")) { int statusCode = method.execute(); if (statusCode >= 300) { - if (statusCode == HTTP_UNAUTHORIZED || statusCode == HTTP_FORBIDDEN) - throw new IOException("Unauthorized to open dataset " + location); - else - throw new IOException(location + " is not a valid URL, return status=" + statusCode); + handleCheckIfResponse(method, statusCode, ServiceType.CdmRemote); } Optional value = method.getResponseHeaderValue("Content-Description"); @@ -442,8 +458,7 @@ private static ServiceType checkIfDods(String location) throws IOException { throw new IOException("OPeNDAP Server Error= " + method.getResponseAsString()); } } - if (status == HTTP_UNAUTHORIZED || status == HTTP_FORBIDDEN) - throw new IOException("Unauthorized to open dataset " + location); + handleCheckIfResponse(method, status, ServiceType.OPENDAP); // not dods return null; @@ -471,6 +486,7 @@ else if (location.endsWith(".html")) return ServiceType.DAP4; } } + handleCheckIfResponse(method, status, ServiceType.DAP4); // not dap4 return null; } @@ -497,14 +513,15 @@ private static boolean checkIfRemoteNcml(String location) throws IOException { method.setRequestHeader("accept-encoding", "identity"); int statusCode = method.execute(); if (statusCode >= 300) { - if (statusCode == HTTP_UNAUTHORIZED) { - throw new IOException("Unauthorized to open dataset " + location); - } else if (statusCode == HTTP_NOT_ACCEPTABLE) { + handleCheckIfResponse(method, statusCode, ServiceType.NCML); + // additional checks + if (statusCode == HTTP_NOT_ACCEPTABLE) { String msg = location + " - this server does not support returning content without any encoding."; msg = msg + " Please download the file locally. Return status=" + statusCode; throw new IOException(msg); } else { - throw new IOException(location + " is not a valid URL, return status=" + statusCode); + throw new IOException(String.format("Error opening %s: %s, (HTTP Status Code %d)", location, + method.getResponseAsString(), statusCode)); } } diff --git a/cdm/core/src/main/java/ucar/unidata/io/http/HTTPRandomAccessFile.java b/cdm/core/src/main/java/ucar/unidata/io/http/HTTPRandomAccessFile.java index 739061aa48..b1b4a16be7 100644 --- a/cdm/core/src/main/java/ucar/unidata/io/http/HTTPRandomAccessFile.java +++ b/cdm/core/src/main/java/ucar/unidata/io/http/HTTPRandomAccessFile.java @@ -1,7 +1,8 @@ /* - * Copyright (c) 1998-2019 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2025 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package ucar.unidata.io.http; import java.io.FileNotFoundException; @@ -40,6 +41,9 @@ public final class HTTPRandomAccessFile extends RemoteRandomAccessFile { private static final long httpMaxCacheSize = Long .parseLong(System.getProperty("ucar.unidata.io.http.maxReadCacheSize", String.valueOf(defaultMaxReadCacheSize))); + private static final String HTTPS = "https"; + private static final String HTTP = "http"; + private static final boolean debug = false, debugDetails = false; /////////////////////////////////////////////////////////////////////////////////// @@ -234,7 +238,7 @@ public long getLastModified() { */ public static class Provider implements RandomAccessFileProvider { - private static final List possibleSchemes = Arrays.asList("http", "https", "nodods", "httpserver"); + private static final List possibleSchemes = Arrays.asList(HTTP, HTTPS, "nodods", "httpserver"); @Override public boolean isOwnerOf(String location) { @@ -250,10 +254,26 @@ public RandomAccessFile open(String location) throws IOException { @Override public RandomAccessFile open(String location, int bufferSize) throws IOException { String scheme = location.split(":")[0]; - if (!scheme.equalsIgnoreCase("https") && !scheme.equalsIgnoreCase("http")) { - location = location.replace(scheme, "http"); + boolean fallback = false; + if (!scheme.equalsIgnoreCase(HTTPS) && !scheme.equalsIgnoreCase(HTTP)) { + location = location.replaceFirst(scheme, HTTPS); + fallback = true; + } + HTTPRandomAccessFile raf; + try { + // try with https first + raf = new HTTPRandomAccessFile(location, bufferSize, httpMaxCacheSize); + } catch (IOException e) { + if (fallback) { + // if we had to guess the scheme, fallback to http, and if it does not work, let the IOError be thrown + raf = new HTTPRandomAccessFile(location.replaceFirst(HTTPS, HTTP), bufferSize, httpMaxCacheSize); + } else { + // didn't try to fallback to http, just throw the original error + throw e; + } } - return new HTTPRandomAccessFile(location, bufferSize, httpMaxCacheSize); + + return raf; } } }