Skip to content

Commit e352072

Browse files
HoustonPutmanHoustonPutman
authored
SOLR-17427: Fix url building in cli (#2678)
1 parent bfc25e2 commit e352072

File tree

5 files changed

+90
-51
lines changed

5 files changed

+90
-51
lines changed

‎solr/core/src/java/org/apache/solr/cli/ApiTool.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ protected String callGet(String url, String credentials) throws Exception {
125125
* @return Solr base url with port and root (from above example http://localhost:8983/solr)
126126
*/
127127
public static String getSolrUrlFromUri(URI uri) {
128-
return uri.getScheme() + "://" + uri.getAuthority() + "/" + uri.getPath().split("/")[1];
128+
return uri.resolve("/" + uri.getPath().split("/")[1]).toString();
129129
}
130130

131131
public static ModifiableSolrParams getSolrParamsFromUri(URI uri) {

‎solr/core/src/java/org/apache/solr/cli/PostTool.java‎

Lines changed: 46 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -379,37 +379,32 @@ private void doArgsMode(String[] args) {
379379
private void doWebMode() {
380380
reset();
381381
int numPagesPosted;
382-
try {
383-
if (type != null) {
384-
throw new IllegalArgumentException(
385-
"Specifying content-type with \"--mode=web\" is not supported");
386-
}
382+
if (type != null) {
383+
throw new IllegalArgumentException(
384+
"Specifying content-type with \"--mode=web\" is not supported");
385+
}
387386

388-
// Set Extracting handler as default
389-
solrUpdateUrl = appendUrlPath(solrUpdateUrl, "/extract");
387+
// Set Extracting handler as default
388+
solrUpdateUrl = appendUrlPath(solrUpdateUrl, "/extract");
390389

391-
info("Posting web pages to Solr url " + solrUpdateUrl);
392-
auto = true;
393-
info(
394-
"Entering auto mode. Indexing pages with content-types corresponding to file endings "
395-
+ fileTypes);
396-
if (recursive > 0) {
397-
if (recursive > MAX_WEB_DEPTH) {
398-
recursive = MAX_WEB_DEPTH;
399-
warn("Too large recursion depth for web mode, limiting to " + MAX_WEB_DEPTH + "...");
400-
}
401-
if (delay < DEFAULT_WEB_DELAY) {
402-
warn(
403-
"Never crawl an external web site faster than every 10 seconds, your IP will probably be blocked");
404-
}
405-
info("Entering recursive mode, depth=" + recursive + ", delay=" + delay + "s");
390+
info("Posting web pages to Solr url " + solrUpdateUrl);
391+
auto = true;
392+
info(
393+
"Entering auto mode. Indexing pages with content-types corresponding to file endings "
394+
+ fileTypes);
395+
if (recursive > 0) {
396+
if (recursive > MAX_WEB_DEPTH) {
397+
recursive = MAX_WEB_DEPTH;
398+
warn("Too large recursion depth for web mode, limiting to " + MAX_WEB_DEPTH + "...");
406399
}
407-
numPagesPosted = postWebPages(args, 0, out);
408-
info(numPagesPosted + " web pages indexed.");
409-
410-
} catch (URISyntaxException e) {
411-
warn("Wrong URL trying to append /extract to " + solrUpdateUrl);
400+
if (delay < DEFAULT_WEB_DELAY) {
401+
warn(
402+
"Never crawl an external web site faster than every 10 seconds, your IP will probably be blocked");
403+
}
404+
info("Entering recursive mode, depth=" + recursive + ", delay=" + delay + "s");
412405
}
406+
numPagesPosted = postWebPages(args, 0, out);
407+
info(numPagesPosted + " web pages indexed.");
413408
}
414409

415410
private void doStdinMode() {
@@ -533,7 +528,7 @@ int postFiles(File[] files, OutputStream out, String type) {
533528
postFile(srcFile, out, type);
534529
Thread.sleep(delay * 1000L);
535530
filesPosted++;
536-
} catch (InterruptedException | URISyntaxException e) {
531+
} catch (InterruptedException | MalformedURLException | URISyntaxException e) {
537532
throw new RuntimeException(e);
538533
}
539534
}
@@ -699,13 +694,14 @@ protected int webCrawl(int level, OutputStream out) {
699694
* @param link the absolute or relative link
700695
* @return the string version of the full URL
701696
*/
702-
protected String computeFullUrl(URL baseUrl, String link) {
697+
protected static String computeFullUrl(URL baseUrl, String link)
698+
throws MalformedURLException, URISyntaxException {
703699
if (link == null || link.length() == 0) {
704700
return null;
705701
}
706702
if (!link.startsWith("http")) {
707703
if (link.startsWith("/")) {
708-
link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + link;
704+
link = baseUrl.toURI().resolve(link).toString();
709705
} else {
710706
if (link.contains(":")) {
711707
return null; // Skip non-relative URLs
@@ -715,10 +711,12 @@ protected String computeFullUrl(URL baseUrl, String link) {
715711
int sep = path.lastIndexOf('/');
716712
String file = path.substring(sep + 1);
717713
if (file.contains(".") || file.contains("?")) {
718-
path = path.substring(0, sep);
714+
path = path.substring(0, sep + 1);
715+
} else {
716+
path = path + "/";
719717
}
720718
}
721-
link = baseUrl.getProtocol() + "://" + baseUrl.getAuthority() + path + "/" + link;
719+
link = baseUrl.toURI().resolve(path + link).toString();
722720
}
723721
}
724722
link = normalizeUrlEnding(link);
@@ -806,7 +804,8 @@ public static String appendParam(String url, String param) {
806804
}
807805

808806
/** Opens the file and posts its contents to the solrUrl, writes to response to output. */
809-
public void postFile(File file, OutputStream output, String type) throws URISyntaxException {
807+
public void postFile(File file, OutputStream output, String type)
808+
throws MalformedURLException, URISyntaxException {
810809
InputStream is = null;
811810

812811
URI uri = solrUpdateUrl;
@@ -884,9 +883,20 @@ public void postFile(File file, OutputStream output, String type) throws URISynt
884883
* @param append the path to append
885884
* @return the final URL version
886885
*/
887-
protected static URI appendUrlPath(URI uri, String append) throws URISyntaxException {
888-
var newPath = uri.getPath() + append;
889-
return new URI(uri.getScheme(), uri.getAuthority(), newPath, uri.getQuery(), uri.getFragment());
886+
protected static URI appendUrlPath(URI uri, String append) {
887+
if (append == null || append.isEmpty()) {
888+
return uri;
889+
}
890+
if (append.startsWith("/")) {
891+
append = append.substring(1);
892+
}
893+
if (uri.getQuery() != null && !uri.getQuery().isEmpty()) {
894+
append += "?" + uri.getQuery();
895+
}
896+
if (!uri.getPath().endsWith("/")) {
897+
append = uri.getPath() + "/" + append;
898+
}
899+
return uri.resolve(append);
890900
}
891901

892902
/**

‎solr/core/src/java/org/apache/solr/cli/SolrCLI.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -636,8 +636,11 @@ public static String normalizeSolrUrl(String solrUrl) {
636636
*/
637637
public static String normalizeSolrUrl(String solrUrl, boolean logUrlFormatWarning) {
638638
if (solrUrl != null) {
639-
if (solrUrl.contains("/solr")) { //
640-
String newSolrUrl = solrUrl.substring(0, solrUrl.indexOf("/solr"));
639+
URI uri = URI.create(solrUrl);
640+
String urlPath = uri.getRawPath();
641+
if (urlPath != null && urlPath.contains("/solr")) {
642+
String newSolrUrl =
643+
uri.resolve(urlPath.substring(0, urlPath.lastIndexOf("/solr") + 1)).toString();
641644
if (logUrlFormatWarning) {
642645
CLIO.out(
643646
"WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. \"/solr\"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from ["

‎solr/core/src/test/org/apache/solr/cli/ApiToolTest.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,4 +99,20 @@ private void assertFindInJson(String json, String find) {
9999
String.format(Locale.ROOT, "Could not find string %s in response: \n%s", find, json),
100100
json.contains(find));
101101
}
102+
103+
@Test
104+
public void testSolrUrlParsing() throws Exception {
105+
assertEquals(
106+
"https://test-host.solr:8983/solr",
107+
ApiTool.getSolrUrlFromUri(URI.create("https://test-host.solr:8983/solr")));
108+
assertEquals(
109+
"https://test-host.solr:8983/solr",
110+
ApiTool.getSolrUrlFromUri(URI.create("https://test-host.solr:8983/solr/test/api")));
111+
assertEquals(
112+
"https://test-host.solr/solr",
113+
ApiTool.getSolrUrlFromUri(URI.create("https://test-host.solr/solr/test/api")));
114+
assertEquals(
115+
"http://test-host.solr/solr",
116+
ApiTool.getSolrUrlFromUri(URI.create("http://test-host.solr/solr/test/api")));
117+
}
102118
}

‎solr/core/src/test/org/apache/solr/cli/PostToolTest.java‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -211,30 +211,25 @@ public void testNormalizeUrlEnding() {
211211
}
212212

213213
@Test
214-
public void testComputeFullUrl() throws IOException {
215-
216-
PostTool webPostTool = new PostTool();
217-
214+
public void testComputeFullUrl() throws IOException, URISyntaxException {
218215
assertEquals(
219216
"http://[ff01::114]/index.html",
220-
webPostTool.computeFullUrl(URI.create("http://[ff01::114]/").toURL(), "/index.html"));
217+
PostTool.computeFullUrl(URI.create("http://[ff01::114]/").toURL(), "/index.html"));
221218
assertEquals(
222219
"http://[ff01::114]/index.html",
223-
webPostTool.computeFullUrl(
224-
URI.create("http://[ff01::114]/foo/bar/").toURL(), "/index.html"));
220+
PostTool.computeFullUrl(URI.create("http://[ff01::114]/foo/bar/").toURL(), "/index.html"));
225221
assertEquals(
226222
"http://[ff01::114]/fil.html",
227-
webPostTool.computeFullUrl(
223+
PostTool.computeFullUrl(
228224
URI.create("http://[ff01::114]/foo.htm?baz#hello").toURL(), "fil.html"));
229225
// TODO: How to know what is the base if URL path ends with "foo"??
230226
// assertEquals("http://[ff01::114]/fil.html", t_web.computeFullUrl(new
231227
// URL("http://[ff01::114]/foo?baz#hello"), "fil.html"));
232-
assertNull(webPostTool.computeFullUrl(URI.create("http://[ff01::114]/").toURL(), "fil.jpg"));
228+
assertNull(PostTool.computeFullUrl(URI.create("http://[ff01::114]/").toURL(), "fil.jpg"));
233229
assertNull(
234-
webPostTool.computeFullUrl(
235-
URI.create("http://[ff01::114]/").toURL(), "mailto:[email protected]"));
230+
PostTool.computeFullUrl(URI.create("http://[ff01::114]/").toURL(), "mailto:[email protected]"));
236231
assertNull(
237-
webPostTool.computeFullUrl(URI.create("http://[ff01::114]/").toURL(), "ftp://server/file"));
232+
PostTool.computeFullUrl(URI.create("http://[ff01::114]/").toURL(), "ftp://server/file"));
238233
}
239234

240235
@Test
@@ -265,6 +260,21 @@ public void testAppendUrlPath() throws URISyntaxException {
265260
assertEquals(
266261
URI.create("http://[ff01::114]/a?foo=bar"),
267262
PostTool.appendUrlPath(URI.create("http://[ff01::114]?foo=bar"), "/a"));
263+
assertEquals(
264+
URI.create("http://[ff01::114]/a?foo=bar"),
265+
PostTool.appendUrlPath(URI.create("http://[ff01::114]/?foo=bar"), "/a"));
266+
assertEquals(
267+
URI.create("http://[ff01::114]/a/b?foo=bar"),
268+
PostTool.appendUrlPath(URI.create("http://[ff01::114]/a?foo=bar"), "/b"));
269+
assertEquals(
270+
URI.create("http://[ff01::114]/a/b?foo=bar"),
271+
PostTool.appendUrlPath(URI.create("http://[ff01::114]/a/?foo=bar"), "/b"));
272+
assertEquals(
273+
URI.create("http://[ff01::114]/a/b?foo=bar"),
274+
PostTool.appendUrlPath(URI.create("http://[ff01::114]/a?foo=bar"), "b"));
275+
assertEquals(
276+
URI.create("http://[ff01::114]/a/b?foo=bar"),
277+
PostTool.appendUrlPath(URI.create("http://[ff01::114]/a/?foo=bar"), "b"));
268278
}
269279

270280
@Test

0 commit comments

Comments
 (0)