Skip to content

Commit 7a64195

Browse files
garyrussellartembilan
garyrussell
authored and
artembilan
committed
GH-3482: (S)FTP: Fix Recursive LS (ARFOG)
Resolves #3482 `.` and `..` should be ignored when recursing. **cherry-pick to 5.4.x, 5.3.x** * Fix checkstyle. * Fix test in `file` module - test was incorrect; it would have detected this problem.
1 parent ab12b6b commit 7a64195

File tree

4 files changed

+116
-9
lines changed

4 files changed

+116
-9
lines changed

‎spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -975,18 +975,23 @@ protected final List<F> filterFiles(F[] files) {
975975
private void processFile(Session<F> session, String directory, String subDirectory, List<F> lsFiles,
976976
boolean recursion, F file) throws IOException {
977977

978+
String fileName = getFilename(file);
979+
String fileSep = this.remoteFileTemplate.getRemoteFileSeparator();
980+
boolean isDots = ".".equals(fileName)
981+
|| "..".equals(fileName)
982+
|| fileName.endsWith(fileSep + ".")
983+
|| fileName.endsWith(fileSep + "..");
978984
if (this.options.contains(Option.SUBDIRS) || !isDirectory(file)) {
979-
if (recursion && StringUtils.hasText(subDirectory)) {
985+
if (recursion && StringUtils.hasText(subDirectory) && (!isDots || this.options.contains(Option.ALL))) {
980986
lsFiles.add(enhanceNameWithSubDirectory(file, subDirectory));
981987
}
982-
else {
988+
else if (this.options.contains(Option.ALL) || !isDots) {
983989
lsFiles.add(file);
984990
}
985991
}
986-
String fileName = getFilename(file);
987-
if (recursion && isDirectory(file) && !(".".equals(fileName)) && !("..".equals(fileName))) {
992+
if (recursion && isDirectory(file) && !isDots) {
988993
lsFiles.addAll(listFilesInRemoteDir(session, directory,
989-
subDirectory + fileName + this.remoteFileTemplate.getRemoteFileSeparator()));
994+
subDirectory + fileName + fileSep));
990995
}
991996
}
992997

‎spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -406,12 +406,13 @@ public void testLs_f_R_dirs() throws Exception {
406406
MessageBuilder<List<TestLsEntry>> out = (MessageBuilder<List<TestLsEntry>>) gw
407407
.handleRequestMessage(new GenericMessage<>("testremote/x"));
408408
assertThat(out).isNotNull();
409-
assertThat(out.getPayload()).hasSize(5);
409+
assertThat(out.getPayload()).hasSize(6);
410410
assertThat(out.getPayload().get(0).getFilename()).isEqualTo("f1");
411411
assertThat(out.getPayload().get(1).getFilename()).isEqualTo("d1");
412412
assertThat(out.getPayload().get(2).getFilename()).isEqualTo("d1/d2");
413-
assertThat(out.getPayload().get(3).getFilename()).isEqualTo("d1/f3");
414-
assertThat(out.getPayload().get(4).getFilename()).isEqualTo("f2");
413+
assertThat(out.getPayload().get(3).getFilename()).isEqualTo("d1/d2/f4");
414+
assertThat(out.getPayload().get(4).getFilename()).isEqualTo("d1/f3");
415+
assertThat(out.getPayload().get(5).getFilename()).isEqualTo("f2");
415416
assertThat(out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)).isEqualTo("testremote/x/");
416417
}
417418

‎spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,39 @@
5959
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
6060
reply-channel="output"/>
6161

62+
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
63+
request-channel="inboundLSRecursive"
64+
command="ls"
65+
expression="payload"
66+
command-options="-R -dirs"
67+
mode="REPLACE_IF_MODIFIED"
68+
filter="dotStarDotTxtFilter"
69+
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
70+
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
71+
reply-channel="output"/>
72+
73+
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
74+
request-channel="inboundLSRecursiveALL"
75+
command="ls"
76+
expression="payload"
77+
command-options="-a -R -dirs"
78+
mode="REPLACE_IF_MODIFIED"
79+
filter="dotStarDotTxtFilter"
80+
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
81+
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
82+
reply-channel="output"/>
83+
84+
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
85+
request-channel="inboundLSRecursiveNoDirs"
86+
command="ls"
87+
expression="payload"
88+
command-options="-R"
89+
mode="REPLACE_IF_MODIFIED"
90+
filter="dotStarDotTxtFilter"
91+
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
92+
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
93+
reply-channel="output"/>
94+
6295
<bean id="dotStarDotTxtFilter"
6396
class="org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter">
6497
<constructor-arg value="^.*\.txt$" />

‎spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2013-2020 the original author or authors.
2+
* Copyright 2013-2021 the original author or authors.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@
3535
import java.util.concurrent.Executors;
3636
import java.util.concurrent.TimeUnit;
3737
import java.util.regex.Matcher;
38+
import java.util.stream.Collectors;
3839

3940
import org.apache.commons.io.FileUtils;
4041
import org.junit.jupiter.api.BeforeEach;
@@ -60,6 +61,7 @@
6061
import org.springframework.integration.sftp.server.SessionClosedEvent;
6162
import org.springframework.integration.sftp.server.SessionOpenedEvent;
6263
import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
64+
import org.springframework.integration.sftp.session.SftpFileInfo;
6365
import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
6466
import org.springframework.integration.support.MessageBuilder;
6567
import org.springframework.integration.test.util.TestUtils;
@@ -100,6 +102,15 @@ public class SftpServerOutboundTests extends SftpTestSupport {
100102
@Autowired
101103
private DirectChannel inboundMGetRecursive;
102104

105+
@Autowired
106+
private DirectChannel inboundLSRecursive;
107+
108+
@Autowired
109+
private DirectChannel inboundLSRecursiveALL;
110+
111+
@Autowired
112+
private DirectChannel inboundLSRecursiveNoDirs;
113+
103114
@Autowired
104115
private DirectChannel inboundMGetRecursiveFiltered;
105116

@@ -254,6 +265,63 @@ public void testInt3172LocalDirectoryExpressionMGETRecursive() throws IOExceptio
254265
FileUtils.copyInputStreamToFile(new ByteArrayInputStream(localAsString.getBytes()), secondRemote);
255266
}
256267

268+
@Test
269+
@SuppressWarnings("unchecked")
270+
void testLSRecursive() throws IOException {
271+
String dir = "sftpSource/";
272+
this.inboundLSRecursive.send(new GenericMessage<Object>(dir));
273+
Message<?> result = this.output.receive(1000);
274+
assertThat(result).isNotNull();
275+
List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
276+
assertThat(files).hasSize(4);
277+
assertThat(files.stream()
278+
.map(fi -> fi.getFilename())
279+
.collect(Collectors.toList())).contains(
280+
" sftpSource1.txt",
281+
"sftpSource2.txt",
282+
"subSftpSource",
283+
"subSftpSource/subSftpSource1.txt");
284+
}
285+
286+
@Test
287+
@SuppressWarnings("unchecked")
288+
void testLSRecursiveALL() throws IOException {
289+
String dir = "sftpSource/";
290+
this.inboundLSRecursiveALL.send(new GenericMessage<Object>(dir));
291+
Message<?> result = this.output.receive(1000);
292+
assertThat(result).isNotNull();
293+
List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
294+
assertThat(files).hasSize(8);
295+
assertThat(files.stream()
296+
.map(fi -> fi.getFilename())
297+
.collect(Collectors.toList())).contains(
298+
" sftpSource1.txt",
299+
"sftpSource2.txt",
300+
"subSftpSource",
301+
"subSftpSource/subSftpSource1.txt",
302+
".",
303+
"..",
304+
"subSftpSource/.",
305+
"subSftpSource/..");
306+
}
307+
308+
@Test
309+
@SuppressWarnings("unchecked")
310+
void testLSRecursiveNoDirs() throws IOException {
311+
String dir = "sftpSource/";
312+
this.inboundLSRecursiveNoDirs.send(new GenericMessage<Object>(dir));
313+
Message<?> result = this.output.receive(1000);
314+
assertThat(result).isNotNull();
315+
List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
316+
assertThat(files).hasSize(3);
317+
assertThat(files.stream()
318+
.map(fi -> fi.getFilename())
319+
.collect(Collectors.toList())).contains(
320+
" sftpSource1.txt",
321+
"sftpSource2.txt",
322+
"subSftpSource/subSftpSource1.txt");
323+
}
324+
257325
private long setModifiedOnSource1() {
258326
File firstRemote = new File(getSourceRemoteDirectory(), " sftpSource1.txt");
259327
firstRemote.setLastModified(System.currentTimeMillis() - 1_000_000);

0 commit comments

Comments
 (0)