Skip to content

Commit 0688fe0

Browse files
dkocherdkocher
authored
Merge pull request #17910 from iterate-ch/bugfix/CTERA-355-directio
Review chunk handling.
2 parents be578c5 + 6b8cf25 commit 0688fe0

File tree

7 files changed

+75
-15
lines changed

7 files changed

+75
-15
lines changed

‎ctera/src/main/java/ch/cyberduck/core/ctera/CteraBulkFeature.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,7 @@ public Map<TransferItem, TransferStatus> pre(final Transfer.Type type, final Map
8989
}
9090
}
9191
else {
92-
if(0L == status.getOffset()) {
93-
status.setUrl(metadata.chunks.get(0).url);
92+
if(metadata.actual_blocks_range.file_size == 0) {
9493
final Map<String, String> parameters = new HashMap<>(status.getParameters());
9594
parameters.put(CteraDirectIOReadFeature.CTERA_WRAPPEDKEY, metadata.encrypt_info.wrapped_key);
9695
status.setParameters(parameters);
@@ -108,12 +107,14 @@ public Map<TransferItem, TransferStatus> pre(final Transfer.Type type, final Map
108107
private DirectIO getMetadata(final Path file) throws IOException, BackgroundException {
109108
final HttpGet request = new HttpGet(String.format("%s%s%s", new HostUrlProvider().withUsername(false).withPath(false)
110109
.get(session.getHost()), CteraDirectIOInterceptor.DIRECTIO_PATH, versionid.getVersionId(file)));
111-
return session.getClient().getClient().execute(request, new AbstractResponseHandler<DirectIO>() {
110+
final DirectIO metadata = session.getClient().getClient().execute(request, new AbstractResponseHandler<DirectIO>() {
112111
@Override
113112
public DirectIO handleEntity(final HttpEntity entity) throws IOException {
114113
final ObjectMapper mapper = new ObjectMapper();
115114
return mapper.readValue(entity.getContent(), DirectIO.class);
116115
}
117116
});
117+
log.debug("DirectIO metadata {} retrieved for {}", metadata, file);
118+
return metadata;
118119
}
119120
}

‎ctera/src/main/java/ch/cyberduck/core/ctera/CteraDelegatingReadFeature.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,10 @@ public CteraDelegatingReadFeature(final CteraSession session) {
3939

4040
@Override
4141
public InputStream read(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
42-
if(StringUtils.isNotBlank(status.getUrl())) {
42+
if(StringUtils.isNotBlank(status.getParameters().get(CteraDirectIOReadFeature.CTERA_WRAPPEDKEY))) {
4343
return new CteraDirectIOReadFeature(session).read(file, status, callback);
4444
}
45-
log.warn("No URL found in status {} for {}", status, file);
45+
log.warn("No key material found in status {} for {}", status, file);
4646
return new CteraReadFeature(session).read(file, status, callback);
4747
}
4848

‎ctera/src/main/java/ch/cyberduck/core/ctera/CteraDirectIOReadFeature.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@
3434

3535
import java.io.IOException;
3636
import java.io.InputStream;
37-
import java.util.ArrayList;
3837
import java.util.Collections;
38+
import java.util.EnumSet;
3939
import java.util.Enumeration;
4040
import java.util.List;
4141

@@ -54,19 +54,25 @@ public CteraDirectIOReadFeature(final CteraSession session) {
5454
public InputStream read(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
5555
try {
5656
final EncryptInfo key = new EncryptInfo(status.getParameters().get(CTERA_WRAPPEDKEY), session.getOrCreateAPIKeys().secretKey);
57-
final List<DirectIO.Chunk> chunks = new ArrayList<>();
57+
if(status.getLength() == 0) {
58+
return new ChunkSequenceInputStream(Collections.emptyList(), key);
59+
}
5860
final DirectIO.Chunk chunk = new DirectIO.Chunk();
5961
chunk.url = status.getUrl();
6062
chunk.len = status.getLength();
61-
chunks.add(chunk);
6263
log.debug("Return chunk {} for file {}", chunk, file);
63-
return new ChunkSequenceInputStream(chunks, key);
64+
return new ChunkSequenceInputStream(Collections.singletonList(chunk), key);
6465
}
6566
catch(IOException e) {
6667
throw new HttpExceptionMappingService().map("Download {0} failed", e, file);
6768
}
6869
}
6970

71+
@Override
72+
public EnumSet<Flags> features(final Path file) {
73+
return EnumSet.noneOf(Flags.class);
74+
}
75+
7076
private final class ChunkSequenceInputStream extends InputStream {
7177

7278
private final Enumeration<DirectIO.Chunk> chunks;

‎ctera/src/main/java/ch/cyberduck/core/ctera/CteraProtocol.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,9 +105,11 @@ public String getTokenPlaceholder() {
105105
@Override
106106
public Map<String, String> getProperties() {
107107
final Map<String, String> properties = new HashMap<>();
108-
properties.put("queue.download.segments.size.dynamic", String.valueOf(false));
109-
properties.put("queue.download.segments.size", String.valueOf(DIRECTIO_CHUNKSIZE));
110-
properties.put("queue.download.segments.threshold", String.valueOf(DIRECTIO_CHUNKSIZE));
108+
if(PreferencesFactory.get().getBoolean("ctera.download.directio.enable")) {
109+
properties.put("queue.download.segments.size.dynamic", String.valueOf(false));
110+
properties.put("queue.download.segments.size", String.valueOf(DIRECTIO_CHUNKSIZE));
111+
properties.put("queue.download.segments.threshold", String.valueOf(0));
112+
}
111113
return properties;
112114
}
113115

‎ctera/src/main/java/ch/cyberduck/core/ctera/model/DirectIO.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ public final class DirectIO {
2424

2525
public EncryptInfo encrypt_info;
2626
public List<Chunk> chunks;
27+
public ActualBlocksRange actual_blocks_range;
2728

2829
public static final class Chunk {
2930
public String url;
@@ -54,10 +55,26 @@ public String toString() {
5455
}
5556
}
5657

58+
public static class ActualBlocksRange {
59+
60+
public long file_size;
61+
public String range;
62+
63+
@Override
64+
public String toString() {
65+
final StringBuilder sb = new StringBuilder("ActualBlocksRange{");
66+
sb.append("file_size=").append(file_size);
67+
sb.append(", range='").append(range).append('\'');
68+
sb.append('}');
69+
return sb.toString();
70+
}
71+
}
72+
5773
@Override
5874
public String toString() {
5975
final StringBuilder sb = new StringBuilder("DirectIO{");
6076
sb.append("encrypt_info=").append(encrypt_info);
77+
sb.append(", actual_blocks_range=").append(actual_blocks_range);
6178
sb.append(", chunks=").append(chunks);
6279
sb.append('}');
6380
return sb.toString();

‎ctera/src/test/java/ch/cyberduck/core/ctera/AbstractCteraDirectIOTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import ch.cyberduck.core.DisabledProgressListener;
2323
import ch.cyberduck.core.Host;
2424
import ch.cyberduck.core.LoginConnectionService;
25+
import ch.cyberduck.core.preferences.PreferencesFactory;
2526
import ch.cyberduck.core.proxy.DisabledProxyFinder;
2627
import ch.cyberduck.core.ssl.DefaultX509KeyManager;
2728
import ch.cyberduck.core.ssl.DisabledX509TrustManager;
@@ -51,6 +52,7 @@ public String getProperty(final String key) {
5152
}
5253
};
5354
host.setDefaultPath("/ServicesPortal/webdav/My Files");
55+
PreferencesFactory.get().setDefault("ctera.download.directio.enable", String.valueOf(true));
5456
session = new CteraSession(host, new DisabledX509TrustManager(), new DefaultX509KeyManager(), new TestPasswordStore());
5557
final LoginConnectionService connect = new LoginConnectionService(new DisabledLoginCallback(), new DisabledHostKeyCallback(),
5658
new TestPasswordStore(), new DisabledProgressListener(), new DisabledProxyFinder());

‎ctera/src/test/java/ch/cyberduck/core/ctera/CteraDirectIOReadFeatureTest.java‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import ch.cyberduck.core.DisabledProgressListener;
2323
import ch.cyberduck.core.Local;
2424
import ch.cyberduck.core.Path;
25+
import ch.cyberduck.core.PathAttributes;
26+
import ch.cyberduck.core.dav.DAVFindFeature;
2527
import ch.cyberduck.core.dav.DAVUploadFeature;
2628
import ch.cyberduck.core.features.Delete;
2729
import ch.cyberduck.core.io.BandwidthThrottle;
@@ -44,8 +46,7 @@
4446
import java.util.Collections;
4547
import java.util.EnumSet;
4648

47-
import static org.junit.Assert.assertArrayEquals;
48-
import static org.junit.Assert.assertNotNull;
49+
import static org.junit.Assert.*;
4950

5051
@Category(IntegrationTest.class)
5152
public class CteraDirectIOReadFeatureTest extends AbstractCteraDirectIOTest {
@@ -64,7 +65,7 @@ public void testReadChunk() throws Exception {
6465
new TransferStatus().setLength(content.length),
6566
new DisabledConnectionCallback());
6667
final TransferStatus status = new TransferStatus();
67-
final TransferStatus segment = new TransferStatus().setSegment(true);
68+
final TransferStatus segment = new TransferStatus().setSegment(true).setLength(content.length);
6869
status.setSegments(Collections.singletonList(segment));
6970
final CteraBulkFeature bulk = new CteraBulkFeature(session, new DefaultVersionIdProvider(session));
7071
bulk.pre(Transfer.Type.download, Collections.singletonMap(new TransferItem(test), status), new DisabledConnectionCallback());
@@ -78,4 +79,35 @@ public void testReadChunk() throws Exception {
7879
assertArrayEquals(content, buffer.toByteArray());
7980
new CteraDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
8081
}
82+
83+
@Test
84+
public void testReadZeroByteFile() throws Exception {
85+
final Path test = new CteraTouchFeature(session).touch(new CteraWriteFeature(session), new Path(new DefaultHomeFinderService(session).find(), new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)), new TransferStatus());
86+
final Local local = new Local(System.getProperty("java.io.tmpdir"), new AlphanumericRandomStringService().random());
87+
final byte[] content = RandomUtils.nextBytes(0);
88+
final OutputStream out = local.getOutputStream(false);
89+
assertNotNull(out);
90+
IOUtils.write(content, out);
91+
out.close();
92+
new DAVUploadFeature(session).upload(
93+
new CteraWriteFeature(session), test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener(),
94+
new TransferStatus().setLength(content.length),
95+
new DisabledConnectionCallback());
96+
final TransferStatus status = new TransferStatus().setLength(content.length);
97+
status.setSegments(Collections.emptyList());
98+
final CteraBulkFeature bulk = new CteraBulkFeature(session, new DefaultVersionIdProvider(session));
99+
bulk.pre(Transfer.Type.download, Collections.singletonMap(new TransferItem(test), status), new DisabledConnectionCallback());
100+
assertNull(status.getUrl());
101+
assertNotNull(status.getParameters().get(CteraDirectIOReadFeature.CTERA_WRAPPEDKEY));
102+
assertTrue(new DAVFindFeature(session).find(test));
103+
final PathAttributes attributes = new CteraAttributesFinderFeature(session).find(test);
104+
assertEquals(content.length, attributes.getSize());
105+
final InputStream in = new CteraDirectIOReadFeature(session).read(test, status, new DisabledConnectionCallback());
106+
assertNotNull(in);
107+
final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length);
108+
new StreamCopier(status, status).transfer(in, buffer);
109+
in.close();
110+
assertArrayEquals(content, buffer.toByteArray());
111+
new CteraDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
112+
}
81113
}

0 commit comments

Comments
 (0)