Skip to content

Commit df03c63

Browse files
thu-davidthu-david
authored
Support Unit Tests for TOS
### What changes are proposed in this pull request? I have created some unit tests for TOS. ### Why are the changes needed? 1、Because it can help us check our code. ### Does this PR introduce any user facing changes? No. pr-link: #18646 change-id: cid-c4f50b6a67b7c1ee5acc9e7dbfe748eaa859a931
1 parent a4ec456 commit df03c63

File tree

5 files changed

+761
-0
lines changed

5 files changed

+761
-0
lines changed

‎underfs/tos/src/test/java/alluxio/underfs/tos/TOSInputStreamTest.java‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
3+
* (the "License"). You may not use this work except in compliance with the License, which is
4+
* available at www.apache.org/licenses/LICENSE-2.0
5+
*
6+
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
7+
* either express or implied, as more fully set forth in the License.
8+
*
9+
* See the NOTICE file distributed with this work for information regarding copyright ownership.
10+
*/
11+
12+
package alluxio.underfs.tos;
13+
14+
import static org.hamcrest.CoreMatchers.is;
15+
import static org.junit.Assert.assertArrayEquals;
16+
import static org.junit.Assert.assertEquals;
17+
import static org.mockito.ArgumentMatchers.argThat;
18+
import static org.mockito.Mockito.mock;
19+
import static org.mockito.Mockito.spy;
20+
import static org.mockito.Mockito.when;
21+
22+
import alluxio.conf.AlluxioConfiguration;
23+
import alluxio.conf.Configuration;
24+
import alluxio.conf.PropertyKey;
25+
import alluxio.retry.CountingRetry;
26+
27+
import com.volcengine.tos.TOSV2;
28+
import com.volcengine.tos.model.object.GetObjectV2Input;
29+
import com.volcengine.tos.model.object.GetObjectV2Output;
30+
import org.junit.Before;
31+
import org.junit.Rule;
32+
import org.junit.Test;
33+
import org.junit.rules.ExpectedException;
34+
35+
import java.io.ByteArrayInputStream;
36+
import java.io.IOException;
37+
import java.io.InputStream;
38+
import java.text.MessageFormat;
39+
import java.util.Arrays;
40+
41+
/**
42+
* Unit tests for the {@link TOSInputStream}.
43+
*/
44+
public class TOSInputStreamTest {
45+
46+
private static final String BUCKET_NAME = "testBucket";
47+
private static final String OBJECT_KEY = "testObjectKey";
48+
private static AlluxioConfiguration sConf = Configuration.global();
49+
50+
private TOSInputStream mTosInputStream;
51+
private TOSV2 mTosClient;
52+
private InputStream[] mInputStreamSpy;
53+
private GetObjectV2Output[] mTosObject;
54+
55+
/**
56+
* The exception expected to be thrown.
57+
*/
58+
@Rule
59+
public final ExpectedException mExceptionRule = ExpectedException.none();
60+
61+
@Before
62+
public void setUp() throws IOException {
63+
mTosClient = mock(TOSV2.class);
64+
65+
byte[] input = new byte[] {1, 2, 3};
66+
mTosObject = new GetObjectV2Output[input.length];
67+
mInputStreamSpy = new InputStream[input.length];
68+
for (int i = 0; i < input.length; ++i) {
69+
final long pos = (long) i;
70+
mTosObject[i] = mock(GetObjectV2Output.class);
71+
when(mTosClient.getObject(argThat(argument -> {
72+
if (argument instanceof GetObjectV2Input) {
73+
String range = ((GetObjectV2Input) argument).getOptions().getRange();
74+
return range.equals(MessageFormat.format("bytes={0}-", pos));
75+
}
76+
return false;
77+
}))).thenReturn(mTosObject[i]);
78+
byte[] mockInput = Arrays.copyOfRange(input, i, input.length);
79+
mInputStreamSpy[i] = spy(new ByteArrayInputStream(mockInput));
80+
when(mTosObject[i].getContent()).thenReturn(mInputStreamSpy[i]);
81+
}
82+
mTosInputStream = new TOSInputStream(BUCKET_NAME, OBJECT_KEY, mTosClient, new CountingRetry(1),
83+
sConf.getBytes(PropertyKey.UNDERFS_OBJECT_STORE_MULTI_RANGE_CHUNK_SIZE));
84+
}
85+
86+
@Test
87+
public void close() throws IOException {
88+
mTosInputStream.close();
89+
90+
mExceptionRule.expect(IOException.class);
91+
mExceptionRule.expectMessage(is("Stream closed"));
92+
mTosInputStream.read();
93+
}
94+
95+
@Test
96+
public void readInt() throws IOException {
97+
assertEquals(1, mTosInputStream.read());
98+
assertEquals(2, mTosInputStream.read());
99+
assertEquals(3, mTosInputStream.read());
100+
}
101+
102+
@Test
103+
public void readByteArray() throws IOException {
104+
byte[] bytes = new byte[3];
105+
int readCount = mTosInputStream.read(bytes, 0, 3);
106+
assertEquals(3, readCount);
107+
assertArrayEquals(new byte[] {1, 2, 3}, bytes);
108+
}
109+
110+
@Test
111+
public void skip() throws IOException {
112+
assertEquals(1, mTosInputStream.read());
113+
mTosInputStream.skip(1);
114+
assertEquals(3, mTosInputStream.read());
115+
}
116+
}

‎underfs/tos/src/test/java/alluxio/underfs/tos/TOSLowLevelOutputStreamTest.java‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
/*
2+
* The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
3+
* (the "License"). You may not use this work except in compliance with the License, which is
4+
* available at www.apache.org/licenses/LICENSE-2.0
5+
*
6+
* This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
7+
* either express or implied, as more fully set forth in the License.
8+
*
9+
* See the NOTICE file distributed with this work for information regarding copyright ownership.
10+
*/
11+
12+
package alluxio.underfs.tos;
13+
14+
import static org.junit.Assert.assertEquals;
15+
import static org.junit.Assert.assertTrue;
16+
import static org.mockito.ArgumentMatchers.any;
17+
import static org.mockito.Mockito.never;
18+
import static org.mockito.Mockito.times;
19+
import static org.mockito.Mockito.when;
20+
21+
import alluxio.conf.Configuration;
22+
import alluxio.conf.InstancedConfiguration;
23+
import alluxio.conf.PropertyKey;
24+
25+
import com.google.common.util.concurrent.ListenableFuture;
26+
import com.google.common.util.concurrent.ListeningExecutorService;
27+
import com.volcengine.tos.TOSV2;
28+
import com.volcengine.tos.model.object.CompleteMultipartUploadV2Input;
29+
import com.volcengine.tos.model.object.CompleteMultipartUploadV2Output;
30+
import com.volcengine.tos.model.object.CreateMultipartUploadInput;
31+
import com.volcengine.tos.model.object.CreateMultipartUploadOutput;
32+
import com.volcengine.tos.model.object.PutObjectInput;
33+
import com.volcengine.tos.model.object.PutObjectOutput;
34+
import com.volcengine.tos.model.object.UploadPartV2Input;
35+
import com.volcengine.tos.model.object.UploadPartV2Output;
36+
import com.volcengine.tos.model.object.UploadedPartV2;
37+
import org.junit.Before;
38+
import org.junit.Test;
39+
import org.junit.runner.RunWith;
40+
import org.mockito.Mockito;
41+
import org.mockito.invocation.InvocationOnMock;
42+
import org.mockito.stubbing.Answer;
43+
import org.powermock.api.mockito.PowerMockito;
44+
import org.powermock.core.classloader.annotations.PrepareForTest;
45+
import org.powermock.modules.junit4.PowerMockRunner;
46+
47+
import java.util.concurrent.Callable;
48+
49+
/**
50+
* Unit tests for the {@link TOSLowLevelOutputStream}.
51+
*/
52+
@RunWith(PowerMockRunner.class)
53+
@PrepareForTest(TOSLowLevelOutputStream.class)
54+
public class TOSLowLevelOutputStreamTest {
55+
private static final String BUCKET_NAME = "testBucket";
56+
private static final String PARTITION_SIZE = "8MB";
57+
private static final String KEY = "testKey";
58+
private static final String UPLOAD_ID = "testUploadId";
59+
private InstancedConfiguration mConf = Configuration.copyGlobal();
60+
61+
private TOSV2 mMockTosClient;
62+
private ListeningExecutorService mMockExecutor;
63+
private ListenableFuture<UploadedPartV2> mMockTag;
64+
private TOSLowLevelOutputStream mStream;
65+
66+
@Before
67+
public void before() throws Exception {
68+
mockTOSClientAndExecutor();
69+
mConf.set(PropertyKey.UNDERFS_TOS_STREAMING_UPLOAD_PARTITION_SIZE, PARTITION_SIZE);
70+
mConf.set(PropertyKey.UNDERFS_TOS_STREAMING_UPLOAD_ENABLED, "true");
71+
mStream = new TOSLowLevelOutputStream(BUCKET_NAME, KEY, mMockTosClient, mMockExecutor, mConf);
72+
}
73+
74+
@Test
75+
public void writeByte() throws Exception {
76+
mStream.write(1);
77+
78+
mStream.close();
79+
Mockito.verify(mMockExecutor, never()).submit(any(Callable.class));
80+
Mockito.verify(mMockTosClient).putObject(any(PutObjectInput.class));
81+
Mockito.verify(mMockTosClient, never())
82+
.createMultipartUpload(any(CreateMultipartUploadInput.class));
83+
Mockito.verify(mMockTosClient, never()).completeMultipartUpload(any(
84+
CompleteMultipartUploadV2Input.class));
85+
assertTrue(mStream.getContentHash().isPresent());
86+
assertEquals("putTag", mStream.getContentHash().get());
87+
}
88+
89+
@Test
90+
public void writeByteArrayForSmallFile() throws Exception {
91+
int partSize = (int) (8 * 1024 * 1024); // 8MB
92+
byte[] b = new byte[partSize];
93+
94+
mStream.write(b, 0, b.length);
95+
96+
mStream.close();
97+
Mockito.verify(mMockExecutor, never()).submit(any(Callable.class));
98+
Mockito.verify(mMockTosClient).putObject(any(PutObjectInput.class));
99+
Mockito.verify(mMockTosClient, never())
100+
.createMultipartUpload(any(CreateMultipartUploadInput.class));
101+
Mockito.verify(mMockTosClient, never())
102+
.completeMultipartUpload(any(CompleteMultipartUploadV2Input.class));
103+
assertTrue(mStream.getContentHash().isPresent());
104+
assertEquals("putTag", mStream.getContentHash().get());
105+
}
106+
107+
@Test
108+
public void writeByteArrayForLargeFile() throws Exception {
109+
int partSize = (int) (8 * 1024 * 1024); // 8MB
110+
byte[] b = new byte[partSize + 1];
111+
112+
mStream.write(b, 0, b.length);
113+
114+
mStream.close();
115+
Mockito.verify(mMockTosClient).createMultipartUpload(any(CreateMultipartUploadInput.class));
116+
Mockito.verify(mMockExecutor, times(2)).submit(any(Callable.class));
117+
Mockito.verify(mMockTosClient)
118+
.completeMultipartUpload(any(CompleteMultipartUploadV2Input.class));
119+
assertTrue(mStream.getContentHash().isPresent());
120+
assertEquals("multiTag", mStream.getContentHash().get());
121+
}
122+
123+
@Test
124+
public void createEmptyFile() throws Exception {
125+
mStream.close();
126+
Mockito.verify(mMockExecutor, never()).submit(any(Callable.class));
127+
Mockito.verify(mMockTosClient, never())
128+
.createMultipartUpload(any(CreateMultipartUploadInput.class));
129+
Mockito.verify(mMockTosClient, never())
130+
.completeMultipartUpload(any(CompleteMultipartUploadV2Input.class));
131+
Mockito.verify(mMockTosClient).putObject(any(PutObjectInput.class));
132+
assertTrue(mStream.getContentHash().isPresent());
133+
assertEquals("emptyTag", mStream.getContentHash().get());
134+
}
135+
136+
@Test
137+
public void flush() throws Exception {
138+
int partSize = (int) (8 * 1024 * 1024); // 8MB
139+
byte[] b = new byte[2 * partSize - 1];
140+
141+
mStream.write(b, 0, b.length);
142+
143+
mStream.flush();
144+
Mockito.verify(mMockExecutor, times(2)).submit(any(Callable.class));
145+
Mockito.verify(mMockTag, times(2)).get();
146+
147+
mStream.close();
148+
Mockito.verify(mMockTosClient)
149+
.completeMultipartUpload(any(CompleteMultipartUploadV2Input.class));
150+
assertTrue(mStream.getContentHash().isPresent());
151+
assertEquals("multiTag", mStream.getContentHash().get());
152+
}
153+
154+
@Test
155+
public void close() throws Exception {
156+
mStream.close();
157+
Mockito.verify(mMockTosClient, never())
158+
.createMultipartUpload(any(CreateMultipartUploadInput.class));
159+
Mockito.verify(mMockTosClient, never())
160+
.completeMultipartUpload(any(CompleteMultipartUploadV2Input.class));
161+
assertTrue(mStream.getContentHash().isPresent());
162+
assertEquals("emptyTag", mStream.getContentHash().get());
163+
}
164+
165+
private void mockTOSClientAndExecutor() throws Exception {
166+
mMockTosClient = PowerMockito.mock(TOSV2.class);
167+
168+
CreateMultipartUploadOutput createOutput = new CreateMultipartUploadOutput();
169+
createOutput.setUploadID(UPLOAD_ID);
170+
when(mMockTosClient.createMultipartUpload(any(CreateMultipartUploadInput.class)))
171+
.thenReturn(createOutput);
172+
173+
UploadPartV2Output uploadPartOutput = new UploadPartV2Output();
174+
uploadPartOutput.setEtag("partTag");
175+
when(mMockTosClient.uploadPart(any(UploadPartV2Input.class))).thenReturn(uploadPartOutput);
176+
177+
// Use Answer to dynamically return PutObjectOutput based on the input
178+
when(mMockTosClient.putObject(any(PutObjectInput.class)))
179+
.thenAnswer(new Answer<PutObjectOutput>() {
180+
@Override
181+
public PutObjectOutput answer(InvocationOnMock invocation) throws Throwable {
182+
PutObjectInput input = invocation.getArgument(0);
183+
PutObjectOutput output = new PutObjectOutput();
184+
// Determine the Etag value based on the input condition
185+
if (input.getContentLength() == 0) {
186+
output.setEtag("emptyTag");
187+
} else {
188+
output.setEtag("putTag");
189+
}
190+
return output;
191+
}
192+
});
193+
194+
CompleteMultipartUploadV2Output completeOutput = new CompleteMultipartUploadV2Output();
195+
completeOutput.setEtag("multiTag");
196+
when(mMockTosClient.completeMultipartUpload(any(CompleteMultipartUploadV2Input.class)))
197+
.thenReturn(completeOutput);
198+
199+
mMockTag = (ListenableFuture<UploadedPartV2>) PowerMockito.mock(ListenableFuture.class);
200+
when(mMockTag.get()).thenReturn(new UploadedPartV2().setPartNumber(1).setEtag("partTag"));
201+
mMockExecutor = Mockito.mock(ListeningExecutorService.class);
202+
when(mMockExecutor.submit(any(Callable.class))).thenReturn(mMockTag);
203+
}
204+
}

0 commit comments

Comments
 (0)