make tests faster

lecousin · lecousin · commit cb61237ec928 · 2018-01-08T07:28:05.000+01:00
diff --git a/net.lecousin.core/src/main/java/net/lecousin/framework/application/StandaloneLCCore.java b/net.lecousin.core/src/main/java/net/lecousin/framework/application/StandaloneLCCore.java
@@ -18,6 +18,7 @@
 import net.lecousin.framework.concurrent.Threading;
 import net.lecousin.framework.concurrent.synch.ISynchronizationPoint;
 import net.lecousin.framework.log.Logger;
+import net.lecousin.framework.memory.MemoryManager;
 import net.lecousin.framework.util.AsyncCloseable;
 import net.lecousin.framework.util.DebugUtil;
 import net.lecousin.framework.util.Pair;
@@ -242,6 +243,11 @@ public void stop() {
 		} while (count < 50);
 
 		long end = System.nanoTime();
+		
+		StringBuilder s = new StringBuilder(2048);
+		MemoryManager.logMemory(s);
+		System.out.println(s.toString());
+		
 		OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean();
 		try {
 			ClassLoader.getSystemClassLoader().loadClass("com.sun.management.OperatingSystemMXBean");
diff --git a/net.lecousin.core/src/main/java/net/lecousin/framework/io/LinkedIO.java b/net.lecousin.core/src/main/java/net/lecousin/framework/io/LinkedIO.java
@@ -872,8 +872,7 @@ protected long seekSync(SeekType type, long move) throws IOException {
 				if (sizes.get(i) == null) {
 					@SuppressWarnings("resource")
 					IO.Readable.Seekable io = (IO.Readable.Seekable)ios.get(i);
-					io.seekSync(SeekType.FROM_END, 0);
-					sizes.set(ioIndex,  Long.valueOf(io.getPosition()));
+					sizes.set(ioIndex,  Long.valueOf(io.seekSync(SeekType.FROM_END, 0)));
 				}
 				p += sizes.get(i).longValue();
 			}
@@ -895,9 +894,7 @@ protected long seekSync(SeekType type, long move) throws IOException {
 			if (s == null) {
 				@SuppressWarnings("resource")
 				IO.Readable.Seekable io = (IO.Readable.Seekable)ios.get(ioIndex);
-				io.seekSync(SeekType.FROM_END, 0);
-				s = Long.valueOf(io.getPosition());
-				sizes.set(ioIndex,  s);
+				sizes.set(ioIndex, s = Long.valueOf(io.seekSync(SeekType.FROM_END, 0)));
 			}
 			if (pos + s.longValue() > move) {
 				posInIO = move - pos;
@@ -924,8 +921,7 @@ protected int readSync(long pos, ByteBuffer buffer) throws IOException {
 			Long s = sizes.get(i);
 			if (s == null) {
 				IO.Readable.Seekable io = (IO.Readable.Seekable)ios.get(i);
-				io.seekSync(SeekType.FROM_END, 0);
-				s = Long.valueOf(io.getPosition());
+				s = Long.valueOf(io.seekSync(SeekType.FROM_END, 0));
 				sizes.set(i,  s);
 			}
 			if (p + s.longValue() > pos) {
@@ -942,11 +938,46 @@ protected int readSync(long pos, ByteBuffer buffer) throws IOException {
 		return -1;
 	}
 
+	@SuppressWarnings("resource")
 	protected AsyncWork<Integer, IOException> readAsync(
 		long pos, ByteBuffer buffer, RunnableWithParameter<Pair<Integer,IOException>> ondone
 	) {
-		// TODO better
-		return operation(IOUtil.readAsyncUsingSync((IO.Readable.Seekable)this, pos, buffer, ondone).getOutput());
+		long p = 0;
+		int i = 0;
+		while (i < ios.size()) {
+			Long s = sizes.get(i);
+			if (s == null) {
+				IO.Readable.Seekable io = (IO.Readable.Seekable)ios.get(i);
+				AsyncWork<Long, IOException> seek = io.seekAsync(SeekType.FROM_END, 0);
+				if (!seek.isUnblocked()) {
+					AsyncWork<Integer, IOException> result = new AsyncWork<>();
+					int ii = i;
+					seek.listenAsync(new Task.Cpu.FromRunnable("LinkedIO.readAsync", getPriority(), () -> {
+						sizes.set(ii, seek.getResult());
+						readAsync(pos, buffer, ondone).listenInline(result);
+					}), result);
+					return result;
+				}
+				sizes.set(i, s = seek.getResult());
+			}
+			if (p + s.longValue() > pos) {
+				IO.Readable.Seekable io = (IO.Readable.Seekable)ios.get(i);
+				ioIndex = i;
+				posInIO = pos - p;
+				AsyncWork<Integer, IOException> result = io.readAsync(pos - p, buffer, (res) -> {
+					if (res.getValue1() != null) {
+						posInIO += res.getValue1().intValue();
+						LinkedIO.this.pos = pos + res.getValue1().intValue();
+					}
+					if (ondone != null) ondone.run(res);
+				});
+				return result;
+			}
+			p += s.longValue();
+			i++;
+		}
+		if (ondone != null) ondone.run(new Pair<>(Integer.valueOf(-1), null));
+		return new AsyncWork<>(Integer.valueOf(-1), null);
 	}
 
 	protected int readFullySync(long pos, ByteBuffer buffer) throws IOException {
diff --git a/net.lecousin.core/src/main/java/net/lecousin/framework/io/buffering/BufferedIO.java b/net.lecousin.core/src/main/java/net/lecousin/framework/io/buffering/BufferedIO.java
@@ -730,12 +730,15 @@ protected int read(byte[] buf, int offset, int len) throws IOException {
 	}
 	
 	protected int readSync(long pos, ByteBuffer buf) throws IOException {
-		if (pos >= size) return -1;
+		if (pos >= size)
+			return -1;
 		int bufferIndex = getBufferIndex(pos);
 		Buffer buffer = useBufferSync(bufferIndex);
 		int start = getBufferOffset(pos);
 		int len = buf.remaining();
 		if (len > buffer.len - start) len = buffer.len - start;
+		if (len <= 0)
+			return 0;
 		buf.put(buffer.buffer, start, len);
 		buffer.lastRead = System.currentTimeMillis();
 		buffer.inUse--;
diff --git a/net.lecousin.core/src/main/java/net/lecousin/framework/memory/MemoryManager.java b/net.lecousin.core/src/main/java/net/lecousin/framework/memory/MemoryManager.java
@@ -223,10 +223,15 @@ public static void logManageableContent() {
 	/** Log memory usage. */
 	public static void logMemory(Level level) {
 		//if (jvm != null) jvm.printOptions();
+		StringBuilder s = new StringBuilder(2048);
+		logMemory(s);
+		logger.log(level, s.toString());
+	}
+	
+	public static void logMemory(StringBuilder s) {
 		long total = Runtime.getRuntime().totalMemory();
 		long free = Runtime.getRuntime().freeMemory();
 		long used = total - free;
-		StringBuilder s = new StringBuilder(2048);
 		s.append("Memory usage: ").append(StringUtil.size(used)).append('/').append(StringUtil.size(total))
 			.append(" (").append(StringUtil.size(free)).append(" free), max=")
 			.append(StringUtil.size(Runtime.getRuntime().maxMemory())).append('\n');
@@ -256,7 +261,6 @@ public static void logMemory(Level level) {
 			}
 			s.append("]\n");
 		}
-		logger.log(level, s.toString());
 	}
 	
 	/** Free memory on memory manageable instances. */
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/collections/maps/TestLongMap.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/collections/maps/TestLongMap.java
@@ -42,7 +42,7 @@ public void testLongMapRandom() {
 		LinkedList<Long> values = new LinkedList<>();
 		Random rand = new Random();
 		long value = 0;
-		for (int i = 0; i < 5000; ++i) {
+		for (int i = 0; i < 2500; ++i) {
 			value += rand.nextLong();
 			values.add(Long.valueOf(value));
 		}
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/collections/sort/TestSortedAssociatedWithInteger.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/collections/sort/TestSortedAssociatedWithInteger.java
@@ -84,7 +84,7 @@ public void testAddRemoveRandom() {
 		Sorted.AssociatedWithInteger<Object> list = createSorted();
 		TreeSet<Integer> order = new TreeSet<>();
 		Random rand = new Random();
-		for (int i = 0; i < 5000; ++i) {
+		for (int i = 0; i < 2000; ++i) {
 			int value = rand.nextInt();
 			while (order.contains(Integer.valueOf(value))) value++;
 			list.add(value, Integer.valueOf(-value));
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/collections/sort/TestSortedAssociatedWithLong.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/collections/sort/TestSortedAssociatedWithLong.java
@@ -84,7 +84,7 @@ public void testAddRemoveRandom() {
 		Sorted.AssociatedWithLong<Object> list = createSorted();
 		TreeSet<Long> order = new TreeSet<>();
 		Random rand = new Random();
-		for (int i = 0; i < 5000; ++i) {
+		for (int i = 0; i < 2000; ++i) {
 			long value = rand.nextLong();
 			while (order.contains(Long.valueOf(value))) value++;
 			list.add(value, Long.valueOf(-value));
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/io/TestFragmented.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/io/TestFragmented.java
@@ -16,6 +16,7 @@ public static class FragmentedFile {
 		public byte[] testBuf;
 		public int nbBuf;
 		public File file;
+		public int realSize;
 		public List<RangeLong> fragments = new LinkedList<RangeLong>();
 	}
 	
@@ -39,11 +40,11 @@ private static synchronized void generateFragmentedFiles() throws IOException {
 	
 	private static void generateFragmentedFile(FragmentedFile f) throws IOException {
 		int size = f.testBuf.length * f.nbBuf;
-		int realSize = size * 3;
-		System.out.println("Generating fragmented file of " + f.nbBuf + " buffers (size = " + size + ", real = " + realSize + ")");
+		f.realSize = size * 3;
+		System.out.println("Generating fragmented file of " + f.nbBuf + " buffers (size = " + size + ", real = " + f.realSize + ")");
 		RandomAccessFile file = new RandomAccessFile(f.file, "rw");
-		file.setLength(realSize);
-		FragmentedRangeInteger free = new FragmentedRangeInteger(new RangeInteger(0, realSize));
+		file.setLength(f.realSize);
+		FragmentedRangeInteger free = new FragmentedRangeInteger(new RangeInteger(0, f.realSize - 1));
 		int pos = 0;
 		int maxWriteLen;
 		if (size < 65536) maxWriteLen = 1024;
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/io/TestReadWrite.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/io/TestReadWrite.java
@@ -72,6 +72,8 @@ public <T extends IO.Readable.Seekable & IO.Writable.Seekable> void testWriteThe
 	@Test(timeout=120000)
 	public <T extends IO.Readable.Seekable & IO.Writable.Seekable> void testRandomWriteAndReadAsync() throws Exception {
 		Assume.assumeTrue(nbBuf > 0);
+		int nbBuf = this.nbBuf;
+		if (nbBuf > 25000) nbBuf = 25000; // make test faster
 		T io = openReadWrite();
 		LinkedList<Integer> buffersToWrite = new LinkedList<>();
 		LinkedList<Integer> buffersToRead = new LinkedList<>();
@@ -84,7 +86,7 @@ public <T extends IO.Readable.Seekable & IO.Writable.Seekable> void testRandomWr
 				prevWrite = writeBuffer(io, prevWrite, buffersToWrite, buffersToRead);
 			prevRead = readBuffer(io, prevWrite, prevRead, buffersToRead);
 			// make some pauses to avoid stack overflow
-			if ((i % 10) == 8) {
+			if ((i % 17) == 8) {
 				prevRead.blockThrow(0);
 				prevRead = null;
 				prevWrite = null;
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/io/TestReadableSeekable.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/test/io/TestReadableSeekable.java
@@ -48,9 +48,9 @@ public void testSeekableByteByByteSync() throws Exception {
 		for (int i = 0; i < nbBuf; ++i) offsets.add(Integer.valueOf(i));
 		boolean faster = false;
 		while (!offsets.isEmpty()) {
-			if (nbBuf > 1000 && (offsets.size() % 100) == 99) {
+			if (nbBuf > 1000 && (offsets.size() % 100) == 50) {
 				// make the test faster
-				for (int skip = 0; skip < 80 && !offsets.isEmpty(); ++skip)
+				for (int skip = 0; skip < 120 && !offsets.isEmpty(); ++skip)
 					offsets.remove(rand.nextInt(offsets.size()));
 				continue;
 			}
@@ -136,9 +136,9 @@ public void run() {
 							// read again to test we cannot read beyond the end of the file
 							offset.set(nbBuf-1);
 						} else {
-							if (nbBuf > 1000 && (offsets.size() % 100) == 99) {
+							if (nbBuf > 1000 && (offsets.size() % 100) == 60) {
 								// make the test faster
-								for (int skip = 0; skip < 80 && offsets.size() > 1; ++skip)
+								for (int skip = 0; skip < 130 && offsets.size() > 1; ++skip)
 									offsets.remove(rand.nextInt(offsets.size()));
 							}
 	
@@ -272,7 +272,7 @@ public void run() {
 	@Test(timeout=300000)
 	public void testConcurrentAccessToSeekableBufferByBufferFullyAsync() throws Exception {
 		IO.Readable.Seekable io = createReadableSeekableFromFile(openFile(), getFileSize());
-		int nbConc = Runtime.getRuntime().availableProcessors() * 5;
+		int nbConc = Runtime.getRuntime().availableProcessors() * 3;
 		JoinPoint<Exception> jp = new JoinPoint<>();
 		jp.addToJoin(nbConc);
 		for (int t = 0; t < nbConc; ++t) {
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestFragmentedSubIOReadableSeekable.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestFragmentedSubIOReadableSeekable.java
@@ -8,6 +8,7 @@
 import org.junit.runners.Parameterized.Parameters;
 
 import net.lecousin.framework.io.FileIO.ReadOnly;
+import net.lecousin.framework.io.buffering.BufferedIO;
 import net.lecousin.framework.core.test.io.TestFragmented;
 import net.lecousin.framework.core.test.io.TestReadableSeekable;
 import net.lecousin.framework.core.test.io.TestFragmented.FragmentedFile;
@@ -31,7 +32,9 @@ public TestFragmentedSubIOReadableSeekable(FragmentedFile f) {
 
 	@Override
 	protected IO.Readable.Seekable createReadableSeekableFromFile(ReadOnly file, long fileSize) throws Exception {
-		return new FragmentedSubIO.Readable(file, f.fragments, true, "fragmented IO");
+		// this test may be very slow, let's add a buffered layer
+		BufferedIO.ReadOnly buffered = new BufferedIO.ReadOnly(file, 32768, f.realSize);
+		return new FragmentedSubIO.Readable(buffered, f.fragments, true, "fragmented IO");
 	}
 	
 }
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestLinkedIOWithSubIOReadableSeekable.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestLinkedIOWithSubIOReadableSeekable.java
@@ -14,6 +14,7 @@
 import net.lecousin.framework.io.IO;
 import net.lecousin.framework.io.LinkedIO;
 import net.lecousin.framework.io.SubIO;
+import net.lecousin.framework.io.buffering.BufferedIO;
 import net.lecousin.framework.math.RangeLong;
 
 @RunWith(Parameterized.class)
@@ -33,11 +34,17 @@ public TestLinkedIOWithSubIOReadableSeekable(FragmentedFile f) {
 
 	@Override
 	protected IO.Readable.Seekable createReadableSeekableFromFile(FileIO.ReadOnly file, long fileSize) throws Exception {
+		// this test may be very slow, let's add a buffered layer
+		BufferedIO.ReadOnly buffered = new BufferedIO.ReadOnly(file, 32768, f.realSize);
 		IO.Readable.Seekable[] ios = new IO.Readable.Seekable[f.fragments.size()];
 		int i = 0;
 		for (RangeLong fragment : f.fragments)
-			ios[i++] = new SubIO.Readable.Seekable(file, fragment.min, fragment.getLength(), "fragment " + i, false);
-		return new LinkedIO.Readable.Seekable("linked IO", ios);
+			ios[i++] = new SubIO.Readable.Seekable(buffered, fragment.min, fragment.getLength(), "fragment " + i, false);
+		LinkedIO.Readable.Seekable res = new LinkedIO.Readable.Seekable("linked IO", ios);
+		res.addCloseListener(() -> {
+			buffered.closeAsync();
+		});
+		return res;
 	}
 	
 	@Override
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestReadableToSeekableReadable.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestReadableToSeekableReadable.java
@@ -30,8 +30,8 @@ public TestReadableToSeekableReadable(File testFile, byte[] testBuf, int nbBuf)
 	@SuppressWarnings("resource")
 	@Override
 	protected IO.Readable.Seekable createReadableFromFile(ReadOnly file, long fileSize) throws Exception {
-		PreBufferedReadable ioBuf = new PreBufferedReadable(file, 256, Task.PRIORITY_IMPORTANT, 512, Task.PRIORITY_IMPORTANT, 10);
-		return new ReadableToSeekable(ioBuf, 512);
+		PreBufferedReadable ioBuf = new PreBufferedReadable(file, 256, Task.PRIORITY_IMPORTANT, 2048, Task.PRIORITY_IMPORTANT, 10);
+		return new ReadableToSeekable(ioBuf, 4096);
 	}
 	
 }
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestReadableToSeekableReadableBuffered.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestReadableToSeekableReadableBuffered.java
@@ -30,8 +30,8 @@ public TestReadableToSeekableReadableBuffered(File testFile, byte[] testBuf, int
 	@SuppressWarnings("resource")
 	@Override
 	protected IO.Readable.Buffered createReadableBufferedFromFile(ReadOnly file, long fileSize) throws Exception {
-		PreBufferedReadable ioBuf = new PreBufferedReadable(file, 256, Task.PRIORITY_IMPORTANT, 512, Task.PRIORITY_IMPORTANT, 10);
-		return new ReadableToSeekable(ioBuf, 512);
+		PreBufferedReadable ioBuf = new PreBufferedReadable(file, 256, Task.PRIORITY_IMPORTANT, 2048, Task.PRIORITY_IMPORTANT, 10);
+		return new ReadableToSeekable(ioBuf, 4096);
 	}
 	
 }
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestReadableToSeekableReadableSeekable.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestReadableToSeekableReadableSeekable.java
@@ -30,8 +30,8 @@ public TestReadableToSeekableReadableSeekable(File testFile, byte[] testBuf, int
 	@SuppressWarnings("resource")
 	@Override
 	protected IO.Readable.Seekable createReadableSeekableFromFile(ReadOnly file, long fileSize) throws Exception {
-		PreBufferedReadable ioBuf = new PreBufferedReadable(file, 256, Task.PRIORITY_IMPORTANT, 512, Task.PRIORITY_IMPORTANT, 10);
-		return new ReadableToSeekable(ioBuf, 512);
+		PreBufferedReadable ioBuf = new PreBufferedReadable(file, 256, Task.PRIORITY_IMPORTANT, 2048, Task.PRIORITY_IMPORTANT, 10);
+		return new ReadableToSeekable(ioBuf, 4096);
 	}
 	
 }
diff --git a/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestSubIOReadableSeekable.java b/net.lecousin.core/src/test/java/net/lecousin/framework/core/tests/io/TestSubIOReadableSeekable.java
@@ -21,8 +21,9 @@ public static Collection<Object[]> parameters() {
 		Collection<Object[]> base = generateTestCases(false);
 		LinkedList<Object[]> list = new LinkedList<>();
 		for (Object[] params : base) {
-			list.add(createParams(params, 0, 0));
 			int nbBuf = ((Integer)params[2]).intValue();
+			if (nbBuf <= 1000)
+				list.add(createParams(params, 0, 0));
 			if (nbBuf < 80) continue;
 			if (nbBuf <= 100)
 				list.add(createParams(params, 60, 15));

Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,7 @@ public void testLongMapRandom() {`
`42`	`42`	`LinkedList<Long> values = new LinkedList<>();`
`43`	`43`	`Random rand = new Random();`
`44`	`44`	`long value = 0;`
`45`		`- for (int i = 0; i < 5000; ++i) {`
	`45`	`+ for (int i = 0; i < 2500; ++i) {`
`46`	`46`	`value += rand.nextLong();`
`47`	`47`	`values.add(Long.valueOf(value));`
`48`	`48`	`}`