minor tweaks, dependency update fix, add integration test script

drm · drm · commit d86fa8960ddd · 2021-01-08T17:22:59.000+01:00
diff --git a/lib/fetch.sh b/lib/fetch.sh
@@ -42,12 +42,13 @@ fetch() {
 
 	fetch_maven_deps \
 		'org.apache.httpcomponents:httpclient:4.5.12' \
+		'commons-cli:commons-cli:1.4' \
 		'junit:junit:4.12' \
 		'org.slf4j:slf4j-log4j12:1.7.7' \
 		'org.jsoup:jsoup:1.11.3'
 
 	curl -sL https://github.com/drm/java-redis-client/releases/download/v2.1.0/java-redis-client-v2.1.0--javac-11.0.9.1.jar -o ./java-redis-client-v2.1.0.jar
-	curl -sL https://github.com/drm/java-redis-collections/releases/download/v1.1.0/java-redis-collections-v1.1.0--javac-11.0.9.1.jar -o ./java-redis-collections-v1.1.0.jar
+	curl -sL https://github.com/drm/java-redis-collections/releases/download/v1.1.1/java-redis-collections-v1.1.1--javac-11.0.9.1.jar -o ./java-redis-collections-v1.1.1.jar
 }
 
 clean() {
diff --git a/lib/java-redis-collections-v1.1.0.jar b/lib/java-redis-collections-v1.1.0.jar
diff --git a/log4j.debugging.properties b/log4j.debugging.properties
@@ -14,7 +14,7 @@ log4j.appender.fileName.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
 
 ### By default everything that is of warning or severity WARN, ERROR or FATAL is logged both to
 ### the console and to the log fileName.
-log4j.rootLogger=trace, stdout, fileName
+log4j.rootLogger=info, stdout, fileName
 
-log4j.logger.nl.melp.linkchecker=trace
+# log4j.logger.nl.melp.linkchecker=trace
 
diff --git a/resources/sample-2/all-fixed/foo.html b/resources/sample-2/all-fixed/foo.html
@@ -0,0 +1 @@
+foo!
diff --git a/resources/sample-2/all-fixed/index.html b/resources/sample-2/all-fixed/index.html
@@ -0,0 +1,7 @@
+<html>
+<p>Hi, welcome to this site. All broken links were removed:</p>
+<p><a href="/foo.html">Foo does exist now</a></p>
+<p><a href="https://example.org/">Does-exist domain</a></p>
+<p>Invalid host</p>
+<p>Has no valid protocol</p>
+</html>
diff --git a/resources/sample-2/with-broken-1/index.html b/resources/sample-2/with-broken-1/index.html
@@ -0,0 +1,7 @@
+<html>
+<p>Hi, welcome to this site. This page contains a few broken links:</p>
+<p><a href="/foo.html">Foo does not exist</a></p>
+<p><a href="https://thisisaninvaliddomainthatshouldnotresultinanythingsensible000.com">Non-existent domain</a></p>
+<p><a href="https://invalid_host/baz">Invalid host</a></p>
+<p><a href="unknown:protocol">Has no valid protocol</a></p>
+</html>
diff --git a/resources/sample-2/with-broken-2/index.html b/resources/sample-2/with-broken-2/index.html
@@ -0,0 +1,7 @@
+<html>
+<p>Hi, welcome to this site. All broken links were removed:</p>
+<p><a href="/foo.html">Foo does exist now</a></p>
+<p><a href="https://example.org/">Does-exist domain</a></p>
+<p>Invalid host</p>
+<p>Has no valid protocol</p>
+</html>
diff --git a/run.sh b/run.sh
@@ -1,4 +1,4 @@
-./build.sh
+#./build.sh
 
 java \
     -Xmx512M \
diff --git a/src/nl/melp/linkchecker/LogMonitor.java b/src/nl/melp/linkchecker/LogMonitor.java
@@ -30,8 +30,8 @@ public void log() {
 
 			logger.info(
 				String.format(
-					"processed: %d, processing: %d, to check: %d; (run time %ds, avg %d/s, mem usage: %d MB of %d MB (%.2f%%))",
-					size,
+					"[%.1f%%] (%d of %d remaining); (run time %ds, avg %d/s, mem usage: %d MB of %d MB (%.2f%%))",
+					((float)size / status.numQueueud()) * 100,
 					status.numPending(),
 					status.numQueueud(),
 					dt,
diff --git a/src/nl/melp/linkchecker/RunConfig.java b/src/nl/melp/linkchecker/RunConfig.java
@@ -3,12 +3,7 @@
 import nl.melp.redis.Redis;
 import nl.melp.redis.collections.ISerializer;
 import nl.melp.redis.collections.SerializedHashMap;
-import nl.melp.redis.collections.SerializedMappedSet;
-import nl.melp.redis.collections.SerializedSortedSet;
-import nl.melp.redis.collections.Serializer;
-import nl.melp.redis.collections.Serializer.StringSerializer;
 import nl.melp.redis.collections.Serializers;
-import org.apache.http.HttpEntity;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
 import org.apache.http.conn.socket.PlainConnectionSocketFactory;
diff --git a/src/nl/melp/linkchecker/backend/InMemory.java b/src/nl/melp/linkchecker/backend/InMemory.java
@@ -18,6 +18,7 @@
 public class InMemory extends Status {
 	private static class MappedSet<K, V> extends ConcurrentHashMap<K, Set<V>> {
 		@Override
+		@SuppressWarnings("unchecked")
 		public Set<V> get(Object key) {
 			Object ret = super.get(key);
 			if (ret == null) {
@@ -86,8 +87,6 @@ protected InMemory(Logger logger, RunConfig config, Map<URI, Integer> statuses,
 		super(logger, config, statuses, urls, reverseLinks, invalidUrls);
 	}
 
-
-
 	@Override
 	public synchronized void add(Result fetch) {
 		super.add(fetch);
diff --git a/test.sh b/test.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+PORT=8000
+
+ROOT="$(cd $(dirname "$0") && pwd)"
+#set -e -u -x
+set -e -u
+
+export SERVER_PID=""
+
+_kill() {
+	echo "SERVER_PID=$SERVER_PID";
+	local server_pid="$SERVER_PID"
+	while [ "$server_pid" != "" ]; do
+		echo "Killing $server_pid"
+		local server_pid="$(ps -o pid= "$server_pid")"
+		if [ "$server_pid" != "" ]; then
+			echo -n "."
+			kill $server_pid;
+			sleep .5
+		fi
+	done
+	echo ""
+}
+
+_serve() {
+	cd "$1";
+	_kill
+	dir="$1"
+	python -m SimpleHTTPServer &
+	SERVER_PID="$!"
+	while ! nc -z 127.0.0.1 "$PORT"; do
+		echo -n "."; sleep .5;
+	done
+	echo "$SERVER_PID running in $1"
+	cd $ROOT
+}
+
+_serve resources/sample-2/with-broken-1
+./run.sh --reset "http://localhost:$PORT/" --report --follow-from-local
+_kill
+
+_serve resources/sample-2/with-broken-2
+./run.sh --reset "http://localhost:$PORT/" --report --follow-from-local
+_kill
+
+_serve resources/sample-2/all-fixed
+./run.sh --recheck --report --report-all --follow-from-local
+_kill
+
diff --git a/test/nl/melp/linkchecker/LinkCheckerTest.java b/test/nl/melp/linkchecker/LinkCheckerTest.java
@@ -105,21 +105,6 @@ public Map<URI, AtomicInteger> getFetchCounts() {
 		}
 	}
 
-	public static LinkChecker createLinkChecker(String... args) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
-		RunConfig config = new MockRunConfig(args);
-		Status status = new MockStatus(config);
-		return new LinkChecker(
-			config,
-			status,
-			new Fetcher(
-				logger,
-				config,
-				new HtmlExtractor(logger),
-				new URIResolver(logger)
-			)
-		);
-	}
-
 	@Test
 	public void testRunInMemory() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException, InterruptedException {
 		MockStatus initialState = new MockStatus(new MockRunConfig());

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-./build.sh`
	`1`	`+#./build.sh`
`2`	`2`
`3`	`3`	`java \`
`4`	`4`	`-Xmx512M \`