test: Improve test framework

Various improvements to lcov's self-test framework:
 - Add test case for lcov --diff
 - Add new verbosity level
 - Enable normalization of coverage data files from stdin
 - Fix lcov_add_concatenated4 test name

Signed-off-by: Peter Oberparleiter <[email protected]>

Author: oberpar

test/Makefile

@@ -11,7 +11,7 @@ info:
 	echo ""
 	echo "Available make variables:"
 	echo "  SIZE   : specify size of test data (small, medium, large)"
-	echo "  V      : specify level of verbosity (0, 1)"
+	echo "  V      : specify level of verbosity (0, 1, 2)"
 
 test:
 	for TEST in $(TESTDIRS) ; do \

test/bin/norminfo

@@ -178,12 +178,16 @@ sub main()
 	my $tn = "";
 	my %allfiles;
 
+	$multi = 1 if (!defined($multi));
 	if (!defined($infofile)) {
-		die("Usage: $0 <coverage-data-file> [<multiplier>]\n");
+		$infofile = "standard input";
+		warn("$0: Reading data from standard input\n");
+		open($fd, "<&STDIN") or
+			die("$0: Could not duplicated stdin: $!\n");
+	} else {
+		open($fd, "<", $infofile) or
+			die("$0: Could not open $infofile: $!\n");
 	}
-	$multi = 1 if (!defined($multi));
-
-	open($fd, "<", $infofile) or die("$0: Could not open $infofile: $!\n");
 
 	# Register starting positions of data sets
 	while (my $line = <$fd>) {

test/bin/test_run

@@ -83,7 +83,7 @@ echo
 # Show log excerpt on failure or if requested
 if [ $RC -ne 0 -o "$V" == "1" ] ; then
 	LEN=$(tail -c "+$POS" "$LOGFILE" | wc -l)
-	if [ "$LEN" -gt "$EXCERPTLEN" ] ; then
+	if [ "$LEN" -gt "$EXCERPTLEN" -a "$V" != "1" ] ; then
 		tail -c "+$POS" "$LOGFILE" | head -n $EXCERPTLEN | t_indent
 		let LEN=$LEN-$EXCERPTLEN
 		echo "    ..."

test/common.mak

@@ -18,6 +18,7 @@ COUNTFILES   := $(ZEROCOUNTS) $(FULLCOUNTS) $(TARGETCOUNTS) $(PART1COUNTS) \
 LCOVRC       := $(TOPDIR)lcovrc
 LCOVFLAGS    := --config-file $(LCOVRC)
 SIZE         := small
+CC           := gcc
 
 export LCOV := lcov $(LCOVFLAGS)
 export PATH := $(TOPDIR)/../bin:$(TOPDIR)/bin:$(PATH)
@@ -41,6 +42,8 @@ clean_common:
 $(INFOFILES) $(COUNTFILES):
 	cd $(TOPDIR) && mkinfo profiles/$(SIZE)
 
+ifneq ($(V),2)
 .SILENT:
+endif
 
 .PHONY: all init exit prepare clean clean_common

test/lcov_add_files/Makefile

@@ -3,7 +3,7 @@ include ../common.mak
 ADDTEST := ./add_test
 
 TESTS := lcov_add_zero lcov_add_zero2 lcov_add_full lcov_add_full2 \
-	 lcov_add_part lcov_add_part2 lcov_add_concatenated
+	 lcov_add_part lcov_add_part2 lcov_add_concatenated4
 
 
 test: $(TESTS)
@@ -35,7 +35,7 @@ lcov_add_part2:
 	# should be same as target file
 	test_run lcov_add_part2 $(ADDTEST) 1 "$(TARGETINFO)" "$(PART1INFO)" "$(PART2INFO)"
 
-lcov_add_concatenated:
+lcov_add_concatenated4:
 	# Add coverage file that consists of 4 concatenation of target files 
 	# and reduce counts to 1/4 - output should be the same as input
 	cat $(TARGETINFO) $(TARGETINFO) $(TARGETINFO) $(TARGETINFO) >concatenated.info

test/lcov_add_files/add_test

@@ -41,6 +41,6 @@ fi
 
 echo "Comparing with reference..."
 if ! diff -u "$REFFILE" "$SORTFILE" ; then
-	echo "Error: Result of addition differs from reference file" >&2
+	echo "Error: Result of combination differs from reference file" >&2
 	exit 1
 fi

test/lcov_diff/Makefile

@@ -0,0 +1,9 @@
+include ../common.mak
+
+test:
+	test_run lcov_diff_apply ./diff_test
+
+clean:
+	make -C old clean
+	make -C new clean
+	rm -f *.info diff

test/lcov_diff/diff_test

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright IBM Corp. 2017
+#
+# Usage: diff_test
+#
+# Check lcov's diff function:
+# - Compile two slightly different test programs
+# - Run the programs and collect coverage data
+# - Generate a patch containing the difference between the source code
+# - Apply the patch to the coverage data
+# - Compare the resulting patched coverage data file with the data from the
+#   patched source file
+#
+
+function die()
+{
+	echo "Error: $@" >&2
+	exit 1
+}
+
+make -C old || die "Failed to compile old source"
+make -C new || die "Failed to compile new source"
+diff -u $PWD/old/prog.c $PWD/new/prog.c > diff
+
+$LCOV --diff old/prog.info diff --convert-filenames -o patched.info -t bla || \
+	die "Failed to apply patch to coverage data file"
+norminfo new/prog.info > new_normalized.info
+norminfo patched.info > patched_normalized.info
+sed -i -e 's/^TN:.*$/TN:/' patched_normalized.info
+
+diff -u patched_normalized.info new_normalized.info || \
+	die "Mismatch in patched coverage data file"
+
+echo "Patched coverage data file matches expected file"

test/lcov_diff/new/Makefile

@@ -0,0 +1,17 @@
+prog.info:
+
+include ../../common.mak
+
+prog.info: prog.gcda
+	$(LCOV) -c -d . -o prog.info
+
+prog.gcda: prog
+	./prog || true
+
+prog: prog.c
+	$(CC) prog.c -o prog --coverage
+
+clean:
+	rm -f prog prog.gcda prog.gcno prog.info
+
+.PHONY: all clean

test/lcov_diff/new/prog.c

@@ -0,0 +1,41 @@
+
+
+
+int fn(int x)
+{
+	switch (x) {
+	case -1: return 0;
+
+
+	case 0:  return 2;
+	case 2:  return 3;
+
+
+	case 12: return 7;
+	default: return 255;
+	}
+
+
+
+}
+
+int fn2()
+{
+
+
+	return 7;
+}
+
+
+
+int main(int argc, char *argv[])
+{
+
+
+	if (argc > 1)
+		return fn(argc);
+
+	return fn2();
+
+
+}