Cleanup sytest.list, add ignore list (#276)

* sytest list cleanup

* add more ignored tests

* add one more ignored entry

Author: ShadowJonathan

cmd/sytest-coverage/main.go

@@ -28,22 +28,9 @@ var (
 // in all files in ./tests - if there's a match it marks that test as converted.
 func main() {
 	verbose := len(os.Args) == 2 && os.Args[1] == "-v"
-	body, err := ioutil.ReadFile("./sytest.list")
-	if err != nil {
-		panic(err)
-	}
-	testLines := strings.Split(string(body), "\n")
-	filenameToTestName := make(map[string][]string)
-	testNameToFilename := make(map[string]string)
-	for _, line := range testLines {
-		name, filename := extract(line)
-		if name == "" || filename == "" {
-			continue
-		}
-		name = "sytest: " + strings.TrimSpace(name)
-		filenameToTestName[filename] = append(filenameToTestName[filename], name)
-		testNameToFilename[name] = strings.TrimSpace(filename)
-	}
+
+	filenameToTestName, testNameToFilename := getList()
+
 	total := len(testNameToFilename)
 
 	convertedTests := make(map[string]bool)
@@ -100,6 +87,60 @@ func main() {
 	fmt.Printf("\nTOTAL: %d/%d tests converted\n", numComplementTests, total)
 }
 
+// filenameToTestName and testNameToFilename
+// will filter ignored tests
+func getList() (map[string][]string, map[string]string) {
+	var ignoredTests = make(map[string]bool)
+	var ignoredPaths []string
+	ignoredBody, err := ioutil.ReadFile("./sytest.ignored.list")
+	if err != nil {
+		// ignore error, set body to nothing
+		ignoredBody = []byte{}
+	}
+	ignoredLines := strings.Split(string(ignoredBody), "\n")
+	for _, ignoredLine := range ignoredLines {
+		ignoredLine = strings.TrimSpace(ignoredLine)
+
+		if len(ignoredLine) == 0 || ignoredLine[0] == '#' {
+			continue
+		}
+
+		if ignoredLine[0] == '!' {
+			ignoredPaths = append(ignoredPaths, ignoredLine[1:])
+		}
+
+		ignoredTests[ignoredLine] = true
+	}
+
+	body, err := ioutil.ReadFile("./sytest.list")
+	if err != nil {
+		panic(err)
+	}
+	testLines := strings.Split(string(body), "\n")
+	filenameToTestName := make(map[string][]string)
+	testNameToFilename := make(map[string]string)
+lines:
+	for _, line := range testLines {
+		name, filename := extract(line)
+		if name == "" || filename == "" {
+			continue
+		}
+		if _, ok := ignoredTests[name]; ok {
+			continue
+		}
+		for _, path := range ignoredPaths {
+			if strings.Contains(filename, path) {
+				continue lines
+			}
+		}
+		name = "sytest: " + strings.TrimSpace(name)
+		filenameToTestName[filename] = append(filenameToTestName[filename], name)
+		testNameToFilename[name] = strings.TrimSpace(filename)
+	}
+
+	return filenameToTestName, testNameToFilename
+}
+
 func sorted(in map[string][]string) []string {
 	out := make([]string, len(in))
 	i := 0
@@ -115,7 +156,7 @@ func sorted(in map[string][]string) []string {
 // ./tests/31sync/16room-summary.pl:test "Room summary counts change when membership changes",
 func extract(line string) (string, string) {
 	line = strings.TrimSpace(line)
-	if len(line) == 0 {
+	if len(line) == 0 || line[0] == '#' {
 		return "", ""
 	}
 	nameGroups := testNameRegexp.FindStringSubmatch(line)

sytest.ignored.list

@@ -0,0 +1,41 @@
+# These list all of the sytests that're ignored, line-by-line
+# Complete with comments as to why particular tests were ignored
+
+# Dummy test in 00prepare.pl
+foo
+
+# Tests synapse admin API
+Can quarantine media in rooms
+
+# Tests deprecated endpoints
+Tags appear in the v1 /initialSync
+Tags appear in the v1 /events stream
+Tags appear in the v1 room initial sync
+Account data appears in v1 /events stream
+Latest account data comes down in /initialSync
+Latest account data comes down in room initialSync
+Room account data appears in v1 /events stream
+GET /events with negative 'limit'
+GET /events with non-numeric 'limit'
+GET /events with non-numeric 'timeout'
+GET /initialSync with non-numeric 'limit'
+GET /events initially
+GET /initialSync initially
+GET /rooms/:room_id/initialSync fetches initial sync state
+All room members see all room members' presence in global initialSync
+New room members see existing users' presence in room initialSync
+New room members see existing members' presence in room initialSync
+New room members see first user's profile information in global initialSync
+New room members see first user's profile information in per-room initialSync
+A departed room is still included in /initialSync (SPEC-216)
+Can get rooms/{roomId}/initialSync for a departed room (SPEC-216)
+initialSync sees my presence status
+Global initialSyncGlobal initialSync with limit=0 gives no messages
+Room initialSync
+Room initialSync with limit=0 gives no messages
+Read receipts are visible to /initialSync
+Newly created users see their own presence in /initialSync (SYT-34)
+Push rules come down in an initial /sync
+Guest user calling /events doesn't tightloop
+Guest user cannot call /events globally
+!53groups