refactor recursive URL parsing

Author: aafeher

sitemap.go

@@ -437,7 +437,10 @@ func (s *S) parseAndFetchUrlsMultiThread(locations []string) {
 				return
 			}
 			content = s.checkAndUnzipContent(content)
-			_ = s.parse(loc, string(content))
+			parsedLocations := s.parse(loc, string(content))
+			if len(parsedLocations) > 0 {
+				s.parseAndFetchUrlsMultiThread(parsedLocations)
+			}
 		}()
 	}
 	wg.Wait()
@@ -457,7 +460,10 @@ func (s *S) parseAndFetchUrlsSequential(locations []string) {
 			continue
 		}
 		content = s.checkAndUnzipContent(content)
-		_ = s.parse(location, string(content))
+		parsedLocations := s.parse(location, string(content))
+		if len(parsedLocations) > 0 {
+			s.parseAndFetchUrlsSequential(parsedLocations)
+		}
 	}
 }
 

sitemap_test.go

@@ -1281,7 +1281,7 @@ func TestS_parseAndFetchUrlsMultiThread(t *testing.T) {
 				fmt.Sprintf("%s/sitemapindex-1.xml", server.URL),
 				fmt.Sprintf("%s/sitemap-04.xml", server.URL),
 			},
-			urlsCount: 1,
+			urlsCount: 7,
 			errsCount: 0,
 		},
 	}
@@ -1336,7 +1336,7 @@ func TestS_parseAndFetchUrlsSequential(t *testing.T) {
 				fmt.Sprintf("%s/sitemapindex-1.xml", server.URL),
 				fmt.Sprintf("%s/sitemap-04.xml", server.URL),
 			},
-			urlsCount: 1,
+			urlsCount: 7,
 			errsCount: 0,
 		},
 	}