Merge branch 'bugfix/case-sensitivity-of-tags'

Author: hauke96

integration-tests/test-generic.mediawiki

@@ -88,18 +88,18 @@ All (raster) images will be scaled down and turned into grayscale images. SVGs s
 
 === Galleries ===
 
-<gallery>
+<GALLery>
 Wikimedia_Servers-0051_19.jpg|With some caption.
 Datei:Wikipedia-logo-v2.svg
-</gallery>
+</gallERY>
 
 === Image maps ===
 
-<imagemap>
+<IMAGEmap>
 Image:Wikimedia_Servers-0051_19.jpg|With some caption
 rect 1 1 741 445 [[Oregon|Oregon]]
 desc none
-</imagemap>
+</imageMAP>
 
 == Tables ==
 
@@ -130,9 +130,9 @@ A bit tricky but they work as well:
 
 == References ==
 
-They will be collected<ref name="another-ref">This is true</ref> and are visible<ref>Some reference</ref> at the end of the document:<ref name="another-ref" /><br>
+They will be collected<ref name="another-ref">This is true</ref> and are visible<REF>Some reference</ref> at the end of the document:<REF name="another-ref" /><br>
 
-There are also grouped references possible.<ref group="some-group">Some grouped ref</ref><br>
+There are also grouped references possible.<ref group="some-group">Some grouped ref</REF><br>
 Even named<ref group="some-group" name="grouped-ref-name"/> grouped references work!<ref group="some-group" name="grouped-ref-name">Some named grouped ref</ref><br>
 
 Normal refs:<br>
@@ -158,7 +158,7 @@ Grouped refs:<br>
 Some text has the <nowiki>keyword</nowiki>.
 
 It can also be used in the following ways:
-* <nowiki>in lists</nowiki>
+* <NOwiki>in lists</noWIKI>
 * also with <nowiki>lists</nowiki> containing text
 
 Also in comments: <!-- this <nowiki>part</nowiki> should not be visible -->

src/parser/image.go

@@ -93,16 +93,16 @@ func (t *Tokenizer) parseGalleries(content string) string {
 		trimmedLine := strings.TrimSpace(line)
 
 		// Gallery ends -> Simply remove line and end "withinGallery" mode
-		if strings.HasPrefix(trimmedLine, "</gallery>") {
+		if util.HasPrefixIgnoreCase(trimmedLine, "</gallery>") {
 			withinGallery = false
 
-			if trimmedLine == "</gallery>" {
+			if util.EqualsIgnoreCase(trimmedLine, "</gallery>") {
 				// This line just contains the tag -> ignore it and proceed with parsing
 				continue
 			}
 
 			// If the line contains more than the closing tag -> Keep it and proceed with the processing
-			line = strings.ReplaceAll(line, "</gallery>", "")
+			line = util.ReplaceAllIgnoreCase(line, "</gallery>", "")
 		} else if galleryStartRegex.MatchString(trimmedLine) {
 			withinGallery = true
 
@@ -159,13 +159,13 @@ func (t *Tokenizer) parseImageMaps(content string) string {
 		line := lines[i]
 
 		// Delete uninteresting lines (end of map or all the polygon-map-stuff in between)
-		if withinImageMap || line == "</imagemap>" {
+		if withinImageMap || util.EqualsIgnoreCase(line, "</imagemap>") {
 			// delete this line i
 			lines = append(lines[:i], lines[i+1:]...)
 			i--
 
 			// Imagemap ends -> end "withinImageMap" mode
-			if line == "</imagemap>" {
+			if util.EqualsIgnoreCase(line, "</imagemap>") {
 				withinImageMap = false
 			}
 

src/parser/image_test.go

@@ -110,7 +110,7 @@ func TestEscapeImages_leadingSpecialChar(t *testing.T) {
 func TestParseGalleries(t *testing.T) {
 	tokenizer := NewTokenizerWithMockWikipediaService()
 	content := tokenizer.parseGalleries(`foo
-<gallery>file0.jpg
+<gallery>file0.jpg|thumb
 file:file1.jpg|captiion
 </gallery>
 bar
@@ -120,7 +120,7 @@ File:file2.jpg|test123
 </gallery>blubb`)
 
 	test.AssertEqual(t, `foo
-[[File:File0.jpg|mini]]
+[[File:File0.jpg|thumb]]
 [[file:File1.jpg|mini|captiion]]
 bar
 [[File:File2.jpg|mini|test123]]
@@ -130,6 +130,29 @@ blubb`, content)
 	test.AssertMapEqual(t, map[string]Token{}, tokenizer.getTokenMap())
 }
 
+func TestParseGalleries_caseInsensitivity(t *testing.T) {
+	tokenizer := NewTokenizerWithMockWikipediaService()
+	content := tokenizer.parseGalleries(`foo
+<GALLERY>file0.jpg|thumb
+file:file1.jpg|captiion
+</gallery>
+bar
+ <gallery some="parameter">
+File:file2.jpg|test123|THUMB
+  file 3.jpg
+</GALLERY>blubb`)
+
+	test.AssertEqual(t, `foo
+[[File:File0.jpg|thumb]]
+[[file:File1.jpg|mini|captiion]]
+bar
+[[File:File2.jpg|test123|THUMB]]
+[[File:File_3.jpg|mini]]
+blubb`, content)
+
+	test.AssertMapEqual(t, map[string]Token{}, tokenizer.getTokenMap())
+}
+
 func TestParseGalleries_emptyGallery(t *testing.T) {
 	tokenizer := NewTokenizerWithMockWikipediaService()
 	content := tokenizer.parseGalleries(`foo
@@ -166,6 +189,29 @@ blubb`, content)
 	test.AssertMapEqual(t, map[string]Token{}, tokenizer.getTokenMap())
 }
 
+func TestParseImagemaps_caseInsensitivity(t *testing.T) {
+	tokenizer := NewTokenizerWithMockWikipediaService()
+	content := tokenizer.parseImageMaps(`foo
+<IMAGEMAP>File:picture.jpg
+some
+stuff
+</imagemap>
+bar
+<imagemap some="parameter">
+Image:picture.jpg
+some stuff
+</IMAGEMAP>
+blubb`)
+
+	test.AssertEqual(t, `foo
+[[File:Picture.jpg]]
+bar
+[[Image:Picture.jpg]]
+blubb`, content)
+
+	test.AssertMapEqual(t, map[string]Token{}, tokenizer.getTokenMap())
+}
+
 func TestParseImages_inlineHappyPath(t *testing.T) {
 	setup()
 	tokenizer := NewTokenizerWithMockWikipediaService()

src/parser/nowiki.go

@@ -1,5 +1,7 @@
 package parser
 
+import "wiki2book/util"
+
 type NowikiToken struct {
 	Token
 	Content string
@@ -19,8 +21,8 @@ func (t *Tokenizer) parseNowiki(content string) string {
 	for i := 0; i < len(content)-nowikiEndLen; i++ {
 		cursor := content[i : i+nowikiStartLen]
 
-		if cursor == nowikiStart {
-			endIndex := FindCorrespondingCloseToken(content, i+nowikiStartLen, nowikiStart, nowikiEnd)
+		if util.EqualsIgnoreCase(cursor, nowikiStart) {
+			endIndex := FindCorrespondingCloseTokenIgnoreCase(content, i+nowikiStartLen, nowikiStart, nowikiEnd)
 
 			token := NowikiToken{
 				Content: content[i+nowikiStartLen : endIndex],

src/parser/nowiki_test.go

@@ -19,6 +19,20 @@ func TestNowiki(t *testing.T) {
 	}, tokenizer.getTokenMap())
 }
 
+func TestNowiki_caseInsensitivity(t *testing.T) {
+	tokenizer := NewTokenizerWithMockWikipediaService()
+	content := "Foo<NOWIKI>something</nowiki> bar <nowiki>something else</NOWIKI> blubb"
+	expectedContent := "Foo" + fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_NOWIKI, 0) + " bar " + fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_NOWIKI, 1) + " blubb"
+
+	newContent := tokenizer.parseNowiki(content)
+
+	test.AssertEqual(t, expectedContent, newContent)
+	test.AssertMapEqual(t, map[string]Token{
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_NOWIKI, 0): NowikiToken{Content: "something"},
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_NOWIKI, 1): NowikiToken{Content: "something else"},
+	}, tokenizer.getTokenMap())
+}
+
 func TestNowiki_endOfText(t *testing.T) {
 	tokenizer := NewTokenizerWithMockWikipediaService()
 	content := "Foo<nowiki>something</nowiki>"

src/parser/pattern.go

@@ -40,9 +40,9 @@ var (
 
 // Media files
 var (
-	galleryStartRegex          = regexp.MustCompile(`^<gallery.*?>`)
-	imagemapStartRegex         = regexp.MustCompile(`^<imagemap.*?>`)
-	hasNonInlineParameterRegex = regexp.MustCompile("(" + strings.Join(imageNonInlineParameters, "|") + ")")
+	galleryStartRegex          = regexp.MustCompile(`(?i)^<gallery.*?>`)
+	imagemapStartRegex         = regexp.MustCompile(`(?i)^<imagemap.*?>`)
+	hasNonInlineParameterRegex = regexp.MustCompile("(?i)(" + strings.Join(imageNonInlineParameters, "|") + ")")
 )
 
 // Tables
@@ -54,9 +54,9 @@ var (
 
 // References
 var (
-	referencePlaceholderShortRegex = regexp.MustCompile(`<references.*?/\s*>`) // <references />
-	referencePlaceholderStartRegex = regexp.MustCompile(`<references.*?\s*>`)  // <references group="foo" >
-	referencePlaceholderEndRegex   = regexp.MustCompile(`</references\s*>`)    // </references>
+	referencePlaceholderShortRegex = regexp.MustCompile(`(?i)<references.*?/\s*>`) // <references />
+	referencePlaceholderStartRegex = regexp.MustCompile(`(?i)<references.*?\s*>`)  // <references group="foo" >
+	referencePlaceholderEndRegex   = regexp.MustCompile(`(?i)</references\s*>`)    // </references>
 )
 
 // Math

src/parser/reference.go

@@ -61,12 +61,12 @@ func (t *Tokenizer) parseReferences(content string) string {
 
 	for i := 0; i < len(content)-refDefStartLen; i++ {
 		cursor := content[i : i+refDefStartLen]
-		if cursor != refDefStart && cursor != refPlaceholderEnd {
+		if !util.EqualsIgnoreCase(cursor, refDefStart) && !util.EqualsIgnoreCase(cursor, refPlaceholderEnd) {
 			// Cursor is not on the beginning of any reference related tag.
 			continue
 		}
 
-		startEndIndex := FindCorrespondingCloseToken(content, i+refDefStartLen, refDefStart, xmlClosing)
+		startEndIndex := FindCorrespondingCloseTokenIgnoreCase(content, i+refDefStartLen, refDefStart, xmlClosing)
 		if startEndIndex == -1 {
 			// XML for <ref not closed -> broken wikitext
 			sigolo.Errorf("XML element for reference start '%s' not closed (i.e. missing '%s'). Text around this location: ...%s...", refDefStart, xmlClosing, util.GetTextAround(content, i, 50))
@@ -117,7 +117,7 @@ func (t *Tokenizer) parseReferences(content string) string {
 				refNumberCounterForCurrentGroup, content = t.parseNamedReferenceUsage(content, i, nameAttributeValue, nameToRefNumberForCurrentGroup, refNumberCounterForCurrentGroup, cursorWithinReferencePlaceholder, startEndIndex)
 			} else {
 				// Reference definition like "<ref name=...>Foobar</ref".
-				refEndIndex := FindCorrespondingCloseToken(content, startEndIndex, refDefStart, refDefLongEnd)
+				refEndIndex := FindCorrespondingCloseTokenIgnoreCase(content, startEndIndex, refDefStart, refDefLongEnd)
 				if refEndIndex == -1 {
 					// No end token found -> probably unsupported wikitext syntax (like nested refs)
 					sigolo.Errorf("No end-part for reference start '%s' found. Text around this location: ...%s...", refDefStart, util.GetTextAround(content, i, 50))

src/parser/reference_test.go

@@ -34,6 +34,34 @@ some footer`
 	}, tokenizer.getTokenMap())
 }
 
+func TestParseReferences_caseInsensitivity(t *testing.T) {
+	tokenizer := NewTokenizerWithMockWikipediaService()
+	content := `some text<REF>bar</ref>
+some<ref name="blubb">blubbeldy</REF> other<REF name="fooref" /> text
+<ReferenCES responsive>
+<REF name="fooref">foo</REF>
+</REFerENces>
+some footer`
+	expectedContent := "some text" + fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_USAGE, 0) + "\n" +
+		"some" + fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_USAGE, 1) + " other" + fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_USAGE, 2) + " text\n" +
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_DEF, 3) + "\n" +
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_DEF, 4) + "\n" +
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_DEF, 5) + "\n" +
+		"some footer"
+
+	newContent := tokenizer.parseReferences(content)
+
+	test.AssertEqual(t, expectedContent, newContent)
+	test.AssertMapEqual(t, map[string]Token{
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_USAGE, 0): RefUsageToken{Index: 0},
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_USAGE, 1): RefUsageToken{Index: 1},
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_USAGE, 2): RefUsageToken{Index: 2},
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_DEF, 3):   RefDefinitionToken{Index: 0, Content: "bar"},
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_DEF, 4):   RefDefinitionToken{Index: 1, Content: "blubbeldy"},
+		fmt.Sprintf(TOKEN_TEMPLATE, TOKEN_REF_DEF, 5):   RefDefinitionToken{Index: 2, Content: "foo"},
+	}, tokenizer.getTokenMap())
+}
+
 func TestParseReferences_tokenizeRefContent(t *testing.T) {
 	tokenizer := NewTokenizerWithMockWikipediaService()
 	content := `some text<ref>foo [[bar|Bar]]</ref>.`

src/parser/util.go

@@ -1,9 +1,25 @@
 package parser
 
-// FindCorrespondingCloseToken determines the index on which the given openingToken at the startIndex is closed.
+import "wiki2book/util"
+
+// FindCorrespondingCloseToken determines the index on which the given openingToken at the startIndex is closed. If the
+// closing token has not been found, -1 is returned. This function is case-sensitive.
 func FindCorrespondingCloseToken(content string, startIndex int, openingToken string, closingToken string) int {
-	// Used as a primitive stack to count the degree of nesting the cursor is in. If a closing token has been found
-	// and the nesting degree is 0, then the correct closing token has been found.
+	return findCorrespondingCloseToken(content, startIndex, openingToken, closingToken, false)
+}
+
+// FindCorrespondingCloseTokenIgnoreCase behaves like FindCorrespondingCloseToken but ignores the case of letters. This
+// function is case-insensitive.
+func FindCorrespondingCloseTokenIgnoreCase(content string, startIndex int, openingToken string, closingToken string) int {
+	return findCorrespondingCloseToken(content, startIndex, openingToken, closingToken, true)
+}
+
+// findCorrespondingCloseToken determines the index on which the given openingToken at the startIndex is closed. If the
+// closing token has not been found, -1 is returned.
+func findCorrespondingCloseToken(content string, startIndex int, openingToken string, closingToken string, ignoreCase bool) int {
+	// Used as a primitive stack to count the degree of nesting the cursor is in. Every opening token increments the
+	// counter, every closing token decrements it. If a closing token has been found and the nesting degree is 0, then
+	// the correct closing token has been found.
 	closeTokenCounter := 0
 
 	// The tokens are considered to be of equal size
@@ -23,12 +39,30 @@ func FindCorrespondingCloseToken(content string, startIndex int, openingToken st
 			cursorClosingToken = content[i : i+closingTokenSize]
 		}
 
-		if openingToken != closingToken && cursorOpeningToken == openingToken {
+		openingAndClosingTokenAreDifferent := false
+		cursorIsOnOpeningToken := false
+		if ignoreCase {
+			openingAndClosingTokenAreDifferent = !util.EqualsIgnoreCase(openingToken, closingToken)
+			cursorIsOnOpeningToken = util.EqualsIgnoreCase(cursorOpeningToken, openingToken)
+		} else {
+			openingAndClosingTokenAreDifferent = openingToken != closingToken
+			cursorIsOnOpeningToken = cursorOpeningToken == openingToken
+		}
+
+		cursorIsOnClosingToken := false
+		if ignoreCase {
+			cursorIsOnClosingToken = util.EqualsIgnoreCase(cursorClosingToken, closingToken)
+		} else {
+			cursorIsOnClosingToken = cursorClosingToken == closingToken
+		}
+
+		foundNewOpeningToken := openingAndClosingTokenAreDifferent && cursorIsOnOpeningToken
+		if foundNewOpeningToken {
 			closeTokenCounter++
 
 			// Skip the found opening token. Use the "-1" to compensate the "+1" by the loop
 			i += openingTokenSize - 1
-		} else if cursorClosingToken == closingToken {
+		} else if cursorIsOnClosingToken {
 			if closeTokenCounter == 0 {
 				return i
 			} else {

src/parser/util_test.go

@@ -103,3 +103,28 @@ func TestFindCorrespondingCloseToken_equalStartAndEndToken(t *testing.T) {
 	index = FindCorrespondingCloseToken("abc$$defghi$$", 5, "$$", "$$")
 	test.AssertEqual(t, 11, index)
 }
+
+func TestFindCorrespondingCloseTokenIgnoreCase(t *testing.T) {
+	var index int
+
+	index = FindCorrespondingCloseTokenIgnoreCase("abcfoodefbarghbari", 0, "foo", "bar")
+	test.AssertEqual(t, 14, index)
+
+	index = FindCorrespondingCloseTokenIgnoreCase("abcFOOdefbarghbari", 0, "foo", "bar")
+	test.AssertEqual(t, 14, index)
+
+	index = FindCorrespondingCloseTokenIgnoreCase("abcfoodefBARghBARi", 0, "foo", "bar")
+	test.AssertEqual(t, 14, index)
+
+	index = FindCorrespondingCloseTokenIgnoreCase("abcFOOdefBARghBARi", 0, "foo", "bar")
+	test.AssertEqual(t, 14, index)
+
+	index = FindCorrespondingCloseTokenIgnoreCase("abcfoodefbarghbari", 0, "FOO", "bar")
+	test.AssertEqual(t, 14, index)
+
+	index = FindCorrespondingCloseTokenIgnoreCase("abcfoodefbarghbari", 0, "foo", "BAR")
+	test.AssertEqual(t, 14, index)
+
+	index = FindCorrespondingCloseTokenIgnoreCase("abcfoodefbarghbari", 0, "FOO", "BAR")
+	test.AssertEqual(t, 14, index)
+}