Merge branch 'main' into feature/repo-keyboard-shortcuts

Author: micahkepe

custom/conf/app.example.ini

@@ -2485,8 +2485,9 @@ LEVEL = Info
 ;[highlight.mapping]
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Extension mapping to highlight class
-;; e.g. .toml=ini
+;; Extension mapping to highlight class, for example:
+;; .toml = ini
+;; .my-js = JavaScript
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

modules/analyze/code_language.go

@@ -4,12 +4,13 @@
 package analyze
 
 import (
-	"path/filepath"
+	"path"
 
 	"github.com/go-enry/go-enry/v2"
 )
 
 // GetCodeLanguage detects code language based on file name and content
+// It can be slow when the content is used for detection
 func GetCodeLanguage(filename string, content []byte) string {
 	if language, ok := enry.GetLanguageByExtension(filename); ok {
 		return language
@@ -23,5 +24,5 @@ func GetCodeLanguage(filename string, content []byte) string {
 		return enry.OtherLanguage
 	}
 
-	return enry.GetLanguage(filepath.Base(filename), content)
+	return enry.GetLanguage(path.Base(filename), content)
 }

modules/highlight/highlight.go

@@ -12,7 +12,6 @@ import (
 	"html/template"
 	"io"
 	"path"
-	"path/filepath"
 	"strings"
 	"sync"
 
@@ -25,35 +24,32 @@ import (
 	"github.com/alecthomas/chroma/v2/formatters/html"
 	"github.com/alecthomas/chroma/v2/lexers"
 	"github.com/alecthomas/chroma/v2/styles"
-	lru "github.com/hashicorp/golang-lru/v2"
+	"github.com/go-enry/go-enry/v2"
 )
 
 // don't index files larger than this many bytes for performance purposes
 const sizeLimit = 1024 * 1024
 
-var (
-	// For custom user mapping
-	highlightMapping = map[string]string{}
-
-	once sync.Once
-
-	cache *lru.TwoQueueCache[string, any]
+type globalVarsType struct {
+	highlightMapping map[string]string
+	githubStyles     *chroma.Style
+}
 
-	githubStyles = styles.Get("github")
+var (
+	globalVarsMu  sync.Mutex
+	globalVarsPtr *globalVarsType
 )
 
-// NewContext loads custom highlight map from local config
-func NewContext() {
-	once.Do(func() {
-		highlightMapping = setting.GetHighlightMapping()
-
-		// The size 512 is simply a conservative rule of thumb
-		c, err := lru.New2Q[string, any](512)
-		if err != nil {
-			panic(fmt.Sprintf("failed to initialize LRU cache for highlighter: %s", err))
-		}
-		cache = c
-	})
+func globalVars() *globalVarsType {
+	// in the future, the globalVars might need to be re-initialized when settings change, so don't use sync.Once here
+	globalVarsMu.Lock()
+	defer globalVarsMu.Unlock()
+	if globalVarsPtr == nil {
+		globalVarsPtr = &globalVarsType{}
+		globalVarsPtr.githubStyles = styles.Get("github")
+		globalVarsPtr.highlightMapping = setting.GetHighlightMapping()
+	}
+	return globalVarsPtr
 }
 
 // UnsafeSplitHighlightedLines splits highlighted code into lines preserving HTML tags
@@ -88,59 +84,85 @@ func UnsafeSplitHighlightedLines(code template.HTML) (ret [][]byte) {
 	}
 }
 
-// Code returns an HTML version of code string with chroma syntax highlighting classes and the matched lexer name
-func Code(fileName, language, code string) (output template.HTML, lexerName string) {
-	NewContext()
-
-	// diff view newline will be passed as empty, change to literal '\n' so it can be copied
-	// preserve literal newline in blame view
-	if code == "" || code == "\n" {
-		return "\n", ""
+func getChromaLexerByLanguage(fileName, lang string) chroma.Lexer {
+	lang, _, _ = strings.Cut(lang, "?") // maybe, the value from gitattributes might contain `?` parameters?
+	ext := path.Ext(fileName)
+	// the "lang" might come from enry, it has different naming for some languages
+	switch lang {
+	case "F#":
+		lang = "FSharp"
+	case "Pascal":
+		lang = "ObjectPascal"
+	case "C":
+		if ext == ".C" || ext == ".H" {
+			lang = "C++"
+		}
 	}
+	// lexers.Get is slow if the language name can't be matched directly: it does extra "Match" call to iterate all lexers
+	return lexers.Get(lang)
+}
 
-	if len(code) > sizeLimit {
-		return template.HTML(template.HTMLEscapeString(code)), ""
+// GetChromaLexerWithFallback returns a chroma lexer by given file name, language and code content. All parameters can be optional.
+// When code content is provided, it will be slow if no lexer is found by file name or language.
+// If no lexer is found, it will return the fallback lexer.
+func GetChromaLexerWithFallback(fileName, lang string, code []byte) (lexer chroma.Lexer) {
+	if lang != "" {
+		lexer = getChromaLexerByLanguage(fileName, lang)
 	}
 
-	var lexer chroma.Lexer
-
-	if len(language) > 0 {
-		lexer = lexers.Get(language)
+	if lexer == nil {
+		fileExt := path.Ext(fileName)
+		if val, ok := globalVars().highlightMapping[fileExt]; ok {
+			lexer = getChromaLexerByLanguage(fileName, val) // use mapped value to find lexer
+		}
+	}
 
+	if lexer == nil {
+		// when using "code" to detect, analyze.GetCodeLanguage is slower, it iterates many rules to detect language from content
+		// this is the old logic: use enry to detect language, and use chroma to render, but their naming is different for some languages
+		enryLanguage := analyze.GetCodeLanguage(fileName, code)
+		lexer = getChromaLexerByLanguage(fileName, enryLanguage)
 		if lexer == nil {
-			// Attempt stripping off the '?'
-			if before, _, ok := strings.Cut(language, "?"); ok {
-				lexer = lexers.Get(before)
+			if enryLanguage != enry.OtherLanguage {
+				log.Warn("No chroma lexer found for enry detected language: %s (file: %s), need to fix the language mapping between enry and chroma.", enryLanguage, fileName)
 			}
+			lexer = lexers.Match(fileName) // lexers.Match will search by its basename and extname
 		}
 	}
 
-	if lexer == nil {
-		if val, ok := highlightMapping[path.Ext(fileName)]; ok {
-			// use mapped value to find lexer
-			lexer = lexers.Get(val)
-		}
+	return util.IfZero(lexer, lexers.Fallback)
+}
+
+func renderCode(fileName, language, code string, slowGuess bool) (output template.HTML, lexerName string) {
+	// diff view newline will be passed as empty, change to literal '\n' so it can be copied
+	// preserve literal newline in blame view
+	if code == "" || code == "\n" {
+		return "\n", ""
 	}
 
-	if lexer == nil {
-		if l, ok := cache.Get(fileName); ok {
-			lexer = l.(chroma.Lexer)
-		}
+	if len(code) > sizeLimit {
+		return template.HTML(template.HTMLEscapeString(code)), ""
 	}
 
-	if lexer == nil {
-		lexer = lexers.Match(fileName)
-		if lexer == nil {
-			lexer = lexers.Fallback
-		}
-		cache.Add(fileName, lexer)
+	var codeForGuessLexer []byte
+	if slowGuess {
+		// it is slower to guess lexer by code content, so only do it when necessary
+		codeForGuessLexer = util.UnsafeStringToBytes(code)
 	}
+	lexer := GetChromaLexerWithFallback(fileName, language, codeForGuessLexer)
+	return RenderCodeByLexer(lexer, code), formatLexerName(lexer.Config().Name)
+}
 
-	return CodeFromLexer(lexer, code), formatLexerName(lexer.Config().Name)
+func RenderCodeFast(fileName, language, code string) (output template.HTML, lexerName string) {
+	return renderCode(fileName, language, code, false)
 }
 
-// CodeFromLexer returns a HTML version of code string with chroma syntax highlighting classes
-func CodeFromLexer(lexer chroma.Lexer, code string) template.HTML {
+func RenderCodeSlowGuess(fileName, language, code string) (output template.HTML, lexerName string) {
+	return renderCode(fileName, language, code, true)
+}
+
+// RenderCodeByLexer returns a HTML version of code string with chroma syntax highlighting classes
+func RenderCodeByLexer(lexer chroma.Lexer, code string) template.HTML {
 	formatter := html.New(html.WithClasses(true),
 		html.WithLineNumbers(false),
 		html.PreventSurroundingPre(true),
@@ -155,7 +177,7 @@ func CodeFromLexer(lexer chroma.Lexer, code string) template.HTML {
 		return template.HTML(template.HTMLEscapeString(code))
 	}
 	// style not used for live site but need to pass something
-	err = formatter.Format(htmlw, githubStyles, iterator)
+	err = formatter.Format(htmlw, globalVars().githubStyles, iterator)
 	if err != nil {
 		log.Error("Can't format code: %v", err)
 		return template.HTML(template.HTMLEscapeString(code))
@@ -167,44 +189,18 @@ func CodeFromLexer(lexer chroma.Lexer, code string) template.HTML {
 	return template.HTML(strings.TrimSuffix(htmlbuf.String(), "\n"))
 }
 
-// File returns a slice of chroma syntax highlighted HTML lines of code and the matched lexer name
-func File(fileName, language string, code []byte) ([]template.HTML, string, error) {
-	NewContext()
-
+// RenderFullFile returns a slice of chroma syntax highlighted HTML lines of code and the matched lexer name
+func RenderFullFile(fileName, language string, code []byte) ([]template.HTML, string, error) {
 	if len(code) > sizeLimit {
-		return PlainText(code), "", nil
+		return RenderPlainText(code), "", nil
 	}
 
 	formatter := html.New(html.WithClasses(true),
 		html.WithLineNumbers(false),
 		html.PreventSurroundingPre(true),
 	)
 
-	var lexer chroma.Lexer
-
-	// provided language overrides everything
-	if language != "" {
-		lexer = lexers.Get(language)
-	}
-
-	if lexer == nil {
-		if val, ok := highlightMapping[filepath.Ext(fileName)]; ok {
-			lexer = lexers.Get(val)
-		}
-	}
-
-	if lexer == nil {
-		guessLanguage := analyze.GetCodeLanguage(fileName, code)
-
-		lexer = lexers.Get(guessLanguage)
-		if lexer == nil {
-			lexer = lexers.Match(fileName)
-			if lexer == nil {
-				lexer = lexers.Fallback
-			}
-		}
-	}
-
+	lexer := GetChromaLexerWithFallback(fileName, language, code)
 	lexerName := formatLexerName(lexer.Config().Name)
 
 	iterator, err := lexer.Tokenise(nil, string(code))
@@ -218,7 +214,7 @@ func File(fileName, language string, code []byte) ([]template.HTML, string, erro
 	lines := make([]template.HTML, 0, len(tokensLines))
 	for _, tokens := range tokensLines {
 		iterator = chroma.Literator(tokens...)
-		err = formatter.Format(htmlBuf, githubStyles, iterator)
+		err = formatter.Format(htmlBuf, globalVars().githubStyles, iterator)
 		if err != nil {
 			return nil, "", fmt.Errorf("can't format code: %w", err)
 		}
@@ -229,8 +225,8 @@ func File(fileName, language string, code []byte) ([]template.HTML, string, erro
 	return lines, lexerName, nil
 }
 
-// PlainText returns non-highlighted HTML for code
-func PlainText(code []byte) []template.HTML {
+// RenderPlainText returns non-highlighted HTML for code
+func RenderPlainText(code []byte) []template.HTML {
 	r := bufio.NewReader(bytes.NewReader(code))
 	m := make([]template.HTML, 0, bytes.Count(code, []byte{'\n'})+1)
 	for {

modules/highlight/highlight_test.go

@@ -112,7 +112,7 @@ c=2
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			out, lexerName, err := File(tt.name, "", []byte(tt.code))
+			out, lexerName, err := RenderFullFile(tt.name, "", []byte(tt.code))
 			assert.NoError(t, err)
 			assert.Equal(t, tt.want, out)
 			assert.Equal(t, tt.lexerName, lexerName)
@@ -176,7 +176,7 @@ c=2`),
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			out := PlainText([]byte(tt.code))
+			out := RenderPlainText([]byte(tt.code))
 			assert.Equal(t, tt.want, out)
 		})
 	}
@@ -199,3 +199,36 @@ func TestUnsafeSplitHighlightedLines(t *testing.T) {
 	assert.Equal(t, "<span>a</span>\n", string(ret[0]))
 	assert.Equal(t, "<span>b\n</span>", string(ret[1]))
 }
+
+func TestGetChromaLexer(t *testing.T) {
+	globalVars().highlightMapping[".my-html"] = "HTML"
+	t.Cleanup(func() { delete(globalVars().highlightMapping, ".my-html") })
+
+	cases := []struct {
+		fileName string
+		language string
+		content  string
+		expected string
+	}{
+		{"test.py", "", "", "Python"},
+
+		{"any-file", "javascript", "", "JavaScript"},
+		{"any-file", "", "/* vim: set filetype=python */", "Python"},
+		{"any-file", "", "", "fallback"},
+
+		{"test.fs", "", "", "Forth"},
+		{"test.fs", "F#", "", "FSharp"},
+		{"test.fs", "", "let x = 1", "FSharp"},
+
+		{"test.c", "", "", "C"},
+		{"test.C", "", "", "C++"},
+		{"OLD-CODE.PAS", "", "", "ObjectPascal"},
+		{"test.my-html", "", "", "HTML"},
+	}
+	for _, c := range cases {
+		lexer := GetChromaLexerWithFallback(c.fileName, c.language, []byte(c.content))
+		if assert.NotNil(t, lexer, "case: %+v", c) {
+			assert.Equal(t, c.expected, lexer.Config().Name, "case: %+v", c)
+		}
+	}
+}

modules/indexer/code/search.go

@@ -72,10 +72,10 @@ func writeStrings(buf *bytes.Buffer, strs ...string) error {
 
 func HighlightSearchResultCode(filename, language string, lineNums []int, code string) []*ResultLine {
 	// we should highlight the whole code block first, otherwise it doesn't work well with multiple line highlighting
-	hl, _ := highlight.Code(filename, language, code)
+	hl, _ := highlight.RenderCodeFast(filename, language, code)
 	highlightedLines := strings.Split(string(hl), "\n")
 
-	// The lineNums outputted by highlight.Code might not match the original lineNums, because "highlight" removes the last `\n`
+	// The lineNums outputted by render might not match the original lineNums, because "highlight" removes the last `\n`
 	lines := make([]*ResultLine, min(len(highlightedLines), len(lineNums)))
 	for i := range lines {
 		lines[i] = &ResultLine{