[fontscan] tweak font resolution order to make it more principled

Author: benoitkugler

fontscan/fontmap.go

@@ -506,18 +506,17 @@ func (fm *FontMap) resolveForLang(candidates []int, lang LangID) *font.Face {
 // ResolveFace select a font based on the current query (set by [FontMap.SetQuery] and [FontMap.SetScript]),
 // and supporting the given rune, applying CSS font selection rules.
 //
-// Fonts are tried in 3 steps :
+// Fonts are tried with the following steps :
 //
 //	1 - Only fonts matching exacly one of the [Query.Families] are considered; the list
 //		is prunned to keep the best match with [Query.Aspect]
 //	2 - Fallback fonts are considered, that is fonts with similar families and fonts
-//		supporting the current script; the list is also prunned according to [Query.Aspect]
-//	3 - All fonts matching the current script (set by [FontMap.SetScript]) are tried,
+//		supporting the current script; the list is also prunned according to [Query.Aspect]4
+//	3 - Fonts added manually by [AddFont] and [AddFace] will be searched,
+//		in the order in which they were added.
+//	4 - All fonts matching the current script (set by [FontMap.SetScript]) are tried,
 //		ignoring [Query.Aspect]
 //
-// The fonts added manually by [AddFont] and [AddFace]
-// will be searched in the order in which they were added.
-//
 // If no fonts match after these steps, an arbitrary face will be returned.
 // This face will be nil only if the underlying font database is empty,
 // or if the file system is broken; otherwise the returned [font.Face] is always valid.
@@ -546,8 +545,13 @@ func (fm *FontMap) ResolveFace(r rune) (face *font.Face) {
 		return face
 	}
 
-	// no need to check from user provided fonts, since the one supporting the given script
-	// are already added in fallback fonts
+	// try manually loaded faces even if the typeface doesn't match, looking for matching aspects
+	// and rune coverage.
+	// Note that, when [SetScript] has been called, this step is actually not needed,
+	// since the fonts supporting the given script are already added in [withFallback] fonts
+	if face := fm.resolveForRune(fm.candidates.manual, r); face != nil {
+		return face
+	}
 
 	fm.logger.Printf("No font matched for aspect %v, script %s, and rune %U (%c) -> searching by script coverage only", fm.query.Aspect, fm.script, r, r)
 	scriptCandidates := fm.scriptMap[fm.script]

fontscan/fontmap_test.go

@@ -6,7 +6,9 @@ import (
 	"io"
 	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -359,30 +361,109 @@ func TestFindSytemFont(t *testing.T) {
 	tu.Assert(t, !ok) // user provided font are ignored
 }
 
-func TestResolve_ScriptBengali(t *testing.T) {
+// the following tests use a "linux" font configuration
+func newSampleFontmap() *FontMap {
 	fm := NewFontMap(log.New(io.Discard, "", 0))
-	fm.appendFootprints(bengaliFontSet...)
+	fm.appendFootprints(linuxSampleFontSet...)
+	for _, fp := range linuxSampleFontSet {
+		fm.cache(fp, &font.Face{Font: new(font.Font)}) // we need a new pointer for each file
+	}
+	return fm
+}
+
+func TestDumpSystemFonts(t *testing.T) {
+	t.Skip()
+	fontset, err := SystemFonts(nil, os.TempDir())
+	tu.AssertNoErr(t, err)
+
+	var trimmed fontSet
+	for _, fp := range fontset {
+		switch fp.Family {
+		case "nimbussans", "lohitbengali", "lohitdevanagari", "lohitodia",
+			"notoloopedthai", "notosanskhmer", "khmeros", "khmerossystem",
+			"freeserif", "freesans", "freemono", "dejavu", "dejavusans":
+			trimmed = append(trimmed, fp)
+		}
+	}
+	code := fmt.Sprintf(`
+	package fontscan
+	import "github.com/go-text/typesetting/font"
+
+	// extracted from a linux system
+	var linuxSampleFontSet = 
+	%#v`, trimmed)
+	code = strings.ReplaceAll(code, "fontscan.", "")
+	code = strings.ReplaceAll(code, "Footprint{", "\n{")
+	code = strings.ReplaceAll(code, ", Index:0x0, Instance:0x0", "")
+	code = strings.ReplaceAll(code, ", isUserProvided:false", "")
+	code = strings.ReplaceAll(code, "Location:", "\nLocation:")
+	code = strings.ReplaceAll(code, "Runes:", "\nRunes:")
+	code = strings.ReplaceAll(code, "Langs:", "\nLangs:")
+	code = strings.ReplaceAll(code, "Aspect:", "\nAspect:")
+
+	err = os.WriteFile("fontmap_sample_test.go", []byte(code), os.ModePerm)
+	tu.AssertNoErr(t, err)
+
+	err = exec.Command("goimports", "-w", "fontmap_sample_test.go").Run()
+	tu.AssertNoErr(t, err)
+}
+
+func TestResolve_ScriptBengali(t *testing.T) {
+	fm := newSampleFontmap()
 
 	// make sure the same font is selected for a given script, when possible
 	text := []rune("হয় না।")
 	fm.SetQuery(Query{Families: []string{"Nimbus Sans"}})
 	runs := (&shaping.Segmenter{}).Split(shaping.Input{Text: text, RunEnd: len(text)}, fm)
 	tu.Assert(t, len(runs) == 1)
-	// only one font is loaded, so there is no Location.File collision
 	family, _ := fm.FontMetadata(runs[0].Face.Font)
 	tu.Assert(t, family == "lohitbengali")
 }
 
 func TestResolve_ScriptThaana(t *testing.T) {
-	fm := NewFontMap(log.New(io.Discard, "", 0))
-	fm.appendFootprints(thaanaFontSet...)
+	fm := newSampleFontmap()
 
 	// make sure the same font is selected for a given script, when possible
 	text := []rune("އުފަންވަނީ، ދަރަޖަ")
 	fm.SetQuery(Query{Families: []string{"Nimbus Sans"}})
 	runs := (&shaping.Segmenter{}).Split(shaping.Input{Text: text, RunEnd: len(text)}, fm)
 	tu.Assert(t, len(runs) == 1)
-	// only one font is loaded, so there is no Location.File collision
 	family, _ := fm.FontMetadata(runs[0].Face.Font)
 	tu.Assert(t, family == "freeserif")
+	tu.Assert(t, strings.HasSuffix(fm.FontLocation(runs[0].Face.Font).File, "FreeSerif.ttf"))
+}
+
+func TestResolve_SciptGujarati(t *testing.T) {
+	fm := newSampleFontmap()
+
+	text := []rune("ମୁଁ କାଚ ଖାଇପାରେ ଏବଂ ତାହା ମୋର କ୍ଷତି କରିନଥାଏ।")
+	fm.SetQuery(Query{Families: []string{"Nimbus Sans"}})
+	runs := (&shaping.Segmenter{}).Split(shaping.Input{Text: text, RunEnd: len(text)}, fm)
+	tu.Assert(t, len(runs) == 1)
+	family, _ := fm.FontMetadata(runs[0].Face.Font)
+	tu.Assert(t, family == "lohitodia")
+}
+
+func TestResolve_SciptArabic(t *testing.T) {
+	fm := newSampleFontmap()
+
+	text := []rune("میں کانچ کھا سکتا ہوں اور مجھے تکلیف نہیں ہوتی ۔")
+	fm.SetQuery(Query{Families: []string{"Nimbus Sans"}})
+	runs := (&shaping.Segmenter{}).Split(shaping.Input{Text: text, RunEnd: len(text)}, fm)
+	tu.Assert(t, len(runs) == 10)
+	family0, _ := fm.FontMetadata(runs[0].Face.Font)
+	family1, _ := fm.FontMetadata(runs[1].Face.Font)
+	tu.Assert(t, family0 == "dejavusans")
+	tu.Assert(t, family1 == "freeserif")
+}
+
+func TestResolve_SciptKhmer(t *testing.T) {
+	fm := newSampleFontmap()
+
+	text := []rune("ខ្ញុំអាចញុំកញ្ចក់បាន ដោយគ្មានបញ្ហារ")
+	fm.SetQuery(Query{Families: []string{"Nimbus Sans"}})
+	runs := (&shaping.Segmenter{}).Split(shaping.Input{Text: text, RunEnd: len(text)}, fm)
+	tu.Assert(t, len(runs) == 1)
+	family, _ := fm.FontMetadata(runs[0].Face.Font)
+	tu.Assert(t, family == "khmeros")
 }

fontscan/footprint.go

@@ -3,6 +3,8 @@ package fontscan
 import (
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/go-text/typesetting/font"
 	ot "github.com/go-text/typesetting/font/opentype"
@@ -98,6 +100,22 @@ func newFootprintFromLoader(ld *ot.Loader, isUserProvided bool, buffer scanBuffe
 	return out, buffer, nil
 }
 
+// returns true for .ttf and .ttc font files
+func (fp *Footprint) isTruetypeHint() bool {
+	switch strings.ToLower(filepath.Ext(fp.Location.File)) {
+	case ".ttf", ".ttc":
+		return true
+	default:
+		return false
+	}
+}
+
+// isMonoHint returns true if "mono" is included in the family name
+// this is not very precise but much more efficient than using [font.Font.IsMonospace]
+func (fp *Footprint) isMonoHint() bool {
+	return strings.Contains(fp.Family, "mono")
+}
+
 // loadFromDisk assume the footprint location refers to the file system
 func (fp *Footprint) loadFromDisk() (*font.Face, error) {
 	location := fp.Location

fontscan/langset.go

@@ -116,3 +116,93 @@ func (ls *LangSet) deserializeFrom(data []byte) (int, error) {
 	}
 	return langSetSize, nil
 }
+
+// This map returns a language tag that is reasonably
+// representative of the script. This will usually be the
+// most widely spoken or used language written in that script:
+// for instance, the sample language for `Cyrillic`
+// is 'ru' (Russian), the sample language for `Arabic` is 'ar'.
+//
+// For some scripts, no sample language will be returned because there
+// is no language that is sufficiently representative. The best
+// example of this is `Han`, where various different
+// variants of written Chinese, Japanese, and Korean all use
+// significantly different sets of Han characters and forms
+// of shared characters. No sample language can be provided
+// for many historical scripts as well.
+//
+// inspired by pango/pango-language.c
+var scriptToLang = map[language.Script]LangID{
+	language.Arabic:   langAr,
+	language.Armenian: langHy,
+	language.Bengali:  langBn,
+	// Used primarily in Taiwan, but not part of the standard
+	// zh-tw orthography
+	language.Bopomofo: 0,
+	language.Cherokee: langChr,
+	language.Coptic:   langCop,
+	language.Cyrillic: langRu,
+	// Deseret was used to write English
+	language.Deseret:    0,
+	language.Devanagari: langHi,
+	language.Ethiopic:   langAm,
+	language.Georgian:   langKa,
+	language.Gothic:     0,
+	language.Greek:      langEl,
+	language.Gujarati:   langGu,
+	language.Gurmukhi:   langPa,
+	language.Han:        0,
+	language.Hangul:     langKo,
+	language.Hebrew:     langHe,
+	language.Hiragana:   langJa,
+	language.Kannada:    langKn,
+	language.Katakana:   langJa,
+	language.Khmer:      langKm,
+	language.Lao:        langLo,
+	language.Latin:      langEn,
+	language.Malayalam:  langMl,
+	language.Mongolian:  langMn,
+	language.Myanmar:    langMy,
+	// Ogham was used to write old Irish
+	language.Ogham:               0,
+	language.Old_Italic:          0,
+	language.Oriya:               langOr,
+	language.Runic:               0,
+	language.Sinhala:             langSi,
+	language.Syriac:              langSyr,
+	language.Tamil:               langTa,
+	language.Telugu:              langTe,
+	language.Thaana:              langDv,
+	language.Thai:                langTh,
+	language.Tibetan:             langBo,
+	language.Canadian_Aboriginal: langIu,
+	language.Yi:                  0,
+	language.Tagalog:             langTl,
+	// Phillipino languages/scripts
+	language.Hanunoo:  langHnn,
+	language.Buhid:    langBku,
+	language.Tagbanwa: langTbw,
+
+	language.Braille: 0,
+	language.Cypriot: 0,
+	language.Limbu:   0,
+	// Used for Somali (so) in the past
+	language.Osmanya: 0,
+	// The Shavian alphabet was designed for English
+	language.Shavian:  0,
+	language.Linear_B: 0,
+	language.Tai_Le:   0,
+	language.Ugaritic: langUga,
+
+	language.New_Tai_Lue: 0,
+	language.Buginese:    langBug,
+	// The original script for Old Church Slavonic (chu), later
+	// written with Cyrillic
+	language.Glagolitic: 0,
+	// Used for for Berber (ber), but Arabic script is more common
+	language.Tifinagh:     0,
+	language.Syloti_Nagri: langSyl,
+	language.Old_Persian:  langPeo,
+
+	language.Nko: langNqo,
+}