Initial support for localization and pluralization with go-i18n-JSON-v2 format (go-gitea#6203)

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6203
Reviewed-by: Gusted <[email protected]>
Reviewed-by: 0ko <[email protected]>

Author: 0ko

.deadcode-out

@@ -246,6 +246,7 @@ code.gitea.io/gitea/modules/translation
 	MockLocale.TrString
 	MockLocale.Tr
 	MockLocale.TrN
+	MockLocale.TrPluralString
 	MockLocale.TrSize
 	MockLocale.PrettyNumber
 

modules/translation/i18n/dummy.go

@@ -22,27 +22,19 @@ func (k *KeyLocale) HasKey(trKey string) bool {
 
 // TrHTML implements Locale.
 func (k *KeyLocale) TrHTML(trKey string, trArgs ...any) template.HTML {
-	args := slices.Clone(trArgs)
-	for i, v := range args {
-		switch v := v.(type) {
-		case nil, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, template.HTML:
-			// for most basic types (including template.HTML which is safe), just do nothing and use it
-		case string:
-			args[i] = template.HTMLEscapeString(v)
-		case fmt.Stringer:
-			args[i] = template.HTMLEscapeString(v.String())
-		default:
-			args[i] = template.HTMLEscapeString(fmt.Sprint(v))
-		}
-	}
-	return template.HTML(k.TrString(trKey, args...))
+	return template.HTML(k.TrString(trKey, PrepareArgsForHTML(trArgs...)...))
 }
 
 // TrString implements Locale.
 func (k *KeyLocale) TrString(trKey string, trArgs ...any) string {
 	return FormatDummy(trKey, trArgs...)
 }
 
+// TrPluralString implements Locale.
+func (k *KeyLocale) TrPluralString(count any, trKey string, trArgs ...any) template.HTML {
+	return template.HTML(FormatDummy(trKey, PrepareArgsForHTML(trArgs...)...))
+}
+
 func FormatDummy(trKey string, args ...any) string {
 	if len(args) == 0 {
 		return fmt.Sprintf("(%s)", trKey)

modules/translation/i18n/errors.go

@@ -8,6 +8,8 @@ import (
 )
 
 var (
-	ErrLocaleAlreadyExist = util.SilentWrap{Message: "lang already exists", Err: util.ErrAlreadyExist}
-	ErrUncertainArguments = util.SilentWrap{Message: "arguments to i18n should not contain uncertain slices", Err: util.ErrInvalidArgument}
+	ErrLocaleAlreadyExist      = util.SilentWrap{Message: "lang already exists", Err: util.ErrAlreadyExist}
+	ErrLocaleDoesNotExist      = util.SilentWrap{Message: "lang does not exist", Err: util.ErrNotExist}
+	ErrTranslationDoesNotExist = util.SilentWrap{Message: "translation does not exist", Err: util.ErrNotExist}
+	ErrUncertainArguments      = util.SilentWrap{Message: "arguments to i18n should not contain uncertain slices", Err: util.ErrInvalidArgument}
 )

modules/translation/i18n/i18n.go

@@ -8,11 +8,28 @@ import (
 	"io"
 )
 
+type (
+	PluralFormIndex uint8
+	PluralFormRule  func(int64) PluralFormIndex
+)
+
+const (
+	PluralFormZero PluralFormIndex = iota
+	PluralFormOne
+	PluralFormTwo
+	PluralFormFew
+	PluralFormMany
+	PluralFormOther
+)
+
 var DefaultLocales = NewLocaleStore()
 
 type Locale interface {
 	// TrString translates a given key and arguments for a language
 	TrString(trKey string, trArgs ...any) string
+	// TrPluralString translates a given pluralized key and arguments for a language.
+	// This function returns an error if new-style support for the given key is not available.
+	TrPluralString(count any, trKey string, trArgs ...any) template.HTML
 	// TrHTML translates a given key and arguments for a language, string arguments are escaped to HTML
 	TrHTML(trKey string, trArgs ...any) template.HTML
 	// HasKey reports if a locale has a translation for a given key
@@ -31,8 +48,10 @@ type LocaleStore interface {
 	Locale(langName string) (Locale, bool)
 	// HasLang returns whether a given language is present in the store
 	HasLang(langName string) bool
-	// AddLocaleByIni adds a new language to the store
-	AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error
+	// AddLocaleByIni adds a new old-style language to the store
+	AddLocaleByIni(langName, langDesc string, pluralRule PluralFormRule, source, moreSource []byte) error
+	// AddLocaleByJSON adds new-style content to an existing language to the store
+	AddToLocaleFromJSON(langName string, source []byte) error
 }
 
 // ResetDefaultLocales resets the current default locales

modules/translation/i18n/i18n_test.go

@@ -12,6 +12,26 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+var MockPluralRule PluralFormRule = func(n int64) PluralFormIndex {
+	if n == 0 {
+		return PluralFormZero
+	}
+	if n == 1 {
+		return PluralFormOne
+	}
+	if n >= 2 && n <= 4 {
+		return PluralFormFew
+	}
+	return PluralFormOther
+}
+
+var MockPluralRuleEnglish PluralFormRule = func(n int64) PluralFormIndex {
+	if n == 1 {
+		return PluralFormOne
+	}
+	return PluralFormOther
+}
+
 func TestLocaleStore(t *testing.T) {
 	testData1 := []byte(`
 .dot.name = Dot Name
@@ -27,11 +47,48 @@ fmt = %[2]s %[1]s
 
 [section]
 sub = Changed Sub String
+commits = fallback value for commits
+`)
+
+	testDataJSON2 := []byte(`
+{
+	"section.json": "the JSON is %s",
+	"section.commits": {
+		"one": "one %d commit",
+		"few": "some %d commits",
+		"other": "lots of %d commits"
+	},
+	"section.incomplete": {
+		"few": "some %d objects (translated)"
+	},
+	"nested": {
+		"outer": {
+			"inner": {
+				"json": "Hello World",
+				"issue": {
+					"one": "one %d issue",
+					"few": "some %d issues",
+					"other": "lots of %d issues"
+				}
+			}
+		}
+	}
+}
+`)
+	testDataJSON1 := []byte(`
+{
+	"section.incomplete": {
+		"one": "[untranslated] some %d object",
+		"other": "[untranslated] some %d objects"
+	}
+}
 `)
 
 	ls := NewLocaleStore()
-	require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, nil))
-	require.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2, nil))
+	require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", MockPluralRuleEnglish, testData1, nil))
+	require.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", MockPluralRule, testData2, nil))
+	require.NoError(t, ls.AddToLocaleFromJSON("lang1", testDataJSON1))
+	require.NoError(t, ls.AddToLocaleFromJSON("lang2", testDataJSON2))
 	ls.SetDefaultLang("lang1")
 
 	lang1, _ := ls.Locale("lang1")
@@ -56,6 +113,45 @@ sub = Changed Sub String
 	result2 := lang2.TrHTML("section.mixed", "a&b")
 	assert.EqualValues(t, `test value; <span style="color: red; background: none;">a&amp;b</span>`, result2)
 
+	result = lang2.TrString("section.json", "valid")
+	assert.Equal(t, "the JSON is valid", result)
+
+	result = lang2.TrString("nested.outer.inner.json")
+	assert.Equal(t, "Hello World", result)
+
+	result = lang2.TrString("section.commits")
+	assert.Equal(t, "lots of %d commits", result)
+
+	result2 = lang2.TrPluralString(1, "section.commits", 1)
+	assert.EqualValues(t, "one 1 commit", result2)
+
+	result2 = lang2.TrPluralString(3, "section.commits", 3)
+	assert.EqualValues(t, "some 3 commits", result2)
+
+	result2 = lang2.TrPluralString(8, "section.commits", 8)
+	assert.EqualValues(t, "lots of 8 commits", result2)
+
+	result2 = lang2.TrPluralString(0, "section.commits")
+	assert.EqualValues(t, "section.commits", result2)
+
+	result2 = lang2.TrPluralString(1, "nested.outer.inner.issue", 1)
+	assert.EqualValues(t, "one 1 issue", result2)
+
+	result2 = lang2.TrPluralString(3, "nested.outer.inner.issue", 3)
+	assert.EqualValues(t, "some 3 issues", result2)
+
+	result2 = lang2.TrPluralString(9, "nested.outer.inner.issue", 9)
+	assert.EqualValues(t, "lots of 9 issues", result2)
+
+	result2 = lang2.TrPluralString(3, "section.incomplete", 3)
+	assert.EqualValues(t, "some 3 objects (translated)", result2)
+
+	result2 = lang2.TrPluralString(1, "section.incomplete", 1)
+	assert.EqualValues(t, "[untranslated] some 1 object", result2)
+
+	result2 = lang2.TrPluralString(7, "section.incomplete", 7)
+	assert.EqualValues(t, "[untranslated] some 7 objects", result2)
+
 	langs, descs := ls.ListLangNameDesc()
 	assert.ElementsMatch(t, []string{"lang1", "lang2"}, langs)
 	assert.ElementsMatch(t, []string{"Lang1", "Lang2"}, descs)
@@ -77,7 +173,7 @@ c=22
 `)
 
 	ls := NewLocaleStore()
-	require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, testData2))
+	require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", MockPluralRule, testData1, testData2))
 	lang1, _ := ls.Locale("lang1")
 	assert.Equal(t, "11", lang1.TrString("a"))
 	assert.Equal(t, "21", lang1.TrString("b"))
@@ -118,7 +214,7 @@ func (e *errorPointerReceiver) Error() string {
 
 func TestLocaleWithTemplate(t *testing.T) {
 	ls := NewLocaleStore()
-	require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", []byte(`key=<a>%s</a>`), nil))
+	require.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", MockPluralRule, []byte(`key=<a>%s</a>`), nil))
 	lang1, _ := ls.Locale("lang1")
 
 	tmpl := template.New("test").Funcs(template.FuncMap{"tr": lang1.TrHTML})
@@ -181,7 +277,7 @@ func TestLocaleStoreQuirks(t *testing.T) {
 
 	for _, testData := range testDataList {
 		ls := NewLocaleStore()
-		err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil)
+		err := ls.AddLocaleByIni("lang1", "Lang1", nil, []byte("a="+testData.in), nil)
 		lang1, _ := ls.Locale("lang1")
 		require.NoError(t, err, testData.hint)
 		assert.Equal(t, testData.out, lang1.TrString("a"), testData.hint)