feat(locale Iter): properly support trPluralString

This requires using the more complicated parsing from localestore.go

In order to avoid future code drift and code duplication,
localestore.go was refactored to call IterateMessagesContent instead of
essentially duplicating the code of RecursivelyAddTranslationsFromJSON
with small adjustments.

locale/utils.go was moved to translation/localeiter/utils.go
in order to avoid spreading translation-related routines among completely
different places.
This commit is contained in:
Ellen Emilia Anna Zscheile 2025-04-02 02:19:51 +02:00
parent f691f03741
commit 95136fa06b
6 changed files with 73 additions and 66 deletions

View file

@ -192,6 +192,9 @@ forgejo.org/modules/translation
MockLocale.HasKey
MockLocale.PrettyNumber
forgejo.org/modules/translation/localeiter
IterateMessagesContent
forgejo.org/modules/util
OptionalArg

View file

@ -18,8 +18,8 @@ import (
tmplParser "text/template/parse"
"forgejo.org/modules/container"
"forgejo.org/modules/locale"
fjTemplates "forgejo.org/modules/templates"
"forgejo.org/modules/translation/localeiter"
"forgejo.org/modules/util"
)
@ -300,10 +300,6 @@ func main() {
}
msgids := make(container.Set[string])
onMsgid := func(trKey, trValue string) error {
msgids[trKey] = struct{}{}
return nil
}
localeFile := filepath.Join(filepath.Join("options", "locale"), "locale_en-US.ini")
localeContent, err := os.ReadFile(localeFile)
@ -312,7 +308,10 @@ func main() {
os.Exit(2)
}
if err = locale.IterateMessagesContent(localeContent, onMsgid); err != nil {
if err = localeiter.IterateMessagesContent(localeContent, func(trKey, trValue string) error {
msgids[trKey] = struct{}{}
return nil
}); err != nil {
fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
os.Exit(2)
}
@ -324,7 +323,11 @@ func main() {
os.Exit(2)
}
if err := locale.IterateMessagesNextContent(localeContent, onMsgid); err != nil {
if err := localeiter.IterateMessagesNextContent(localeContent, func(trKey, pluralForm, trValue string) error {
// ignore plural form
msgids[trKey] = struct{}{}
return nil
}); err != nil {
fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
os.Exit(2)
}

View file

@ -14,7 +14,7 @@ import (
"slices"
"strings"
"forgejo.org/modules/locale"
"forgejo.org/modules/translation/localeiter"
"github.com/microcosm-cc/bluemonday"
"github.com/sergi/go-diff/diffmatchpatch"
@ -100,7 +100,7 @@ func checkValue(trKey, value string) []string {
func checkLocaleContent(localeContent []byte) []string {
errors := []string{}
if err := locale.IterateMessagesContent(localeContent, func(trKey, trValue string) error {
if err := localeiter.IterateMessagesContent(localeContent, func(trKey, trValue string) error {
errors = append(errors, checkValue(trKey, trValue)...)
return nil
}); err != nil {
@ -113,8 +113,12 @@ func checkLocaleContent(localeContent []byte) []string {
func checkLocaleNextContent(localeContent []byte) []string {
errors := []string{}
if err := locale.IterateMessagesNextContent(localeContent, func(trKey, trValue string) error {
errors = append(errors, checkValue(trKey, trValue)...)
if err := localeiter.IterateMessagesNextContent(localeContent, func(trKey, pluralForm, trValue string) error {
fullKey := trKey
if pluralForm != "" {
fullKey = trKey + "." + pluralForm
}
errors = append(errors, checkValue(fullKey, trValue)...)
return nil
}); err != nil {
panic(err)

View file

@ -91,4 +91,16 @@ func TestNextLocalizationPolicy(t *testing.T) {
"settings.hidden_comment_types_description": "\"<not-an-allowed-key> <label>\""
}`)))
})
t.Run("Plural form", func(t *testing.T) {
assert.Equal(t, []string{"repo.pulls.title_desc: key = \x1b[31m<a href=\"https://example.com\">\x1b[0m"}, checkLocaleNextContent([]byte(`{"repo.pulls.title_desc": {
"few": "key = <a href=\"%s\">",
"other": "key = <a href=\"https://example.com\">"
}}`)))
assert.Equal(t, []string{"repo.pulls.title_desc.few: key = \x1b[31m<a href=\"https://example.com\">\x1b[0m"}, checkLocaleNextContent([]byte(`{"repo.pulls.title_desc": {
"few": "key = <a href=\"https://example.com\">",
"other": "key = <a href=\"%s\">"
}}`)))
})
}

View file

@ -8,9 +8,9 @@ import (
"html/template"
"slices"
"forgejo.org/modules/json"
"forgejo.org/modules/log"
"forgejo.org/modules/setting"
"forgejo.org/modules/translation/localeiter"
"forgejo.org/modules/util"
)
@ -94,55 +94,20 @@ func (store *localeStore) AddLocaleByIni(langName, langDesc string, pluralRule P
return nil
}
func RecursivelyAddTranslationsFromJSON(locale *locale, object map[string]any, prefix string) error {
for key, value := range object {
var fullkey string
if prefix != "" {
fullkey = prefix + "." + key
} else {
fullkey = key
}
switch v := value.(type) {
case string:
// Check whether we are adding a plural form to the parent object, or a new nested JSON object.
switch key {
case "zero", "one", "two", "few", "many":
locale.newStyleMessages[prefix+PluralFormSeparator+key] = v
case "other":
locale.newStyleMessages[prefix] = v
default:
locale.newStyleMessages[fullkey] = v
}
case map[string]any:
err := RecursivelyAddTranslationsFromJSON(locale, v, fullkey)
if err != nil {
return err
}
case nil:
default:
return fmt.Errorf("Unrecognized JSON value '%s'", value)
}
}
return nil
}
func (store *localeStore) AddToLocaleFromJSON(langName string, source []byte) error {
locale, ok := store.localeMap[langName]
if !ok {
return ErrLocaleDoesNotExist
}
var result map[string]any
if err := json.Unmarshal(source, &result); err != nil {
return err
}
return RecursivelyAddTranslationsFromJSON(locale, result, "")
return localeiter.IterateMessagesNextContent(source, func(key, pluralForm, value string) error {
msgKey := key
if pluralForm != "" {
msgKey = key + PluralFormSeparator + pluralForm
}
locale.newStyleMessages[msgKey] = value
return nil
})
}
func (l *locale) LookupNewStyleMessage(trKey string) string {

View file

@ -4,7 +4,7 @@
// extracted from `/build/lint-locale.go`, `/build/lint-locale-usage.go`
package locale
package localeiter
import (
"encoding/json" //nolint:depguard
@ -27,6 +27,9 @@ func IterateMessagesContent(localeContent []byte, onMsgid func(string, string) e
for _, section := range cfg.Sections() {
for _, key := range section.Keys() {
var trKey string
// see https://codeberg.org/forgejo/discussions/issues/104
// https://github.com/WeblateOrg/weblate/issues/10831
// for an explanation of why "common" is an alternative
if section.Name() == "" || section.Name() == "DEFAULT" || section.Name() == "common" {
trKey = key.Name()
} else {
@ -41,34 +44,51 @@ func IterateMessagesContent(localeContent []byte, onMsgid func(string, string) e
return nil
}
func iterateMessagesNextInner(onMsgid func(string, string) error, data map[string]any, trKey ...string) error {
func iterateMessagesNextInner(onMsgid func(string, string, string) error, data map[string]any, trKey string) error {
for key, value := range data {
currentKey := key
if len(trKey) == 1 {
currentKey = trKey[0] + "." + key
fullKey := key
if trKey != "" {
fullKey = trKey + "." + key
}
switch value := value.(type) {
case string:
if err := onMsgid(currentKey, value); err != nil {
// Check whether we are adding a plural form to the parent object, or a new nested JSON object.
realKey := trKey
pluralSuffix := ""
switch key {
case "zero", "one", "two", "few", "many":
pluralSuffix = key
case "other":
// do nothing
default:
realKey = fullKey
}
if err := onMsgid(realKey, pluralSuffix, value); err != nil {
return err
}
case map[string]any:
if err := iterateMessagesNextInner(onMsgid, value, currentKey); err != nil {
if err := iterateMessagesNextInner(onMsgid, value, fullKey); err != nil {
return err
}
case nil:
// do nothing
default:
return fmt.Errorf("unexpected type: %s - %T", currentKey, value)
return fmt.Errorf("Unexpected JSON type: %s - %T", fullKey, value)
}
}
return nil
}
func IterateMessagesNextContent(localeContent []byte, onMsgid func(string, string) error) error {
func IterateMessagesNextContent(localeContent []byte, onMsgid func(string, string, string) error) error {
var localeData map[string]any
if err := json.Unmarshal(localeContent, &localeData); err != nil {
return err
}
return iterateMessagesNextInner(onMsgid, localeData)
return iterateMessagesNextInner(onMsgid, localeData, "")
}