From d41a598e844383a2ca0cf97c0203dc1150fecd8f Mon Sep 17 00:00:00 2001 From: ahmedqasid Date: Fri, 15 May 2026 16:00:58 +0300 Subject: [PATCH 1/3] fix: transliterate non-Latin titles in URL slugs Pure-Arabic (and other non-Latin) titles previously got stripped by slugify and collapsed to the "topic" fallback, so every Arabic question landed at /questions//topic. Mirror the existing convertChinese pre-step using go-unidecode so titles in Arabic, Cyrillic, Hebrew, Thai etc. produce a readable ASCII slug. Latin-only and Chinese-only inputs short-circuit and remain byte-identical to the previous output. Gated by a package-level atomic flag (default on) exposed via SetTransliterateNonLatin so an admin toggle can be wired up in a follow-up PR without re-plumbing call sites. --- go.mod | 1 + go.sum | 2 + pkg/htmltext/htmltext.go | 49 +++++++++++++++++++++++++ pkg/htmltext/htmltext_test.go | 69 +++++++++++++++++++++++++++++++++++ 4 files changed, 121 insertions(+) diff --git a/go.mod b/go.mod index 89fd71460..11c3a8168 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/mark3labs/mcp-go v0.43.2 github.com/microcosm-cc/bluemonday v1.0.27 github.com/mozillazg/go-pinyin v0.20.0 + github.com/mozillazg/go-unidecode v0.2.0 github.com/ory/dockertest/v3 v3.11.0 github.com/robfig/cron/v3 v3.0.1 github.com/sashabaranov/go-openai v1.41.2 diff --git a/go.sum b/go.sum index bf30d85b3..be61e11f6 100644 --- a/go.sum +++ b/go.sum @@ -462,6 +462,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ= github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc= +github.com/mozillazg/go-unidecode v0.2.0 h1:vFGEzAH9KSwyWmXCOblazEWDh7fOkpmy/Z4ArmamSUc= +github.com/mozillazg/go-unidecode v0.2.0/go.mod h1:zB48+/Z5toiRolOZy9ksLryJ976VIwmDmpQ2quyt1aA= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= diff --git a/pkg/htmltext/htmltext.go b/pkg/htmltext/htmltext.go index e2e017c8d..929080838 100644 --- a/pkg/htmltext/htmltext.go +++ b/pkg/htmltext/htmltext.go @@ -25,6 +25,8 @@ import ( "net/url" "regexp" "strings" + "sync/atomic" + "unicode" "unicode/utf8" "github.com/Machiel/slugify" @@ -32,6 +34,7 @@ import ( "github.com/apache/answer/pkg/converter" strip "github.com/grokify/html-strip-tags-go" "github.com/mozillazg/go-pinyin" + "github.com/mozillazg/go-unidecode" ) var ( @@ -47,8 +50,27 @@ var ( "\r", " ", "\t", " ", ) + + // Without this, pure non-Latin titles (Arabic, Cyrillic, Hebrew, ...) get + // stripped by slugify and collapse to the "topic" fallback. Chinese is + // handled separately by convertChinese. + transliterateNonLatin atomic.Bool ) +func init() { + transliterateNonLatin.Store(true) +} + +// SetTransliterateNonLatin toggles non-Latin script transliteration for URL slugs. +func SetTransliterateNonLatin(enabled bool) { + transliterateNonLatin.Store(enabled) +} + +// IsTransliterateNonLatinEnabled reports whether non-Latin transliteration is on. +func IsTransliterateNonLatinEnabled() bool { + return transliterateNonLatin.Load() +} + // ClearText clear HTML, get the clear text func ClearText(html string) string { if html == "" { @@ -66,6 +88,9 @@ func ClearText(html string) string { func UrlTitle(title string) (text string) { title = convertChinese(title) + if transliterateNonLatin.Load() { + title = convertNonLatin(title) + } title = clearEmoji(title) title = slugify.Slugify(title) title = url.QueryEscape(title) @@ -95,6 +120,30 @@ func convertChinese(content string) string { return strings.Join(pinyin.LazyConvert(content, nil), "-") } +// Short-circuits on Latin-only / Chinese-only input so existing slugs stay byte-identical. +func convertNonLatin(content string) string { + if !containsNonLatin(content) { + return content + } + return unidecode.Unidecode(content) +} + +func containsNonLatin(content string) bool { + for _, r := range content { + switch { + case r < 0x0080: // ASCII + continue + case r >= 0x0080 && r <= 0x024F: // Latin-1 Supplement, Latin Extended-A/B + continue + case unicode.Is(unicode.Han, r): // handled by convertChinese + continue + case unicode.IsLetter(r): + return true + } + } + return false +} + func cutLongTitle(title string) string { maxBytes := 150 if len(title) <= maxBytes { diff --git a/pkg/htmltext/htmltext_test.go b/pkg/htmltext/htmltext_test.go index 39de9e960..3fd263db6 100644 --- a/pkg/htmltext/htmltext_test.go +++ b/pkg/htmltext/htmltext_test.go @@ -87,6 +87,75 @@ func TestUrlTitle(t *testing.T) { } } +func TestUrlTitleTable(t *testing.T) { + // Long pure-Arabic title: 50 copies of the same Arabic word, joined by spaces. + // Unidecode of "كيف" is "kyf", so the slug becomes "kyf-" repeated and + // exceeds cutLongTitle's 150-byte cap. + longArabic := strings.Repeat("كيف ", 50) + wantLongArabic := strings.Repeat("kyf-", 37) + "ky" // 37*4 + 2 = 150 bytes + + cases := []struct { + name string + title string + want string + }{ + { + name: "empty", + title: "", + want: "topic", + }, + { + name: "pure latin unchanged", + title: "hello world", + want: "hello-world", + }, + { + // Pinyin conversion drops Latin runes by design — matches pre-fix behavior. + name: "pure chinese unchanged", + title: "这是一个，标题，title", + want: "zhe-shi-yi-ge-biao-ti", + }, + { + name: "pure arabic transliterated", + title: "كيف حالك", + want: "kyf-hlk", + }, + { + name: "mixed latin and arabic", + title: "hello كيف", + want: "hello-kyf", + }, + { + name: "emoji only falls back to topic", + title: "😂😂😂", + want: "topic", + }, + { + name: "long arabic truncates at cutLongTitle boundary", + title: longArabic, + want: wantLongArabic, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := UrlTitle(tc.title) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestUrlTitleTransliterationToggle(t *testing.T) { + defer SetTransliterateNonLatin(true) + + SetTransliterateNonLatin(false) + // With transliteration off, pure-Arabic titles collapse to the existing + // "topic" fallback (the pre-fix behavior). + assert.Equal(t, "topic", UrlTitle("كيف حالك")) + + SetTransliterateNonLatin(true) + assert.Equal(t, "kyf-hlk", UrlTitle("كيف حالك")) +} + func TestFindFirstMatchedWord(t *testing.T) { var ( expectedWord, From fd84008766a4b196d10bd54a4d9762f28274e628 Mon Sep 17 00:00:00 2001 From: ahmedqasid Date: Tue, 19 May 2026 12:29:09 +0300 Subject: [PATCH 2/3] test: cover broader non-Latin script matrix in UrlTitle Reviewer pointed out the fix changes slug generation for many non-Latin scripts, not just Arabic. Pin the actual behavior across Thai, Japanese hiragana, Korean, Hebrew, and Cyrillic so the test surface matches the real scope of the change. Also pin the pre-existing Japanese-kanji-via-pinyin path so reviewers can see it is unchanged by this PR. --- pkg/htmltext/htmltext_test.go | 42 ++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/pkg/htmltext/htmltext_test.go b/pkg/htmltext/htmltext_test.go index 3fd263db6..bcedcb3c8 100644 --- a/pkg/htmltext/htmltext_test.go +++ b/pkg/htmltext/htmltext_test.go @@ -116,14 +116,50 @@ func TestUrlTitleTable(t *testing.T) { want: "zhe-shi-yi-ge-biao-ti", }, { - name: "pure arabic transliterated", + // The fix: previously collapsed to "topic" for all of these scripts. + // Outputs are an ASCII approximation, not linguistically correct + // romanization — see PR description. + name: "arabic transliterated", title: "كيف حالك", want: "kyf-hlk", }, { name: "mixed latin and arabic", - title: "hello كيف", - want: "hello-kyf", + title: "مرحبا hello", + want: "mrhb-hello", + }, + { + name: "thai transliterated", + title: "ไทย ไทย", + want: "aithy-aithy", + }, + { + name: "japanese hiragana transliterated", + title: "こんにちは", + want: "konnichiha", + }, + { + // Japanese with Han-block kanji is caught by the pre-existing pinyin + // pre-step (Chinese reading, not Japanese), so this path is unchanged + // by this PR. Pinning to document the existing behavior. + name: "japanese kanji goes through pinyin path unchanged", + title: "日本", + want: "ri-ben", + }, + { + name: "korean transliterated", + title: "안녕하세요", + want: "annyeonghaseyo", + }, + { + name: "hebrew transliterated", + title: "שלום עולם", + want: "shlvm-vlm", + }, + { + name: "cyrillic transliterated", + title: "Привет мир", + want: "privet-mir", }, { name: "emoji only falls back to topic", From 8ab3b9f96f920dd0a14f1c52f18ac58eb112f6c0 Mon Sep 17 00:00:00 2001 From: ahmedqasid Date: Tue, 2 Jun 2026 01:42:10 +0300 Subject: [PATCH 3/3] docs: add license metadata for go-unidecode dependency Add the upstream MIT license text for github.com/mozillazg/go-unidecode under docs/release/licenses/ and register it in the release license inventory (docs/release/LICENSE), consistent with the existing mozillazg-go-pinyin and Machiel-slugify entries. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/release/LICENSE | 1 + .../LICENSE-mozillazg-go-unidecode.txt | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 docs/release/licenses/LICENSE-mozillazg-go-unidecode.txt diff --git a/docs/release/LICENSE b/docs/release/LICENSE index 926d64b79..ed0e73930 100644 --- a/docs/release/LICENSE +++ b/docs/release/LICENSE @@ -251,6 +251,7 @@ The following components are provided under the MIT License. See project link fo (MIT License) swr (https://github.com/vercel/swr) [link](./licenses/LICENSE-vercel-swr.txt) (MIT License) zustand (https://github.com/pmndrs/zustand) [link](./licenses/LICENSE-pmndrs-zustand.txt) (MIT License) mozillazg-go-pinyin (https://github.com/mozillazg/go-pinyin) [link](./licenses/LICENSE-mozillazg-go-pinyin.txt) + (MIT License) mozillazg-go-unidecode (https://github.com/mozillazg/go-unidecode) [link](./licenses/LICENSE-mozillazg-go-unidecode.txt) (MIT License) Machiel-slugify (https://github.com/Machiel/slugify) [link](./licenses/LICENSE-Machiel-slugify.txt) (MIT License) Masterminds-semver (https://github.com/Masterminds/semver) [link](./licenses/LICENSE-Masterminds-semver.txt) (MIT License) anargu-gin-brotli (https://github.com/anargu/gin-brotli) [link](./licenses/LICENSE-anargu-gin-brotli.txt) diff --git a/docs/release/licenses/LICENSE-mozillazg-go-unidecode.txt b/docs/release/licenses/LICENSE-mozillazg-go-unidecode.txt new file mode 100644 index 000000000..8a7780fcc --- /dev/null +++ b/docs/release/licenses/LICENSE-mozillazg-go-unidecode.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 mozillazg + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.