fix: avoid topic fallback for non-Latin titles via pragmatic ASCII transliteration

docs/release/LICENSE

@@ -251,6 +251,7 @@ The following components are provided under the MIT License. See project link fo
    (MIT License) swr (https://github.com/vercel/swr) [link](./licenses/LICENSE-vercel-swr.txt)
    (MIT License) zustand (https://github.com/pmndrs/zustand) [link](./licenses/LICENSE-pmndrs-zustand.txt)
    (MIT License) mozillazg-go-pinyin (https://github.com/mozillazg/go-pinyin) [link](./licenses/LICENSE-mozillazg-go-pinyin.txt)
+   (MIT License) mozillazg-go-unidecode (https://github.com/mozillazg/go-unidecode) [link](./licenses/LICENSE-mozillazg-go-unidecode.txt)
    (MIT License) Machiel-slugify (https://github.com/Machiel/slugify) [link](./licenses/LICENSE-Machiel-slugify.txt)
    (MIT License) Masterminds-semver (https://github.com/Masterminds/semver) [link](./licenses/LICENSE-Masterminds-semver.txt)
    (MIT License) anargu-gin-brotli (https://github.com/anargu/gin-brotli) [link](./licenses/LICENSE-anargu-gin-brotli.txt)

docs/release/licenses/LICENSE-mozillazg-go-unidecode.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 mozillazg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/mark3labs/mcp-go v0.43.2
 	github.com/microcosm-cc/bluemonday v1.0.27
 	github.com/mozillazg/go-pinyin v0.20.0
+	github.com/mozillazg/go-unidecode v0.2.0
 	github.com/ory/dockertest/v3 v3.11.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/sashabaranov/go-openai v1.41.2

go.sum

@@ -462,6 +462,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mozillazg/go-pinyin v0.20.0 h1:BtR3DsxpApHfKReaPO1fCqF4pThRwH9uwvXzm+GnMFQ=
 github.com/mozillazg/go-pinyin v0.20.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
+github.com/mozillazg/go-unidecode v0.2.0 h1:vFGEzAH9KSwyWmXCOblazEWDh7fOkpmy/Z4ArmamSUc=
+github.com/mozillazg/go-unidecode v0.2.0/go.mod h1:zB48+/Z5toiRolOZy9ksLryJ976VIwmDmpQ2quyt1aA=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
 github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=

pkg/htmltext/htmltext.go

@@ -25,13 +25,16 @@ import (
 	"net/url"
 	"regexp"
 	"strings"
+	"sync/atomic"
+	"unicode"
 	"unicode/utf8"
 
 	"github.com/Machiel/slugify"
 	"github.com/apache/answer/pkg/checker"
 	"github.com/apache/answer/pkg/converter"
 	strip "github.com/grokify/html-strip-tags-go"
 	"github.com/mozillazg/go-pinyin"
+	"github.com/mozillazg/go-unidecode"
 )
 
 var (
@@ -47,8 +50,27 @@ var (
 		"\r", " ",
 		"\t", " ",
 	)
+
+	// Without this, pure non-Latin titles (Arabic, Cyrillic, Hebrew, ...) get
+	// stripped by slugify and collapse to the "topic" fallback. Chinese is
+	// handled separately by convertChinese.
+	transliterateNonLatin atomic.Bool
 )
 
+func init() {
+	transliterateNonLatin.Store(true)
+}
+
+// SetTransliterateNonLatin toggles non-Latin script transliteration for URL slugs.
+func SetTransliterateNonLatin(enabled bool) {
+	transliterateNonLatin.Store(enabled)
+}
+
+// IsTransliterateNonLatinEnabled reports whether non-Latin transliteration is on.
+func IsTransliterateNonLatinEnabled() bool {
+	return transliterateNonLatin.Load()
+}
+
 // ClearText clear HTML, get the clear text
 func ClearText(html string) string {
 	if html == "" {
@@ -66,6 +88,9 @@ func ClearText(html string) string {
 
 func UrlTitle(title string) (text string) {
 	title = convertChinese(title)
+	if transliterateNonLatin.Load() {
+		title = convertNonLatin(title)
+	}
 	title = clearEmoji(title)
 	title = slugify.Slugify(title)
 	title = url.QueryEscape(title)
@@ -95,6 +120,30 @@ func convertChinese(content string) string {
 	return strings.Join(pinyin.LazyConvert(content, nil), "-")
 }
 
+// Short-circuits on Latin-only / Chinese-only input so existing slugs stay byte-identical.
+func convertNonLatin(content string) string {
+	if !containsNonLatin(content) {
+		return content
+	}
+	return unidecode.Unidecode(content)
+}
+
+func containsNonLatin(content string) bool {
+	for _, r := range content {
+		switch {
+		case r < 0x0080: // ASCII
+			continue
+		case r >= 0x0080 && r <= 0x024F: // Latin-1 Supplement, Latin Extended-A/B
+			continue
+		case unicode.Is(unicode.Han, r): // handled by convertChinese
+			continue
+		case unicode.IsLetter(r):
+			return true
+		}
+	}
+	return false
+}
+
 func cutLongTitle(title string) string {
 	maxBytes := 150
 	if len(title) <= maxBytes {

pkg/htmltext/htmltext_test.go

@@ -87,6 +87,111 @@ func TestUrlTitle(t *testing.T) {
 	}
 }
 
+func TestUrlTitleTable(t *testing.T) {
+	// Long pure-Arabic title: 50 copies of the same Arabic word, joined by spaces.
+	// Unidecode of "كيف" is "kyf", so the slug becomes "kyf-" repeated and
+	// exceeds cutLongTitle's 150-byte cap.
+	longArabic := strings.Repeat("كيف ", 50)
+	wantLongArabic := strings.Repeat("kyf-", 37) + "ky" // 37*4 + 2 = 150 bytes
+
+	cases := []struct {
+		name  string
+		title string
+		want  string
+	}{
+		{
+			name:  "empty",
+			title: "",
+			want:  "topic",
+		},
+		{
+			name:  "pure latin unchanged",
+			title: "hello world",
+			want:  "hello-world",
+		},
+		{
+			// Pinyin conversion drops Latin runes by design — matches pre-fix behavior.
+			name:  "pure chinese unchanged",
+			title: "这是一个，标题，title",
+			want:  "zhe-shi-yi-ge-biao-ti",
+		},
+		{
+			// The fix: previously collapsed to "topic" for all of these scripts.
+			// Outputs are an ASCII approximation, not linguistically correct
+			// romanization — see PR description.
+			name:  "arabic transliterated",
+			title: "كيف حالك",
+			want:  "kyf-hlk",
+		},
+		{
+			name:  "mixed latin and arabic",
+			title: "مرحبا hello",
+			want:  "mrhb-hello",
+		},
+		{
+			name:  "thai transliterated",
+			title: "ไทย ไทย",
+			want:  "aithy-aithy",
+		},
+		{
+			name:  "japanese hiragana transliterated",
+			title: "こんにちは",
+			want:  "konnichiha",
+		},
+		{
+			// Japanese with Han-block kanji is caught by the pre-existing pinyin
+			// pre-step (Chinese reading, not Japanese), so this path is unchanged
+			// by this PR. Pinning to document the existing behavior.
+			name:  "japanese kanji goes through pinyin path unchanged",
+			title: "日本",
+			want:  "ri-ben",
+		},
+		{
+			name:  "korean transliterated",
+			title: "안녕하세요",
+			want:  "annyeonghaseyo",
+		},
+		{
+			name:  "hebrew transliterated",
+			title: "שלום עולם",
+			want:  "shlvm-vlm",
+		},
+		{
+			name:  "cyrillic transliterated",
+			title: "Привет мир",
+			want:  "privet-mir",
+		},
+		{
+			name:  "emoji only falls back to topic",
+			title: "😂😂😂",
+			want:  "topic",
+		},
+		{
+			name:  "long arabic truncates at cutLongTitle boundary",
+			title: longArabic,
+			want:  wantLongArabic,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := UrlTitle(tc.title)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestUrlTitleTransliterationToggle(t *testing.T) {
+	defer SetTransliterateNonLatin(true)
+
+	SetTransliterateNonLatin(false)
+	// With transliteration off, pure-Arabic titles collapse to the existing
+	// "topic" fallback (the pre-fix behavior).
+	assert.Equal(t, "topic", UrlTitle("كيف حالك"))
+
+	SetTransliterateNonLatin(true)
+	assert.Equal(t, "kyf-hlk", UrlTitle("كيف حالك"))
+}
+
 func TestFindFirstMatchedWord(t *testing.T) {
 	var (
 		expectedWord,