Revert "Add CJK (Chinese, Japanese, Korean) search support"

This commit is contained in:
Jorge Manrubia
2026-01-08 21:07:14 +01:00
committed by GitHub
parent ca68264c65
commit 3457c871e1
8 changed files with 5 additions and 174 deletions
-28
View File
@@ -82,34 +82,6 @@ class Search::HighlighterTest < ActiveSupport::TestCase
assert_equal "&lt;script&gt;#{mark('test')}&lt;/script&gt;", result
end
test "highlight Chinese characters" do
highlighter = Search::Highlighter.new("测试")
result = highlighter.highlight("这是一个测试文本")
assert_equal "这是一个#{mark('测试')}文本", result
end
test "highlight Japanese characters" do
highlighter = Search::Highlighter.new("テスト")
result = highlighter.highlight("これはテストです")
assert_equal "これは#{mark('テスト')}です", result
end
test "highlight Korean characters" do
highlighter = Search::Highlighter.new("테스트")
result = highlighter.highlight("이것은 테스트입니다")
assert_equal "이것은 #{mark('테스트')}입니다", result
end
test "highlight mixed CJK and English" do
highlighter = Search::Highlighter.new("world 世界")
result = highlighter.highlight("hello world 你好世界")
assert_equal "hello #{mark('world')} 你好#{mark('世界')}", result
end
private
def mark(text)
"#{Search::Highlighter::OPENING_MARK}#{text}#{Search::Highlighter::CLOSING_MARK}"
-57
View File
@@ -1,57 +0,0 @@
require "test_helper"
class Search::QueryTest < ActiveSupport::TestCase
setup do
@account = accounts(:"37s")
Current.account = @account
end
test "sanitize preserves ASCII words" do
query = build_query("hello world")
assert_equal "hello world", query.terms
end
test "sanitize preserves Chinese characters" do
query = build_query("测试文本")
assert_equal "测试文本", query.terms
end
test "sanitize preserves Japanese characters" do
query = build_query("テスト")
assert_equal "テスト", query.terms
end
test "sanitize preserves Korean characters" do
query = build_query("테스트")
assert_equal "테스트", query.terms
end
test "sanitize preserves mixed CJK and English" do
query = build_query("hello 世界 test")
assert_equal "hello 世界 test", query.terms
end
test "sanitize removes special characters but preserves CJK" do
query = build_query("测试@文本")
assert_equal "测试 文本", query.terms
end
test "sanitize preserves quoted phrases with CJK" do
query = build_query('"你好世界"')
assert_equal '"你好世界"', query.terms
end
private
def build_query(terms)
query = Search::Query.wrap(terms)
query.validate
query
end
end
-36
View File
@@ -12,40 +12,4 @@ class Search::StemmerTest < ActiveSupport::TestCase
assert_equal "test run jump walk", result
end
test "split Chinese characters for FTS indexing" do
result = Search::Stemmer.stem("测试")
assert_equal "测 试", result
end
test "split Japanese characters for FTS indexing" do
result = Search::Stemmer.stem("テスト")
assert_equal "テ ス ト", result
end
test "split Korean characters for FTS indexing" do
result = Search::Stemmer.stem("테스트")
assert_equal "테 스 트", result
end
test "mixed CJK and English" do
result = Search::Stemmer.stem("running 测试 test")
assert_equal "run 测 试 test", result
end
test "mixed CJK and English without spaces" do
result = Search::Stemmer.stem("hello世界test")
assert_equal "hello 世 界 test", result
end
test "CJK punctuation is treated as separator" do
result = Search::Stemmer.stem("你好。世界")
assert_equal "你 好 世 界", result
end
end