2a4fbecbda
The search functionality was silently dropping CJK characters because:
1. Query sanitization used `\w` which only matches ASCII word characters
2. Stemmer split by whitespace, which doesn't work for CJK languages
3. Highlighter used `\b` word boundaries that don't apply to CJK
This commit fixes all three issues:
- Query: Use `\p{L}\p{N}` (Unicode letters/numbers) instead of `\w`
- Stemmer: Preserve CJK characters as-is without stemming, since CJK
languages don't have stemming rules like English
- Highlighter: Skip word boundary matching for CJK terms
Also extracts `CJK_PATTERN` to `Search` module to avoid duplication.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
118 lines
3.8 KiB
Ruby
118 lines
3.8 KiB
Ruby
require "test_helper"
|
|
|
|
class Search::HighlighterTest < ActiveSupport::TestCase
|
|
test "highlight simple word match" do
|
|
highlighter = Search::Highlighter.new("hello")
|
|
result = highlighter.highlight("Hello world")
|
|
|
|
assert_equal "#{mark('Hello')} world", result
|
|
end
|
|
|
|
test "highlight multiple occurrences" do
|
|
highlighter = Search::Highlighter.new("test")
|
|
result = highlighter.highlight("This is a test and another test")
|
|
|
|
assert_equal "This is a #{mark('test')} and another #{mark('test')}", result
|
|
end
|
|
|
|
test "highlight case insensitive" do
|
|
highlighter = Search::Highlighter.new("ruby")
|
|
result = highlighter.highlight("Ruby is great and RUBY rocks")
|
|
|
|
assert_equal "#{mark('Ruby')} is great and #{mark('RUBY')} rocks", result
|
|
end
|
|
|
|
test "highlight quoted phrases" do
|
|
highlighter = Search::Highlighter.new('"hello world"')
|
|
result = highlighter.highlight("Say hello world to everyone")
|
|
|
|
assert_equal "Say #{mark('hello world')} to everyone", result
|
|
end
|
|
|
|
test "snippet returns full text with highlights when under max words" do
|
|
highlighter = Search::Highlighter.new("ruby")
|
|
result = highlighter.snippet("Ruby is great", max_words: 20)
|
|
|
|
assert_equal "#{mark('Ruby')} is great", result
|
|
end
|
|
|
|
test "snippet creates excerpt around match" do
|
|
highlighter = Search::Highlighter.new("match")
|
|
text = "word " * 10 + "match " + "word " * 10
|
|
result = highlighter.snippet(text, max_words: 10)
|
|
|
|
assert result.start_with?("...")
|
|
assert result.end_with?("...")
|
|
assert_includes result, mark("match")
|
|
end
|
|
|
|
test "snippet adds leading ellipsis when match is not at start" do
|
|
highlighter = Search::Highlighter.new("middle")
|
|
text = "word " * 20 + "middle"
|
|
result = highlighter.snippet(text, max_words: 10)
|
|
|
|
assert result.start_with?("...")
|
|
assert_not result.end_with?("...")
|
|
assert_includes result, mark("middle")
|
|
end
|
|
|
|
test "snippet adds trailing ellipsis when text continues after excerpt" do
|
|
highlighter = Search::Highlighter.new("start")
|
|
text = "start " + "word " * 30
|
|
result = highlighter.snippet(text, max_words: 10)
|
|
|
|
assert result.end_with?("...")
|
|
assert_not result.start_with?("...")
|
|
assert_includes result, mark("start")
|
|
end
|
|
|
|
test "snippet falls back to truncation when no match found" do
|
|
highlighter = Search::Highlighter.new("nomatch")
|
|
text = "This text does not contain the search term " + "word " * 50
|
|
result = highlighter.snippet(text, max_words: 10)
|
|
|
|
assert_includes result, "..."
|
|
assert_not_includes result, Search::Highlighter::OPENING_MARK
|
|
end
|
|
|
|
test "highlight escapes HTML and preserves marks" do
|
|
highlighter = Search::Highlighter.new("test")
|
|
result = highlighter.highlight("<script>test</script>")
|
|
|
|
assert_equal "<script>#{mark('test')}</script>", result
|
|
end
|
|
|
|
test "highlight Chinese characters" do
|
|
highlighter = Search::Highlighter.new("测试")
|
|
result = highlighter.highlight("这是一个测试文本")
|
|
|
|
assert_equal "这是一个#{mark('测试')}文本", result
|
|
end
|
|
|
|
test "highlight Japanese characters" do
|
|
highlighter = Search::Highlighter.new("テスト")
|
|
result = highlighter.highlight("これはテストです")
|
|
|
|
assert_equal "これは#{mark('テスト')}です", result
|
|
end
|
|
|
|
test "highlight Korean characters" do
|
|
highlighter = Search::Highlighter.new("테스트")
|
|
result = highlighter.highlight("이것은 테스트입니다")
|
|
|
|
assert_equal "이것은 #{mark('테스트')}입니다", result
|
|
end
|
|
|
|
test "highlight mixed CJK and English" do
|
|
highlighter = Search::Highlighter.new("world 世界")
|
|
result = highlighter.highlight("hello world 你好世界")
|
|
|
|
assert_equal "hello #{mark('world')} 你好#{mark('世界')}", result
|
|
end
|
|
|
|
private
|
|
def mark(text)
|
|
"#{Search::Highlighter::OPENING_MARK}#{text}#{Search::Highlighter::CLOSING_MARK}"
|
|
end
|
|
end
|