Files
fizzy/test/models/search/highlighter_test.rb
T
LU 2a4fbecbda Add CJK (Chinese, Japanese, Korean) search support
The search functionality was silently dropping CJK characters because:

1. Query sanitization used `\w` which only matches ASCII word characters
2. Stemmer split by whitespace, which doesn't work for CJK languages
3. Highlighter used `\b` word boundaries that don't apply to CJK

This commit fixes all three issues:

- Query: Use `\p{L}\p{N}` (Unicode letters/numbers) instead of `\w`
- Stemmer: Preserve CJK characters as-is without stemming, since CJK
  languages don't have stemming rules like English
- Highlighter: Skip word boundary matching for CJK terms

Also extracts `CJK_PATTERN` to `Search` module to avoid duplication.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-06 17:22:42 +09:00

118 lines
3.8 KiB
Ruby

require "test_helper"
class Search::HighlighterTest < ActiveSupport::TestCase
test "highlight simple word match" do
highlighter = Search::Highlighter.new("hello")
result = highlighter.highlight("Hello world")
assert_equal "#{mark('Hello')} world", result
end
test "highlight multiple occurrences" do
highlighter = Search::Highlighter.new("test")
result = highlighter.highlight("This is a test and another test")
assert_equal "This is a #{mark('test')} and another #{mark('test')}", result
end
test "highlight case insensitive" do
highlighter = Search::Highlighter.new("ruby")
result = highlighter.highlight("Ruby is great and RUBY rocks")
assert_equal "#{mark('Ruby')} is great and #{mark('RUBY')} rocks", result
end
test "highlight quoted phrases" do
highlighter = Search::Highlighter.new('"hello world"')
result = highlighter.highlight("Say hello world to everyone")
assert_equal "Say #{mark('hello world')} to everyone", result
end
test "snippet returns full text with highlights when under max words" do
highlighter = Search::Highlighter.new("ruby")
result = highlighter.snippet("Ruby is great", max_words: 20)
assert_equal "#{mark('Ruby')} is great", result
end
test "snippet creates excerpt around match" do
highlighter = Search::Highlighter.new("match")
text = "word " * 10 + "match " + "word " * 10
result = highlighter.snippet(text, max_words: 10)
assert result.start_with?("...")
assert result.end_with?("...")
assert_includes result, mark("match")
end
test "snippet adds leading ellipsis when match is not at start" do
highlighter = Search::Highlighter.new("middle")
text = "word " * 20 + "middle"
result = highlighter.snippet(text, max_words: 10)
assert result.start_with?("...")
assert_not result.end_with?("...")
assert_includes result, mark("middle")
end
test "snippet adds trailing ellipsis when text continues after excerpt" do
highlighter = Search::Highlighter.new("start")
text = "start " + "word " * 30
result = highlighter.snippet(text, max_words: 10)
assert result.end_with?("...")
assert_not result.start_with?("...")
assert_includes result, mark("start")
end
test "snippet falls back to truncation when no match found" do
highlighter = Search::Highlighter.new("nomatch")
text = "This text does not contain the search term " + "word " * 50
result = highlighter.snippet(text, max_words: 10)
assert_includes result, "..."
assert_not_includes result, Search::Highlighter::OPENING_MARK
end
test "highlight escapes HTML and preserves marks" do
highlighter = Search::Highlighter.new("test")
result = highlighter.highlight("<script>test</script>")
assert_equal "&lt;script&gt;#{mark('test')}&lt;/script&gt;", result
end
test "highlight Chinese characters" do
highlighter = Search::Highlighter.new("测试")
result = highlighter.highlight("这是一个测试文本")
assert_equal "这是一个#{mark('测试')}文本", result
end
test "highlight Japanese characters" do
highlighter = Search::Highlighter.new("テスト")
result = highlighter.highlight("これはテストです")
assert_equal "これは#{mark('テスト')}です", result
end
test "highlight Korean characters" do
highlighter = Search::Highlighter.new("테스트")
result = highlighter.highlight("이것은 테스트입니다")
assert_equal "이것은 #{mark('테스트')}입니다", result
end
test "highlight mixed CJK and English" do
highlighter = Search::Highlighter.new("world 世界")
result = highlighter.highlight("hello world 你好世界")
assert_equal "hello #{mark('world')} 你好#{mark('世界')}", result
end
private
def mark(text)
"#{Search::Highlighter::OPENING_MARK}#{text}#{Search::Highlighter::CLOSING_MARK}"
end
end