LU
2a4fbecbda
The search functionality was silently dropping CJK characters because:
1. Query sanitization used `\w` which only matches ASCII word characters
2. Stemmer split by whitespace, which doesn't work for CJK languages
3. Highlighter used `\b` word boundaries that don't apply to CJK
This commit fixes all three issues:
- Query: Use `\p{L}\p{N}` (Unicode letters/numbers) instead of `\w`
- Stemmer: Preserve CJK characters as-is without stemming, since CJK
languages don't have stemming rules like English
- Highlighter: Skip word boundary matching for CJK terms
Also extracts `CJK_PATTERN` to `Search` module to avoid duplication.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
75 lines
1.9 KiB
Ruby
75 lines
1.9 KiB
Ruby
class Search::Highlighter
|
|
OPENING_MARK = "<mark class=\"circled-text\"><span></span>"
|
|
CLOSING_MARK = "</mark>"
|
|
ELIPSIS = "..."
|
|
|
|
attr_reader :query
|
|
|
|
def initialize(query)
|
|
@query = query
|
|
end
|
|
|
|
def highlight(text)
|
|
result = text.dup
|
|
|
|
terms.each do |term|
|
|
if term.match?(Search::CJK_PATTERN)
|
|
result.gsub!(/(#{Regexp.escape(term)})/i) do |match|
|
|
"#{OPENING_MARK}#{match}#{CLOSING_MARK}"
|
|
end
|
|
else
|
|
result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match|
|
|
"#{OPENING_MARK}#{match}#{CLOSING_MARK}"
|
|
end
|
|
end
|
|
end
|
|
|
|
escape_highlight_marks(result)
|
|
end
|
|
|
|
def snippet(text, max_words: 20)
|
|
words = text.split(/\s+/)
|
|
match_index = words.index { |word| terms.any? { |term| word.downcase.include?(term.downcase) } }
|
|
|
|
if words.length <= max_words
|
|
highlight(text)
|
|
elsif match_index
|
|
start_index = [ 0, match_index - max_words / 2 ].max
|
|
end_index = [ words.length - 1, start_index + max_words - 1 ].min
|
|
|
|
snippet_text = words[start_index..end_index].join(" ")
|
|
snippet_text = "...#{snippet_text}" if start_index > 0
|
|
snippet_text = "#{snippet_text}..." if end_index < words.length - 1
|
|
|
|
highlight(snippet_text)
|
|
else
|
|
text.truncate_words(max_words, omission: "...")
|
|
end
|
|
end
|
|
|
|
private
|
|
def terms
|
|
@terms ||= begin
|
|
terms = []
|
|
|
|
query.scan(/"([^"]+)"/) do |phrase|
|
|
terms << phrase.first
|
|
end
|
|
|
|
unquoted = query.gsub(/"[^"]+"/, "")
|
|
unquoted.split(/\s+/).each do |word|
|
|
terms << word if word.present?
|
|
end
|
|
|
|
terms.uniq
|
|
end
|
|
end
|
|
|
|
def escape_highlight_marks(html)
|
|
CGI.escapeHTML(html)
|
|
.gsub(CGI.escapeHTML(OPENING_MARK), OPENING_MARK.html_safe)
|
|
.gsub(CGI.escapeHTML(CLOSING_MARK), CLOSING_MARK.html_safe)
|
|
.html_safe
|
|
end
|
|
end
|