LU
366588e263
- Split each CJK character into individual tokens for FTS5 indexing - Add stem_content callback to SQLite adapter to pre-process content - Stem search queries before matching against FTS5 index - Update stemmer tests to reflect character-splitting behavior This enables CJK search on SQLite installations where the FTS5 tokenizer cannot natively segment CJK text. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
48 lines
938 B
Ruby
48 lines
938 B
Ruby
module Search::Stemmer
|
|
extend self
|
|
|
|
STEMMER = Mittens::Stemmer.new
|
|
|
|
def stem(value)
|
|
if value.present?
|
|
tokenize(value).join(" ")
|
|
else
|
|
value
|
|
end
|
|
end
|
|
|
|
private
|
|
def tokenize(value)
|
|
tokens = []
|
|
current_word = +""
|
|
|
|
value.each_char do |char|
|
|
if cjk_character?(char)
|
|
if current_word.present?
|
|
tokens << stem_word(current_word)
|
|
current_word = +""
|
|
end
|
|
tokens << char
|
|
elsif char =~ /[\p{L}\p{N}_]/
|
|
current_word << char
|
|
else
|
|
if current_word.present?
|
|
tokens << stem_word(current_word)
|
|
current_word = +""
|
|
end
|
|
end
|
|
end
|
|
|
|
tokens << stem_word(current_word) if current_word.present?
|
|
tokens
|
|
end
|
|
|
|
def cjk_character?(char)
|
|
char.match?(Search::CJK_PATTERN)
|
|
end
|
|
|
|
def stem_word(word)
|
|
STEMMER.stem(word.downcase)
|
|
end
|
|
end
|