From 2a4fbecbdacbdd59ba6f0c6added91971991cee7 Mon Sep 17 00:00:00 2001 From: LU Date: Tue, 6 Jan 2026 17:22:42 +0900 Subject: [PATCH] Add CJK (Chinese, Japanese, Korean) search support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The search functionality was silently dropping CJK characters because: 1. Query sanitization used `\w` which only matches ASCII word characters 2. Stemmer split by whitespace, which doesn't work for CJK languages 3. Highlighter used `\b` word boundaries that don't apply to CJK This commit fixes all three issues: - Query: Use `\p{L}\p{N}` (Unicode letters/numbers) instead of `\w` - Stemmer: Preserve CJK characters as-is without stemming, since CJK languages don't have stemming rules like English - Highlighter: Skip word boundary matching for CJK terms Also extracts `CJK_PATTERN` to `Search` module to avoid duplication. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- app/models/search.rb | 2 + app/models/search/highlighter.rb | 10 ++++- app/models/search/query.rb | 2 +- app/models/search/stemmer.rb | 46 ++++++++++++++++++++- test/models/search/highlighter_test.rb | 28 +++++++++++++ test/models/search/query_test.rb | 57 ++++++++++++++++++++++++++ test/models/search/stemmer_test.rb | 36 ++++++++++++++++ 7 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 test/models/search/query_test.rb diff --git a/app/models/search.rb b/app/models/search.rb index f81b9a9d8..3b8127ef1 100644 --- a/app/models/search.rb +++ b/app/models/search.rb @@ -1,4 +1,6 @@ module Search + CJK_PATTERN = /\p{Han}|\p{Hiragana}|\p{Katakana}|\p{Hangul}/ + def self.table_name_prefix "search_" end diff --git a/app/models/search/highlighter.rb b/app/models/search/highlighter.rb index d2b53d772..8672ed3c8 100644 --- a/app/models/search/highlighter.rb +++ b/app/models/search/highlighter.rb @@ -13,8 +13,14 @@ class Search::Highlighter result = text.dup terms.each do |term| - result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match| - "#{OPENING_MARK}#{match}#{CLOSING_MARK}" + if term.match?(Search::CJK_PATTERN) + result.gsub!(/(#{Regexp.escape(term)})/i) do |match| + "#{OPENING_MARK}#{match}#{CLOSING_MARK}" + end + else + result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match| + "#{OPENING_MARK}#{match}#{CLOSING_MARK}" + end end end diff --git a/app/models/search/query.rb b/app/models/search/query.rb index 5fe0829e4..6bf06ef42 100644 --- a/app/models/search/query.rb +++ b/app/models/search/query.rb @@ -33,7 +33,7 @@ class Search::Query < ApplicationRecord end def remove_invalid_search_characters(terms) - terms.gsub(/[^\w"]/, " ") + terms.gsub(/[^\p{L}\p{N}_"]/, " ") end def remove_unbalanced_quotes(terms) diff --git a/app/models/search/stemmer.rb b/app/models/search/stemmer.rb index f6a6c56d4..e19413a8c 100644 --- a/app/models/search/stemmer.rb +++ b/app/models/search/stemmer.rb @@ -5,9 +5,53 @@ module Search::Stemmer def stem(value) if value.present? - value.gsub(/[^\w\s]/, "").split(/\s+/).map { |word| STEMMER.stem(word.downcase) }.join(" ") + tokenize(value).join(" ") else value end end + + private + def tokenize(value) + tokens = [] + current_word = +"" + current_cjk = +"" + + value.each_char do |char| + if cjk_character?(char) + if current_word.present? + tokens << stem_word(current_word) + current_word = +"" + end + current_cjk << char + elsif char =~ /[\p{L}\p{N}_]/ + if current_cjk.present? + tokens << current_cjk + current_cjk = +"" + end + current_word << char + else + if current_word.present? + tokens << stem_word(current_word) + current_word = +"" + end + if current_cjk.present? + tokens << current_cjk + current_cjk = +"" + end + end + end + + tokens << stem_word(current_word) if current_word.present? + tokens << current_cjk if current_cjk.present? + tokens + end + + def cjk_character?(char) + char.match?(Search::CJK_PATTERN) + end + + def stem_word(word) + STEMMER.stem(word.downcase) + end end diff --git a/test/models/search/highlighter_test.rb b/test/models/search/highlighter_test.rb index a065c4da1..822e4913d 100644 --- a/test/models/search/highlighter_test.rb +++ b/test/models/search/highlighter_test.rb @@ -82,6 +82,34 @@ class Search::HighlighterTest < ActiveSupport::TestCase assert_equal "<script>#{mark('test')}</script>", result end + test "highlight Chinese characters" do + highlighter = Search::Highlighter.new("测试") + result = highlighter.highlight("这是一个测试文本") + + assert_equal "这是一个#{mark('测试')}文本", result + end + + test "highlight Japanese characters" do + highlighter = Search::Highlighter.new("テスト") + result = highlighter.highlight("これはテストです") + + assert_equal "これは#{mark('テスト')}です", result + end + + test "highlight Korean characters" do + highlighter = Search::Highlighter.new("테스트") + result = highlighter.highlight("이것은 테스트입니다") + + assert_equal "이것은 #{mark('테스트')}입니다", result + end + + test "highlight mixed CJK and English" do + highlighter = Search::Highlighter.new("world 世界") + result = highlighter.highlight("hello world 你好世界") + + assert_equal "hello #{mark('world')} 你好#{mark('世界')}", result + end + private def mark(text) "#{Search::Highlighter::OPENING_MARK}#{text}#{Search::Highlighter::CLOSING_MARK}" diff --git a/test/models/search/query_test.rb b/test/models/search/query_test.rb new file mode 100644 index 000000000..628b9e69e --- /dev/null +++ b/test/models/search/query_test.rb @@ -0,0 +1,57 @@ +require "test_helper" + +class Search::QueryTest < ActiveSupport::TestCase + setup do + @account = accounts(:"37s") + Current.account = @account + end + + test "sanitize preserves ASCII words" do + query = build_query("hello world") + + assert_equal "hello world", query.terms + end + + test "sanitize preserves Chinese characters" do + query = build_query("测试文本") + + assert_equal "测试文本", query.terms + end + + test "sanitize preserves Japanese characters" do + query = build_query("テスト") + + assert_equal "テスト", query.terms + end + + test "sanitize preserves Korean characters" do + query = build_query("테스트") + + assert_equal "테스트", query.terms + end + + test "sanitize preserves mixed CJK and English" do + query = build_query("hello 世界 test") + + assert_equal "hello 世界 test", query.terms + end + + test "sanitize removes special characters but preserves CJK" do + query = build_query("测试@文本") + + assert_equal "测试 文本", query.terms + end + + test "sanitize preserves quoted phrases with CJK" do + query = build_query('"你好世界"') + + assert_equal '"你好世界"', query.terms + end + + private + def build_query(terms) + query = Search::Query.wrap(terms) + query.validate + query + end +end diff --git a/test/models/search/stemmer_test.rb b/test/models/search/stemmer_test.rb index 858ed2dca..3974b73fc 100644 --- a/test/models/search/stemmer_test.rb +++ b/test/models/search/stemmer_test.rb @@ -12,4 +12,40 @@ class Search::StemmerTest < ActiveSupport::TestCase assert_equal "test run jump walk", result end + + test "preserve Chinese characters" do + result = Search::Stemmer.stem("测试") + + assert_equal "测试", result + end + + test "preserve Japanese characters" do + result = Search::Stemmer.stem("テスト") + + assert_equal "テスト", result + end + + test "preserve Korean characters" do + result = Search::Stemmer.stem("테스트") + + assert_equal "테스트", result + end + + test "mixed CJK and English" do + result = Search::Stemmer.stem("running 测试 test") + + assert_equal "run 测试 test", result + end + + test "mixed CJK and English without spaces" do + result = Search::Stemmer.stem("hello世界test") + + assert_equal "hello 世界 test", result + end + + test "CJK punctuation is treated as separator" do + result = Search::Stemmer.stem("你好。世界") + + assert_equal "你好 世界", result + end end