Add CJK (Chinese, Japanese, Korean) search support

The search functionality was silently dropping CJK characters because:

1. Query sanitization used `\w` which only matches ASCII word characters
2. Stemmer split by whitespace, which doesn't work for CJK languages
3. Highlighter used `\b` word boundaries that don't apply to CJK

This commit fixes all three issues:

- Query: Use `\p{L}\p{N}` (Unicode letters/numbers) instead of `\w`
- Stemmer: Preserve CJK characters as-is without stemming, since CJK
  languages don't have stemming rules like English
- Highlighter: Skip word boundary matching for CJK terms

Also extracts `CJK_PATTERN` to `Search` module to avoid duplication.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

app/models/search.rb

@@ -1,4 +1,6 @@
 module Search
+  CJK_PATTERN = /\p{Han}|\p{Hiragana}|\p{Katakana}|\p{Hangul}/
+
   def self.table_name_prefix
     "search_"
   end

app/models/search/highlighter.rb

@@ -13,8 +13,14 @@ class Search::Highlighter
     result = text.dup
 
     terms.each do |term|
-      result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match|
-        "#{OPENING_MARK}#{match}#{CLOSING_MARK}"
+      if term.match?(Search::CJK_PATTERN)
+        result.gsub!(/(#{Regexp.escape(term)})/i) do |match|
+          "#{OPENING_MARK}#{match}#{CLOSING_MARK}"
+        end
+      else
+        result.gsub!(/\b(#{Regexp.escape(term)}\w*)\b/i) do |match|
+          "#{OPENING_MARK}#{match}#{CLOSING_MARK}"
+        end
       end
     end
 

app/models/search/query.rb

@@ -33,7 +33,7 @@ class Search::Query < ApplicationRecord
     end
 
     def remove_invalid_search_characters(terms)
-      terms.gsub(/[^\w"]/, " ")
+      terms.gsub(/[^\p{L}\p{N}_"]/, " ")
     end
 
     def remove_unbalanced_quotes(terms)

app/models/search/stemmer.rb

@@ -5,9 +5,53 @@ module Search::Stemmer
 
   def stem(value)
     if value.present?
-      value.gsub(/[^\w\s]/, "").split(/\s+/).map { |word| STEMMER.stem(word.downcase) }.join(" ")
+      tokenize(value).join(" ")
     else
       value
     end
   end
+
+  private
+    def tokenize(value)
+      tokens = []
+      current_word = +""
+      current_cjk = +""
+
+      value.each_char do |char|
+        if cjk_character?(char)
+          if current_word.present?
+            tokens << stem_word(current_word)
+            current_word = +""
+          end
+          current_cjk << char
+        elsif char =~ /[\p{L}\p{N}_]/
+          if current_cjk.present?
+            tokens << current_cjk
+            current_cjk = +""
+          end
+          current_word << char
+        else
+          if current_word.present?
+            tokens << stem_word(current_word)
+            current_word = +""
+          end
+          if current_cjk.present?
+            tokens << current_cjk
+            current_cjk = +""
+          end
+        end
+      end
+
+      tokens << stem_word(current_word) if current_word.present?
+      tokens << current_cjk if current_cjk.present?
+      tokens
+    end
+
+    def cjk_character?(char)
+      char.match?(Search::CJK_PATTERN)
+    end
+
+    def stem_word(word)
+      STEMMER.stem(word.downcase)
+    end
 end

test/models/search/highlighter_test.rb

@@ -82,6 +82,34 @@ class Search::HighlighterTest < ActiveSupport::TestCase
     assert_equal "&lt;script&gt;#{mark('test')}&lt;/script&gt;", result
   end
 
+  test "highlight Chinese characters" do
+    highlighter = Search::Highlighter.new("测试")
+    result = highlighter.highlight("这是一个测试文本")
+
+    assert_equal "这是一个#{mark('测试')}文本", result
+  end
+
+  test "highlight Japanese characters" do
+    highlighter = Search::Highlighter.new("テスト")
+    result = highlighter.highlight("これはテストです")
+
+    assert_equal "これは#{mark('テスト')}です", result
+  end
+
+  test "highlight Korean characters" do
+    highlighter = Search::Highlighter.new("테스트")
+    result = highlighter.highlight("이것은 테스트입니다")
+
+    assert_equal "이것은 #{mark('테스트')}입니다", result
+  end
+
+  test "highlight mixed CJK and English" do
+    highlighter = Search::Highlighter.new("world 世界")
+    result = highlighter.highlight("hello world 你好世界")
+
+    assert_equal "hello #{mark('world')} 你好#{mark('世界')}", result
+  end
+
   private
     def mark(text)
       "#{Search::Highlighter::OPENING_MARK}#{text}#{Search::Highlighter::CLOSING_MARK}"

test/models/search/query_test.rb

@@ -0,0 +1,57 @@
+require "test_helper"
+
+class Search::QueryTest < ActiveSupport::TestCase
+  setup do
+    @account = accounts(:"37s")
+    Current.account = @account
+  end
+
+  test "sanitize preserves ASCII words" do
+    query = build_query("hello world")
+
+    assert_equal "hello world", query.terms
+  end
+
+  test "sanitize preserves Chinese characters" do
+    query = build_query("测试文本")
+
+    assert_equal "测试文本", query.terms
+  end
+
+  test "sanitize preserves Japanese characters" do
+    query = build_query("テスト")
+
+    assert_equal "テスト", query.terms
+  end
+
+  test "sanitize preserves Korean characters" do
+    query = build_query("테스트")
+
+    assert_equal "테스트", query.terms
+  end
+
+  test "sanitize preserves mixed CJK and English" do
+    query = build_query("hello 世界 test")
+
+    assert_equal "hello 世界 test", query.terms
+  end
+
+  test "sanitize removes special characters but preserves CJK" do
+    query = build_query("测试@文本")
+
+    assert_equal "测试 文本", query.terms
+  end
+
+  test "sanitize preserves quoted phrases with CJK" do
+    query = build_query('"你好世界"')
+
+    assert_equal '"你好世界"', query.terms
+  end
+
+  private
+    def build_query(terms)
+      query = Search::Query.wrap(terms)
+      query.validate
+      query
+    end
+end

test/models/search/stemmer_test.rb

@@ -12,4 +12,40 @@ class Search::StemmerTest < ActiveSupport::TestCase
 
     assert_equal "test run jump walk", result
   end
+
+  test "preserve Chinese characters" do
+    result = Search::Stemmer.stem("测试")
+
+    assert_equal "测试", result
+  end
+
+  test "preserve Japanese characters" do
+    result = Search::Stemmer.stem("テスト")
+
+    assert_equal "テスト", result
+  end
+
+  test "preserve Korean characters" do
+    result = Search::Stemmer.stem("테스트")
+
+    assert_equal "테스트", result
+  end
+
+  test "mixed CJK and English" do
+    result = Search::Stemmer.stem("running 测试 test")
+
+    assert_equal "run 测试 test", result
+  end
+
+  test "mixed CJK and English without spaces" do
+    result = Search::Stemmer.stem("hello世界test")
+
+    assert_equal "hello 世界 test", result
+  end
+
+  test "CJK punctuation is treated as separator" do
+    result = Search::Stemmer.stem("你好。世界")
+
+    assert_equal "你好 世界", result
+  end
 end