Add a stemmer and highlighter

Gemfile

@@ -33,6 +33,7 @@ gem "platform_agent"
 gem "aws-sdk-s3", require: false
 gem "web-push"
 gem "net-http-persistent"
+gem "mittens"
 
 # Telemetry, logging, and operations
 gem "mission_control-jobs"

Gemfile.lock

@@ -301,6 +301,7 @@ GEM
       railties (>= 7.1)
       stimulus-rails
       turbo-rails
+    mittens (0.3.0)
     mocha (2.7.1)
       ruby2_keywords (>= 0.0.5)
     msgpack (1.8.0)
@@ -597,6 +598,7 @@ DEPENDENCIES
   letter_opener
   lexxy
   mission_control-jobs
+  mittens
   mocha
   net-http-persistent
   platform_agent

app/models/card/searchable.rb

@@ -14,11 +14,11 @@ module Card::Searchable
 
   private
     def search_title
-      title
+      Search::Stemmer.stem title
     end
 
     def search_content
-      description.to_plain_text
+      Search::Stemmer.stem description.to_plain_text
     end
 
     def search_card_id

app/models/comment/searchable.rb

@@ -11,7 +11,7 @@ module Comment::Searchable
     end
 
     def search_content
-      body.to_plain_text
+      Search::Stemmer.stem body.to_plain_text
     end
 
     def search_card_id

app/models/search.rb

@@ -1,9 +1,6 @@
 class Search
   attr_reader :user, :query
 
-  HIGHLIGHT_OPENING_MARK = "<mark class=\"circled-text\"><span></span>"
-  HIGHLIGHT_CLOSING_MARK = "</mark>"
-
   def self.table_name_prefix
     "search_"
   end
@@ -29,6 +26,7 @@ class Search
     def perform_search
       query_string = query.to_s
       sanitized_query = Search::Result.connection.quote(query_string)
+      sanitized_raw_query = Search::Result.connection.quote(query.terms)
 
       Search::Result.from("search_index")
         .joins("INNER JOIN cards ON search_index.card_id = cards.id")
@@ -44,7 +42,8 @@ class Search
           "boards.name as board_name",
           "cards.creator_id",
           "search_index.created_at as created_at",
-          "MATCH(search_index.content, search_index.title) AGAINST(#{sanitized_query} IN BOOLEAN MODE) AS score"
+          "MATCH(search_index.content, search_index.title) AGAINST(#{sanitized_query} IN BOOLEAN MODE) AS score",
+          "#{sanitized_raw_query} AS query"
         ].join(","))
         .order("search_index.created_at DESC")
     end

app/models/search/highlighter.rb

@@ -0,0 +1,69 @@
+class Search::Highlighter
+  HIGHLIGHT_OPENING_MARK = "<mark class=\"circled-text\"><span></span>"
+  HIGHLIGHT_CLOSING_MARK = "</mark>"
+
+  attr_reader :query
+
+  def initialize(query)
+    @query = query
+  end
+
+  def highlight(text)
+    result = text.dup
+
+    terms.each do |term|
+      result.gsub!(/(#{Regexp.escape(term)})/i) do |match|
+        "#{HIGHLIGHT_OPENING_MARK}#{match}#{HIGHLIGHT_CLOSING_MARK}"
+      end
+    end
+
+    escape_highlight_marks(result)
+  end
+
+  def snippet(text, max_words: 20)
+    words = text.split(/\s+/)
+    return highlight(text) if words.length <= max_words
+
+    match_index = words.index { |word| terms.any? { |term| word.downcase.include?(term.downcase) } }
+
+    if match_index
+      start_index = [0, match_index - max_words / 2].max
+      end_index = [words.length - 1, start_index + max_words - 1].min
+
+      snippet_words = words[start_index..end_index]
+      snippet_text = snippet_words.join(" ")
+
+      snippet_text = "...#{snippet_text}" if start_index > 0
+      snippet_text = "#{snippet_text}..." if end_index < words.length - 1
+
+      highlight(snippet_text)
+    else
+      text.truncate_words(max_words, omission: "...")
+    end
+  end
+
+  private
+    def terms
+      @terms ||= begin
+        terms = []
+
+        query.scan(/"([^"]+)"/) do |phrase|
+          terms << phrase.first
+        end
+
+        unquoted = query.gsub(/"[^"]+"/, '')
+        unquoted.split(/\s+/).each do |word|
+          terms << word if word.present?
+        end
+
+        terms.uniq
+      end
+    end
+
+    def escape_highlight_marks(html)
+      CGI.escapeHTML(html)
+        .gsub(CGI.escapeHTML(HIGHLIGHT_OPENING_MARK), HIGHLIGHT_OPENING_MARK.html_safe)
+        .gsub(CGI.escapeHTML(HIGHLIGHT_CLOSING_MARK), HIGHLIGHT_CLOSING_MARK.html_safe)
+        .html_safe
+    end
+end

app/models/search/query.rb

@@ -12,7 +12,9 @@ class Search::Query < ApplicationRecord
     end
   end
 
-  alias_attribute :to_s, :terms
+  def to_s
+    Search::Stemmer.stem(terms.to_s)
+  end
 
   private
     def sanitize_terms

app/models/search/result.rb

@@ -4,15 +4,15 @@ class Search::Result < ApplicationRecord
   belongs_to :comment, foreign_key: :comment_id, optional: true
 
   def card_title
-    escape_highlight card_title_in_database
+    highlight(card_title_in_database, show: :full)
   end
 
   def card_description
-    escape_highlight card_description_in_database
+    highlight(card_description_in_database, show: :snippet)
   end
 
   def comment_body
-    escape_highlight comment_body_in_database
+    highlight(comment_body_in_database, show: :snippet)
   end
 
   def source
@@ -24,14 +24,12 @@ class Search::Result < ApplicationRecord
   end
 
   private
-    def escape_highlight(html)
-      if html
-        CGI.escapeHTML(html)
-          .gsub(CGI.escapeHTML(Search::HIGHLIGHT_OPENING_MARK), Search::HIGHLIGHT_OPENING_MARK.html_safe)
-          .gsub(CGI.escapeHTML(Search::HIGHLIGHT_CLOSING_MARK), Search::HIGHLIGHT_CLOSING_MARK.html_safe)
-          .html_safe
+    def highlight(text, show:)
+      if text.present? && attribute?(:query)
+        highlighter = Search::Highlighter.new(query)
+        show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text)
       else
-        nil
+        text
       end
     end
 end

app/models/search/stemmer.rb

@@ -0,0 +1,13 @@
+module Search::Stemmer
+  extend self
+
+  STEMMER = Mittens::Stemmer.new
+
+  def stem(value)
+    if value.present?
+      value.split(/\s+/).map { |word| STEMMER.stem(word.downcase) }.join(" ")
+    else
+      value
+    end
+  end
+end