diff --git a/Gemfile b/Gemfile
index 5df3f6d74..05565e6e6 100644
--- a/Gemfile
+++ b/Gemfile
@@ -33,6 +33,7 @@ gem "platform_agent"
gem "aws-sdk-s3", require: false
gem "web-push"
gem "net-http-persistent"
+gem "mittens"
# Telemetry, logging, and operations
gem "mission_control-jobs"
diff --git a/Gemfile.lock b/Gemfile.lock
index b0066fd78..8aa3d5974 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -301,6 +301,7 @@ GEM
railties (>= 7.1)
stimulus-rails
turbo-rails
+ mittens (0.3.0)
mocha (2.7.1)
ruby2_keywords (>= 0.0.5)
msgpack (1.8.0)
@@ -597,6 +598,7 @@ DEPENDENCIES
letter_opener
lexxy
mission_control-jobs
+ mittens
mocha
net-http-persistent
platform_agent
diff --git a/app/models/card/searchable.rb b/app/models/card/searchable.rb
index b115db1ea..fc01cce20 100644
--- a/app/models/card/searchable.rb
+++ b/app/models/card/searchable.rb
@@ -14,11 +14,11 @@ module Card::Searchable
private
def search_title
- title
+ Search::Stemmer.stem title
end
def search_content
- description.to_plain_text
+ Search::Stemmer.stem description.to_plain_text
end
def search_card_id
diff --git a/app/models/comment/searchable.rb b/app/models/comment/searchable.rb
index 92fdde7f5..4b8846106 100644
--- a/app/models/comment/searchable.rb
+++ b/app/models/comment/searchable.rb
@@ -11,7 +11,7 @@ module Comment::Searchable
end
def search_content
- body.to_plain_text
+ Search::Stemmer.stem body.to_plain_text
end
def search_card_id
diff --git a/app/models/search.rb b/app/models/search.rb
index fd7ca072e..d534016fc 100644
--- a/app/models/search.rb
+++ b/app/models/search.rb
@@ -1,9 +1,6 @@
class Search
attr_reader :user, :query
- HIGHLIGHT_OPENING_MARK = ""
- HIGHLIGHT_CLOSING_MARK = ""
-
def self.table_name_prefix
"search_"
end
@@ -29,6 +26,7 @@ class Search
def perform_search
query_string = query.to_s
sanitized_query = Search::Result.connection.quote(query_string)
+ sanitized_raw_query = Search::Result.connection.quote(query.terms)
Search::Result.from("search_index")
.joins("INNER JOIN cards ON search_index.card_id = cards.id")
@@ -44,7 +42,8 @@ class Search
"boards.name as board_name",
"cards.creator_id",
"search_index.created_at as created_at",
- "MATCH(search_index.content, search_index.title) AGAINST(#{sanitized_query} IN BOOLEAN MODE) AS score"
+ "MATCH(search_index.content, search_index.title) AGAINST(#{sanitized_query} IN BOOLEAN MODE) AS score",
+ "#{sanitized_raw_query} AS query"
].join(","))
.order("search_index.created_at DESC")
end
diff --git a/app/models/search/highlighter.rb b/app/models/search/highlighter.rb
new file mode 100644
index 000000000..044986414
--- /dev/null
+++ b/app/models/search/highlighter.rb
@@ -0,0 +1,69 @@
+class Search::Highlighter
+ HIGHLIGHT_OPENING_MARK = ""
+ HIGHLIGHT_CLOSING_MARK = ""
+
+ attr_reader :query
+
+ def initialize(query)
+ @query = query
+ end
+
+ def highlight(text)
+ result = text.dup
+
+ terms.each do |term|
+ result.gsub!(/(#{Regexp.escape(term)})/i) do |match|
+ "#{HIGHLIGHT_OPENING_MARK}#{match}#{HIGHLIGHT_CLOSING_MARK}"
+ end
+ end
+
+ escape_highlight_marks(result)
+ end
+
+ def snippet(text, max_words: 20)
+ words = text.split(/\s+/)
+ return highlight(text) if words.length <= max_words
+
+ match_index = words.index { |word| terms.any? { |term| word.downcase.include?(term.downcase) } }
+
+ if match_index
+ start_index = [0, match_index - max_words / 2].max
+ end_index = [words.length - 1, start_index + max_words - 1].min
+
+ snippet_words = words[start_index..end_index]
+ snippet_text = snippet_words.join(" ")
+
+ snippet_text = "...#{snippet_text}" if start_index > 0
+ snippet_text = "#{snippet_text}..." if end_index < words.length - 1
+
+ highlight(snippet_text)
+ else
+ text.truncate_words(max_words, omission: "...")
+ end
+ end
+
+ private
+ def terms
+ @terms ||= begin
+ terms = []
+
+ query.scan(/"([^"]+)"/) do |phrase|
+ terms << phrase.first
+ end
+
+ unquoted = query.gsub(/"[^"]+"/, '')
+ unquoted.split(/\s+/).each do |word|
+ terms << word if word.present?
+ end
+
+ terms.uniq
+ end
+ end
+
+ def escape_highlight_marks(html)
+ CGI.escapeHTML(html)
+ .gsub(CGI.escapeHTML(HIGHLIGHT_OPENING_MARK), HIGHLIGHT_OPENING_MARK.html_safe)
+ .gsub(CGI.escapeHTML(HIGHLIGHT_CLOSING_MARK), HIGHLIGHT_CLOSING_MARK.html_safe)
+ .html_safe
+ end
+end
diff --git a/app/models/search/query.rb b/app/models/search/query.rb
index 400e0a48d..0186cc424 100644
--- a/app/models/search/query.rb
+++ b/app/models/search/query.rb
@@ -12,7 +12,9 @@ class Search::Query < ApplicationRecord
end
end
- alias_attribute :to_s, :terms
+ def to_s
+ Search::Stemmer.stem(terms.to_s)
+ end
private
def sanitize_terms
diff --git a/app/models/search/result.rb b/app/models/search/result.rb
index 28e1f7172..d90411bd5 100644
--- a/app/models/search/result.rb
+++ b/app/models/search/result.rb
@@ -4,15 +4,15 @@ class Search::Result < ApplicationRecord
belongs_to :comment, foreign_key: :comment_id, optional: true
def card_title
- escape_highlight card_title_in_database
+ highlight(card_title_in_database, show: :full)
end
def card_description
- escape_highlight card_description_in_database
+ highlight(card_description_in_database, show: :snippet)
end
def comment_body
- escape_highlight comment_body_in_database
+ highlight(comment_body_in_database, show: :snippet)
end
def source
@@ -24,14 +24,12 @@ class Search::Result < ApplicationRecord
end
private
- def escape_highlight(html)
- if html
- CGI.escapeHTML(html)
- .gsub(CGI.escapeHTML(Search::HIGHLIGHT_OPENING_MARK), Search::HIGHLIGHT_OPENING_MARK.html_safe)
- .gsub(CGI.escapeHTML(Search::HIGHLIGHT_CLOSING_MARK), Search::HIGHLIGHT_CLOSING_MARK.html_safe)
- .html_safe
+ def highlight(text, show:)
+ if text.present? && attribute?(:query)
+ highlighter = Search::Highlighter.new(query)
+ show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text)
else
- nil
+ text
end
end
end
diff --git a/app/models/search/stemmer.rb b/app/models/search/stemmer.rb
new file mode 100644
index 000000000..a92143965
--- /dev/null
+++ b/app/models/search/stemmer.rb
@@ -0,0 +1,13 @@
+module Search::Stemmer
+ extend self
+
+ STEMMER = Mittens::Stemmer.new
+
+ def stem(value)
+ if value.present?
+ value.split(/\s+/).map { |word| STEMMER.stem(word.downcase) }.join(" ")
+ else
+ value
+ end
+ end
+end