diff --git a/Gemfile b/Gemfile index 5df3f6d74..05565e6e6 100644 --- a/Gemfile +++ b/Gemfile @@ -33,6 +33,7 @@ gem "platform_agent" gem "aws-sdk-s3", require: false gem "web-push" gem "net-http-persistent" +gem "mittens" # Telemetry, logging, and operations gem "mission_control-jobs" diff --git a/Gemfile.lock b/Gemfile.lock index b0066fd78..8aa3d5974 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -301,6 +301,7 @@ GEM railties (>= 7.1) stimulus-rails turbo-rails + mittens (0.3.0) mocha (2.7.1) ruby2_keywords (>= 0.0.5) msgpack (1.8.0) @@ -597,6 +598,7 @@ DEPENDENCIES letter_opener lexxy mission_control-jobs + mittens mocha net-http-persistent platform_agent diff --git a/app/models/card/searchable.rb b/app/models/card/searchable.rb index b115db1ea..fc01cce20 100644 --- a/app/models/card/searchable.rb +++ b/app/models/card/searchable.rb @@ -14,11 +14,11 @@ module Card::Searchable private def search_title - title + Search::Stemmer.stem title end def search_content - description.to_plain_text + Search::Stemmer.stem description.to_plain_text end def search_card_id diff --git a/app/models/comment/searchable.rb b/app/models/comment/searchable.rb index 92fdde7f5..4b8846106 100644 --- a/app/models/comment/searchable.rb +++ b/app/models/comment/searchable.rb @@ -11,7 +11,7 @@ module Comment::Searchable end def search_content - body.to_plain_text + Search::Stemmer.stem body.to_plain_text end def search_card_id diff --git a/app/models/search.rb b/app/models/search.rb index fd7ca072e..d534016fc 100644 --- a/app/models/search.rb +++ b/app/models/search.rb @@ -1,9 +1,6 @@ class Search attr_reader :user, :query - HIGHLIGHT_OPENING_MARK = "" - HIGHLIGHT_CLOSING_MARK = "" - def self.table_name_prefix "search_" end @@ -29,6 +26,7 @@ class Search def perform_search query_string = query.to_s sanitized_query = Search::Result.connection.quote(query_string) + sanitized_raw_query = Search::Result.connection.quote(query.terms) Search::Result.from("search_index") .joins("INNER JOIN cards ON search_index.card_id = cards.id") @@ -44,7 +42,8 @@ class Search "boards.name as board_name", "cards.creator_id", "search_index.created_at as created_at", - "MATCH(search_index.content, search_index.title) AGAINST(#{sanitized_query} IN BOOLEAN MODE) AS score" + "MATCH(search_index.content, search_index.title) AGAINST(#{sanitized_query} IN BOOLEAN MODE) AS score", + "#{sanitized_raw_query} AS query" ].join(",")) .order("search_index.created_at DESC") end diff --git a/app/models/search/highlighter.rb b/app/models/search/highlighter.rb new file mode 100644 index 000000000..044986414 --- /dev/null +++ b/app/models/search/highlighter.rb @@ -0,0 +1,69 @@ +class Search::Highlighter + HIGHLIGHT_OPENING_MARK = "" + HIGHLIGHT_CLOSING_MARK = "" + + attr_reader :query + + def initialize(query) + @query = query + end + + def highlight(text) + result = text.dup + + terms.each do |term| + result.gsub!(/(#{Regexp.escape(term)})/i) do |match| + "#{HIGHLIGHT_OPENING_MARK}#{match}#{HIGHLIGHT_CLOSING_MARK}" + end + end + + escape_highlight_marks(result) + end + + def snippet(text, max_words: 20) + words = text.split(/\s+/) + return highlight(text) if words.length <= max_words + + match_index = words.index { |word| terms.any? { |term| word.downcase.include?(term.downcase) } } + + if match_index + start_index = [0, match_index - max_words / 2].max + end_index = [words.length - 1, start_index + max_words - 1].min + + snippet_words = words[start_index..end_index] + snippet_text = snippet_words.join(" ") + + snippet_text = "...#{snippet_text}" if start_index > 0 + snippet_text = "#{snippet_text}..." if end_index < words.length - 1 + + highlight(snippet_text) + else + text.truncate_words(max_words, omission: "...") + end + end + + private + def terms + @terms ||= begin + terms = [] + + query.scan(/"([^"]+)"/) do |phrase| + terms << phrase.first + end + + unquoted = query.gsub(/"[^"]+"/, '') + unquoted.split(/\s+/).each do |word| + terms << word if word.present? + end + + terms.uniq + end + end + + def escape_highlight_marks(html) + CGI.escapeHTML(html) + .gsub(CGI.escapeHTML(HIGHLIGHT_OPENING_MARK), HIGHLIGHT_OPENING_MARK.html_safe) + .gsub(CGI.escapeHTML(HIGHLIGHT_CLOSING_MARK), HIGHLIGHT_CLOSING_MARK.html_safe) + .html_safe + end +end diff --git a/app/models/search/query.rb b/app/models/search/query.rb index 400e0a48d..0186cc424 100644 --- a/app/models/search/query.rb +++ b/app/models/search/query.rb @@ -12,7 +12,9 @@ class Search::Query < ApplicationRecord end end - alias_attribute :to_s, :terms + def to_s + Search::Stemmer.stem(terms.to_s) + end private def sanitize_terms diff --git a/app/models/search/result.rb b/app/models/search/result.rb index 28e1f7172..d90411bd5 100644 --- a/app/models/search/result.rb +++ b/app/models/search/result.rb @@ -4,15 +4,15 @@ class Search::Result < ApplicationRecord belongs_to :comment, foreign_key: :comment_id, optional: true def card_title - escape_highlight card_title_in_database + highlight(card_title_in_database, show: :full) end def card_description - escape_highlight card_description_in_database + highlight(card_description_in_database, show: :snippet) end def comment_body - escape_highlight comment_body_in_database + highlight(comment_body_in_database, show: :snippet) end def source @@ -24,14 +24,12 @@ class Search::Result < ApplicationRecord end private - def escape_highlight(html) - if html - CGI.escapeHTML(html) - .gsub(CGI.escapeHTML(Search::HIGHLIGHT_OPENING_MARK), Search::HIGHLIGHT_OPENING_MARK.html_safe) - .gsub(CGI.escapeHTML(Search::HIGHLIGHT_CLOSING_MARK), Search::HIGHLIGHT_CLOSING_MARK.html_safe) - .html_safe + def highlight(text, show:) + if text.present? && attribute?(:query) + highlighter = Search::Highlighter.new(query) + show == :snippet ? highlighter.snippet(text) : highlighter.highlight(text) else - nil + text end end end diff --git a/app/models/search/stemmer.rb b/app/models/search/stemmer.rb new file mode 100644 index 000000000..a92143965 --- /dev/null +++ b/app/models/search/stemmer.rb @@ -0,0 +1,13 @@ +module Search::Stemmer + extend self + + STEMMER = Mittens::Stemmer.new + + def stem(value) + if value.present? + value.split(/\s+/).map { |word| STEMMER.stem(word.downcase) }.join(" ") + else + value + end + end +end