Move the hack to fetch completed cards to the insight command logic

We need to rework this as part of the filters revamp. "indexed_by" being
a mix of filters keeps biting us.

app/models/ai/prompts.rb

@@ -1,12 +1,21 @@
 module Ai::Prompts
   private
     def current_view_prompt
+      current_card_contents = if context.viewing_card_contents?
+        <<~PROMPT
+          BEGIN OF CURRENT CARD
+          #{context.cards.first.to_prompt}
+          END OF CURRENT CARD
+        PROMPT
+      end
+
       <<~PROMPT
         ## Current context:
 
-        Today is #{Time.current}
+        * Today: #{Time.current}
+        * **Current view where the user is**: #{context.viewing_card_contents? ? 'inside a card' : 'viewing a list of cards' }.
 
-        The user is currently #{context.viewing_card_contents? ? 'inside a card' : 'viewing a list of cards' }.     
+        #{current_card_contents}
       PROMPT
     end
 

app/models/card.rb

@@ -20,18 +20,18 @@ class Card < ApplicationRecord
 
   scope :indexed_by, ->(index) do
     case index
-    when "newest"  then reverse_chronologically
-    when "oldest"  then chronologically
-    when "latest"  then latest
+    when "newest" then reverse_chronologically
+    when "oldest" then chronologically
+    when "latest" then latest
     when "stalled" then stalled.chronologically
     when "closing_soon" then closing_soon.chronologically
     when "falling_back_soon" then falling_back_soon.chronologically
-    when "closed"  then closed
+    when "closed" then closed
     end
   end
 
   def cache_key
-    [ super, collection.name ].compact.join("/")
+    [super, collection.name].compact.join("/")
   end
 
   def card

app/models/card/promptable.rb

@@ -9,7 +9,7 @@ module Card::Promptable
 
   def to_prompt
     <<~PROMPT
-      ### Card #{id}
+      BEGIN OF CARD #{id}
 
       **Title:** #{title}
       **Description:**
@@ -31,6 +31,7 @@ module Card::Promptable
       * Path: #{collection_card_path(collection, self, script_name: Account.script_name)}
 
       #{comments.last(MAX_COMMENTS).collect(&:to_prompt).join("\n")}
+      END OF CARD #{id}
     PROMPT
   end
 end

app/models/command/ai/translator.rb

@@ -57,7 +57,7 @@ class Command::Ai::Translator
           "context": {                 // omit if empty
             "terms":        string[],  // filter cards by keywords
             "indexed_by":   "newest" | "oldest" | "latest" | "stalled"
-                            | "closed" | "closing_soon" | "falling_back_soon" | "all", // "all" is only possible in combination with indexed_by
+                            | "closed" | "closing_soon" | "falling_back_soon",
             "assignee_ids": <person>[],
             "assignment_status": "unassigned",
             "card_ids":     <card_id>[],
@@ -158,6 +158,7 @@ class Command::Ai::Translator
           * Answer questions about the data.
           * Getting insight about data: how things are progressing, blockers, highlights, etc.
           * Perform advanced querying and filtering on the data, not supported by the preset filters.
+          * Look for cards similar to a given card.
           * Summarize information
           * Check what a person has done
           * Any other question about cards, comments, discussions, persons, etc.
@@ -170,16 +171,14 @@ class Command::Ai::Translator
 
         ### Context to get insight
 
-        - When answering implies analyzing cards and comments, it always needs a context filter.
-          * When there is no suitable filter, use `indexed_by` with either `latest` or `all`:
-            -> Use "latest" when the query needs to consider open cards only.
-            -> Use "all" when asking about people working on cards.
-            -> Use "all" when the query needs to consider both open and closed cards.
+        - When answering implies querying certain cards, it always needs a context filter.
+          * When there is no suitable filter, use `indexed_by` with `latest`.
         - Queries that require analyzing cards, comments, people activity, etc. to extract information, ALWAYS
           require a `context` filter.
-        - If the current context is "inside a card" and the query makes sense in that context, you
+        - If the current context is "inside a card" and the query doesn't need other cards to be answered, you
           can omit context filter properties.
           * Inside a card you are seeing the card description and the discussion around it. The query may refer to that context.
+          * You will still need a filter if the request requires finding other cards. E.g: looking for similar cards.
 
         ## Examples
 
@@ -210,6 +209,8 @@ class Command::Ai::Translator
         - 123 → `/search 123` # Notice there is no "card" mention
         - package 123 → `/search package 123`
 
+        Don't use `card_ids` when passing a card id that serves as context for a /insight command.
+
         #### Filter by terms
 
         When user explicitly asks for cards about some topic, use the `terms` filter with the topic. Consider this
@@ -318,18 +319,20 @@ class Command::Ai::Translator
         #### Getting insight
 
         - most commented cards → { context: { indexed_by: "latest" }, commands: ["/insight most commented cards"] }
-        - very active cards → { context: { indexed_by: "all" }, commands: ["/insight very active cards"] }
-        - what has mike done → { context: { indexed_by: "all" }, commands: ["/insight what has mike done"] }
+        - very active cards → { context: { indexed_by: "latest" }, commands: ["/insight very active cards"] }
+        - what has mike done → { context: { indexed_by: "latest" }, commands: ["/insight what has mike done"] }
+        - where are the problems → { context: { indexed_by: "latest" }, commands: ["/insight where are the problems"] }
         - summarize cards completed by mike → { context: { closer_ids: ["mike"] }, commands: ["/insight summarize"] }
-        - who is working on the most challenging stuff → { context: { indexed_by: "all" }, commands: ["/insight who is working on the most challenging stuff"] }
+        - who is working on the most challenging stuff → { context: { indexed_by: "latest" }, commands: ["/insight who is working on the most challenging stuff"] }
 
         ### Filters and commands combined
 
         - cards related to infrastructure assigned to mike → { context: { assignee_ids: "mike", terms: ["infrastructure"] } }
         - assign john to the current #design cards and tag them with #v2  → { context: { tag_ids: ["design"] }, commands: ["/assign john", "/tag #v2"] }
         - close cards assigned to mike and assign them to roger → { context: {assignee_ids: ["mike"]}, commands: ["/close", "/assign roger"] }
+        - similar cards → { context: { indexed_by: "latest"}, commands: ["/insight similar cards"] }
         - summarize the cards assigned to jz → { context: { assignee_ids: ["jz"] }, commands: ["/insight summarize"] }
-        - summarize the work that ann has done recently → { context: { indexed_by: ["all"] }, commands: ["/insight summarize the work that ann has done recently"] }
+        - summarize the work that ann has done recently → { context: { indexed_by: "latest" }, commands: ["/insight summarize the work that ann has done recently"] }
       PROMPT
     end
 

app/models/command/get_insight.rb

@@ -1,7 +1,7 @@
 class Command::GetInsight < Command
   include ::Ai::Prompts, Command::Cards
 
-  store_accessor :data, :query
+  store_accessor :data, :query, :params
 
   def title
     "Insight query '#{query}'"
@@ -21,7 +21,7 @@ class Command::GetInsight < Command
   end
 
   private
-    MAX_CARDS = 100
+    MAX_COMPLETED_CARDS = 50
 
     def chat
       chat = RubyLLM.chat(model: "chatgpt-4o-latest")
@@ -40,6 +40,15 @@ class Command::GetInsight < Command
         - When asking for summaries, try to highlight key outcomes.
         - If you need further details or clarifications, indicate it.
         - When referencing cards or comments, always link them (see rules below).
+        - **NEVER** answer with cards that don't exist.
+
+        ## Critical rules
+
+        - Always assume that the user is querying about information in the system, not asking you to generate similar data.
+        - Never include cards that don't exist in your answers.
+        - When asking for similar cards, tickets, bugs, etc., never imagine those, just analyze the data and suggest existing
+          cards that are similar.
+        - If you are missing cards or information, indicate it instead of making up a response.
 
         ## Linking rules
 
@@ -48,11 +57,35 @@ class Command::GetInsight < Command
           * Don't add these as standalone links, but referencing words from the insight
         - Markdown link format: [anchor text](/full/path/).
           - Preserve the path exactly as provided (including the leading "/").
+        - Prefer anchor text links that read naturally over numbers.
         - When showing the card title as the link anchor text, also include #<card id> at the end between parentheses.
       PROMPT
     end
 
     def cards_context
-      cards.limit(MAX_CARDS).collect(&:to_prompt).join("\n")
+      promptable_cards.collect(&:to_prompt).join("\n")
+    end
+
+    def promptable_cards
+      if filter.indexed_by.latest? && context.viewing_list_of_cards?
+        cards_including_closed_ones
+      else
+        cards
+      end
+    end
+
+    def cards_including_closed_ones
+      closed_cards = user.accessible_cards.closed.recently_closed_first.limit(MAX_COMPLETED_CARDS)
+      open_cards = cards
+
+      open_sql = open_cards.to_sql
+      closed_sql = "SELECT * FROM (#{closed_cards.to_sql})" # isolate limit
+      sql = " SELECT * FROM ( #{open_sql} UNION ALL #{closed_sql} ) AS cards "
+
+      Card.from("(#{sql}) AS cards")
+    end
+
+    def filter
+      user.filters.from_params(params.reverse_merge(**FilterScoped::DEFAULT_PARAMS))
     end
 end

app/models/command/parser.rb

@@ -56,7 +56,7 @@ class Command::Parser
       when "/do"
         Command::Do.new(card_ids: cards.ids)
       when "/insight"
-        Command::GetInsight.new(query: combined_arguments, card_ids: cards.ids)
+        Command::GetInsight.new(query: combined_arguments, card_ids: cards.ids, params: filter.as_params)
       when "/add"
         Command::AddCard.new(card_title: combined_arguments, collection_id: guess_collection&.id)
       when "/search"

app/models/command/parser/context.rb

@@ -1,6 +1,8 @@
 class Command::Parser::Context
   attr_reader :user, :url, :script_name
 
+  MAX_CARDS = 100
+
   def initialize(user, url:, script_name: "")
     @user = user
     @url = url
@@ -10,15 +12,7 @@ class Command::Parser::Context
   end
 
   def cards
-    if viewing_card_contents?
-      user.accessible_cards.where id: params[:id]
-    elsif viewing_cards_index?
-      filter.cards.published
-    elsif viewing_search_results?
-      user.accessible_cards.where(id: user.search(params[:q]).select(:card_id))
-    else
-      Card.none
-    end
+    cards_from_current_view.limit(MAX_CARDS)
   end
 
   def viewing_card_contents?
@@ -70,6 +64,18 @@ class Command::Parser::Context
   private
     attr_reader :controller, :action, :params
 
+    def cards_from_current_view
+      if viewing_card_contents?
+        user.accessible_cards.where id: params[:id]
+      elsif viewing_cards_index?
+        filter.cards.published
+      elsif viewing_search_results?
+        user.accessible_cards.where(id: user.search(params[:q]).select(:card_id))
+      else
+        Card.none
+      end
+    end
+
     def viewing_card_perma?
       controller == "cards" && action == "show"
     end

app/models/comment/promptable.rb

@@ -7,7 +7,7 @@ module Comment::Promptable
 
   def to_prompt
     <<~PROMPT
-        ### Comment #{id}
+        BEGIN OF COMMENT #{id}
 
         **Content:**
 
@@ -21,6 +21,7 @@ module Comment::Promptable
         * Created by: #{creator.name}}
         * Created at: #{created_at}}
         * Path: #{collection_card_path(card.collection, card, anchor: ActionView::RecordIdentifier.dom_id(self), script_name: Account.script_name)}
+        END OF COMMENT #{id}
       PROMPT
   end
 end

app/models/event/promptable.rb

@@ -3,12 +3,14 @@ module Event::Promptable
 
   def to_prompt
     <<~PROMPT
+        BEGIN OF EVENT #{id}
         ## Event #{action} (#{eventable_type} #{eventable_id}))
 
         * Created at: #{created_at}
         * Created by: #{creator.name}
 
         #{eventable.to_prompt}
+        END OF EVENT #{id}
       PROMPT
   end
 end

app/models/filter.rb

@@ -61,6 +61,6 @@ class Filter < ApplicationRecord
 
   private
     def include_closed_cards?
-      indexed_by.closed? || closure_window || closers.present? || card_ids.present? || indexed_by.all?
+      indexed_by.closed? || closure_window || closers.present? || card_ids.present?
     end
 end

test/models/command/ai/translator_test.rb

@@ -62,6 +62,8 @@ class Command::Ai::TranslatorTest < ActionDispatch::IntegrationTest
   end
 
   test "filter by card id" do
+    assert_command({ context: { card_ids: [ cards(:logo).id ] } }, "this card", context: :card)
+
     assert_command({ context: { card_ids: [ 123 ] } }, "card 123")
     assert_command({ context: { card_ids: [ 123, 456 ] } }, "card 123, 456")
     assert_command({ commands: [ "/search 123" ] }, "123") # Notice existing cards will be intercepted earlier
@@ -167,19 +169,22 @@ class Command::Ai::TranslatorTest < ActionDispatch::IntegrationTest
   vcr_record!
 
   test "get insight" do
-    assert_command({ context: { indexed_by: "all" }, commands: [ "/insight cards where mike has commented on" ] }, "cards where mike has commented on")
+    assert_command({ context: { indexed_by: "latest" }, commands: [ "/insight cards similar to 1234" ] }, "cards similar to 1234")
+
+    assert_command({ context: { indexed_by: "latest" }, commands: [ "/insight cards where mike has commented on" ] }, "cards where mike has commented on")
     assert_command({ context: { assignee_ids: ["jz"] }, commands: [ "/insight cards where mike has commented" ] }, "cards where mike has commented assigned to jz")
     assert_command({ context: { indexed_by: "latest" }, commands: [ "/insight are there blockers here?" ] }, "are there blockers here?")
-    assert_command({ context: { indexed_by: "all" }, commands: [ "/insight stuff that jz has done lately" ] }, "stuff that jz has done lately")
+    assert_command({ context: { indexed_by: "latest" }, commands: [ "/insight stuff that jz has done lately" ] }, "stuff that jz has done lately")
     assert_command({ context: { indexed_by: "latest" }, commands: [ "/insight challenging cards" ] }, "challenging cards")
-    assert_command({ context: { indexed_by: "all" }, commands: [ "/insight who is everywhere?" ] }, "who is everywhere?")
-    assert_command({ context: { indexed_by: "all" }, commands: [ "/insight who is overloaded" ] }, "who is overloaded")
+    assert_command({ context: { indexed_by: "latest" }, commands: [ "/insight who is everywhere?" ] }, "who is everywhere?")
+    assert_command({ context: { indexed_by: "latest" }, commands: [ "/insight who is overloaded" ] }, "who is overloaded")
 
-    assert_command({ commands: [ "/insight summarize this" ] }, "summarize this")
-    assert_command({ commands: [ "/insight summarize this" ] }, "summarize this", context: :card)
+    assert_command({ context: { indexed_by: "latest" }, commands: [ "/insight cards similar to this one" ] }, "cards similar to this one", context: :card)
+
+    assert_command({ commands: [ "/insight summarize" ] }, "summarize", context: :card)
     assert_command({ commands: [ "/insight are there blockers?" ] }, "are there blockers?", context: :card)
 
-    assert_command({ context: { indexed_by: "all" }, commands: [ "/insight cards with much activity" ] }, "cards with much activity")
+    assert_command({ context: { indexed_by: "latest" }, commands: [ "/insight cards with much activity" ] }, "cards with much activity")
   end
 
   test "combine commands and filters" do