Merge pull request #1107 from basecamp/summary-size

Truncate summaries to keep within API limits

Gemfile.lock

@@ -563,11 +563,11 @@ GEM
     thruster (0.1.15-arm64-darwin)
     thruster (0.1.15-x86_64-darwin)
     thruster (0.1.15-x86_64-linux)
-    tiktoken_ruby (0.0.11.1)
-      rb_sys (= 0.9.106)
-    tiktoken_ruby (0.0.11.1-arm64-darwin)
-    tiktoken_ruby (0.0.11.1-x86_64-darwin)
-    tiktoken_ruby (0.0.11.1-x86_64-linux)
+    tiktoken_ruby (0.0.12)
+      rb_sys (~> 0.9)
+    tiktoken_ruby (0.0.12-arm64-darwin)
+    tiktoken_ruby (0.0.12-x86_64-darwin)
+    tiktoken_ruby (0.0.12-x86_64-linux)
     timeout (0.4.3)
     tsort (0.2.0)
     turbo-rails (2.0.16)

app/models/ai/tokenizer.rb

@@ -1,15 +1,16 @@
 class Ai::Tokenizer
-  attr_reader :text, :max_input_tokens
+  attr_reader :text, :max_input_tokens, :model
 
   class << self
-    def truncate(text, max_input_tokens: 8196)
-      new(text, max_input_tokens:).truncated
+    def truncate(text, max_input_tokens: 8196, model: "text-embedding-3-small")
+      new(text, max_input_tokens:, model:).truncated
     end
   end
 
-  def initialize(text, max_input_tokens: 8196)
+  def initialize(text, max_input_tokens: 8196, model: "text-embedding-3-small")
     @text = text
     @max_input_tokens = max_input_tokens
+    @model = model == "gpt-5-chat-latest" ? "chatgpt-4o-latest" : model # Not supported by tiktoken yet
   end
 
   def truncated
@@ -19,13 +20,13 @@ class Ai::Tokenizer
     (1..4).each do |i|
       tokens = tokenizer.encode(text)[0..(max_input_tokens - 20 - i)]
       return tokenizer.decode(tokens)
-    rescue Tiktoken::UnicodeError
-      raise if i == 4
+    rescue Encoding::UndefinedConversionError
+       raise if i == 4
     end
   end
 
   private
     def tokenizer
-      @tokenizer ||= Tiktoken.encoding_for_model("text-embedding-3-small")
+      @tokenizer ||= Tiktoken.encoding_for_model(model)
     end
 end

app/models/event/summarizer.rb

@@ -43,8 +43,14 @@ class Event::Summarizer
   private
     attr_reader :prompt, :llm_model
 
+    MAX_TOKENS = 128000
+
     def llm_response
-      @llm_response ||= chat.ask join_prompts("Summarize the following content:", summarizable_content)
+      @llm_response ||= chat.ask Ai::Tokenizer.truncate(llm_query, max_input_tokens: MAX_TOKENS, model: llm_model)
+    end
+
+    def llm_query
+      join_prompts("Summarize the following content:", summarizable_content)
     end
 
     def chat