Add anthropic and google pdf processing (#728)

Author: mattlindsey

app/models/assistant.rb

@@ -13,6 +13,7 @@ class Assistant < ApplicationRecord
   has_many :messages, dependent: :destroy
 
   delegate :supports_images?, to: :language_model
+  delegate :supports_pdf?, to: :language_model
   delegate :api_service, to: :language_model
 
   belongs_to :language_model

app/services/ai_backend/anthropic.rb

@@ -187,16 +187,34 @@ def preceding_conversation_messages
             }
           ]
         }
-      elsif @assistant.supports_images? && message.documents.present?
+      elsif @assistant.supports_images? && message.documents.present? && message.role == "user"
+        # Handle mixed content (images and PDFs)
         content = [{ type: "text", text: message.content_text }]
-        content += message.documents.collect do |document|
-          { type: "image",
-            source: {
-              type: "base64",
-              media_type: document.file.blob.content_type,
-              data: document.file_base64(:large),
+
+        message.documents.each do |document|
+          if document.has_image?
+            content << { type: "image",
+              source: {
+                type: "base64",
+                media_type: document.file.blob.content_type,
+                data: document.file_base64(:large),
+              }
             }
-          }
+          elsif document.has_document_pdf?
+            # Extract text from PDF and include it in the conversation
+            pdf_text = document.extract_pdf_text
+            if pdf_text.present?
+              content << {
+                type: "text",
+                text: "\n\n[PDF Document: #{document.filename}]\n#{pdf_text}"
+              }
+            else
+              content << {
+                type: "text",
+                text: "\n[PDF Document: #{document.filename} - Unable to extract text from this PDF]"
+              }
+            end
+          end
         end
 
         {

app/services/ai_backend/gemini.rb

@@ -119,15 +119,30 @@ def system_message(content)
 
   def preceding_conversation_messages
     @conversation.messages.for_conversation_version(@message.version).where("messages.index < ?", @message.index).collect do |message|
-      if @assistant.supports_images? && message.documents.present?
-
+      if @assistant.supports_images? && message.documents.present? && message.role == "user"
+        # Handle mixed content (images and PDFs)
         content = [{ text: message.content_text }]
-        content += message.documents.collect do |document|
-          { inline_data: {
-              mime_type: document.file.blob.content_type,
-              data: document.file_base64(:large),
+
+        message.documents.each do |document|
+          if document.has_image?
+            content << { inline_data: {
+                mime_type: document.file.blob.content_type,
+                data: document.file_base64(:large),
+              }
             }
-          }
+          elsif document.has_document_pdf?
+            # Extract text from PDF and include it in the conversation
+            pdf_text = document.extract_pdf_text
+            if pdf_text.present?
+              content << {
+                text: "\n\n[PDF Document: #{document.filename}]\n#{pdf_text}"
+              }
+            else
+              content << {
+                text: "\n[PDF Document: #{document.filename} - Unable to extract text from this PDF]"
+              }
+            end
+          end
         end
 
         {

test/services/ai_backend/anthropic_test.rb

@@ -1,6 +1,7 @@
 require "test_helper"
 
 class AIBackend::AnthropicTest < ActiveSupport::TestCase
+  include ActionDispatch::TestProcess::FixtureFile
   setup do
     @conversation = conversations(:hello_claude)
     @assistant = assistants(:keith_claude35)
@@ -304,4 +305,128 @@ class AIBackend::AnthropicTest < ActiveSupport::TestCase
       assert_equal "openmeteo_get_current_and_todays_weather", tool_calls.dig(0, :function, :name)
     end
   end
+
+  test "preceding_conversation_messages processes PDF documents" do
+    # Create a new conversation with a message that has a PDF document
+    assistant = assistants(:keith_claude35)
+    assistant.language_model.update!(supports_pdf: true)
+
+    # Verify supports_pdf? works
+    assert assistant.supports_pdf?, "Assistant should support PDF processing"
+
+    conversation = Conversation.create!(
+      user: users(:keith),
+      assistant: assistant,
+      title: "PDF Test Conversation"
+    )
+
+    # Create a simple PDF file for testing
+    pdf_content = "%PDF-1.4\n1 0 obj\n<<\n/Type /Catalog\n/Pages 2 0 R\n>>\nendobj\n2 0 obj\n<<\n/Type /Pages\n/Kids [3 0 R]\n/Count 1\n>>\nendobj\n3 0 obj\n<<\n/Type /Page\n/Parent 2 0 R\n/MediaBox [0 0 612 792]\n/Contents 4 0 R\n>>\nendobj\n4 0 obj\n<<\n/Length 44\n>>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Hello World) Tj\nET\nendstream\nendobj\nxref\n0 5\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \n0000000115 00000 n \n0000000200 00000 n \ntrailer\n<<\n/Size 5\n/Root 1 0 R\n>>\nstartxref\n294\n%%EOF"
+
+    # Create a temporary PDF file
+    test_file = Tempfile.new(["test", ".pdf"])
+    test_file.write(pdf_content)
+    test_file.rewind
+
+    # Create a message with PDF attachment
+    message = conversation.messages.create!(
+      role: "user",
+      content_text: "Please analyze this PDF",
+      assistant: assistant
+    )
+
+    # Attach the PDF file
+    document = message.documents.create!(
+      file: fixture_file_upload(test_file.path, "application/pdf"),
+      filename: "test.pdf"
+    )
+
+    # Verify the document was created and is a PDF
+    assert document.has_document_pdf?, "Document should be identified as a PDF"
+
+    # Create a second message to test with
+    second_message = conversation.messages.create!(
+      role: "assistant",
+      content_text: "I'll analyze the PDF for you",
+      assistant: assistant
+    )
+
+    anthropic = AIBackend::Anthropic.new(users(:keith), assistant, conversation, second_message)
+    messages = anthropic.send(:preceding_conversation_messages)
+
+
+    # Find the message with PDF content
+    pdf_message = messages.find { |m| m[:content].is_a?(Array) && m[:content].any? { |c| c[:text]&.include?("PDF Document: test.pdf") } }
+
+    assert pdf_message, "Should find a message with PDF content. Messages: #{messages.inspect}"
+    assert_equal "user", pdf_message[:role]
+
+    # Check that the PDF content was processed (either successfully or with error message)
+    pdf_content_part = pdf_message[:content].find { |c| c[:text]&.include?("PDF Document: test.pdf") }
+    assert pdf_content_part, "Should find PDF content part"
+    # The PDF extraction might fail with our test PDF, so we check for either success or error message
+    assert pdf_content_part[:text].include?("PDF Document: test.pdf"), "Should include PDF document reference"
+    # Since our test PDF is not valid, we expect the error message
+    assert pdf_content_part[:text].include?("Unable to extract text from this PDF"), "Should include error message for failed PDF extraction"
+
+    test_file.close
+    test_file.unlink
+  end
+
+  test "preceding_conversation_messages handles PDF extraction errors gracefully" do
+    # Create a new conversation with a message that has a corrupted PDF document
+    assistant = assistants(:keith_claude35)
+    assistant.language_model.update!(supports_pdf: true)
+
+    conversation = Conversation.create!(
+      user: users(:keith),
+      assistant: assistant,
+      title: "PDF Error Test Conversation"
+    )
+
+    # Create a corrupted PDF file
+    corrupted_pdf_content = "%PDF-1.4\ncorrupted content"
+
+    # Create a temporary PDF file
+    test_file = Tempfile.new(["test", ".pdf"])
+    test_file.write(corrupted_pdf_content)
+    test_file.rewind
+
+    # Create a message with corrupted PDF attachment
+    message = conversation.messages.create!(
+      role: "user",
+      content_text: "Please analyze this PDF",
+      assistant: assistant
+    )
+
+    # Attach the corrupted PDF file
+    message.documents.create!(
+      file: fixture_file_upload(test_file.path, "application/pdf"),
+      filename: "corrupted.pdf"
+    )
+
+    # Create a second message to test with
+    second_message = conversation.messages.create!(
+      role: "assistant",
+      content_text: "I'll try to analyze the PDF for you",
+      assistant: assistant
+    )
+
+    anthropic = AIBackend::Anthropic.new(users(:keith), assistant, conversation, second_message)
+    messages = anthropic.send(:preceding_conversation_messages)
+
+    # Find the message with PDF content
+    pdf_message = messages.find { |m| m[:content].is_a?(Array) && m[:content].any? { |c| c[:text]&.include?("PDF Document: corrupted.pdf") } }
+
+    assert pdf_message, "Should find a message with PDF content"
+    assert_equal "user", pdf_message[:role]
+
+    # Check that the error message was included
+    pdf_content_part = pdf_message[:content].find { |c| c[:text]&.include?("PDF Document: corrupted.pdf") }
+    assert pdf_content_part, "Should find PDF content part"
+    assert_includes pdf_content_part[:text], "Unable to extract text from this PDF"
+
+    test_file.close
+    test_file.unlink
+  end
 end

test/services/ai_backend/gemini_test.rb

@@ -0,0 +1,139 @@
+require "test_helper"
+
+class AIBackend::GeminiTest < ActiveSupport::TestCase
+  include ActionDispatch::TestProcess::FixtureFile
+  setup do
+    @conversation = conversations(:hello_claude)
+    @assistant = assistants(:keith_claude35)
+    @assistant.language_model.update!(supports_tools: false)
+    @gemini = AIBackend::Gemini.new(
+      users(:keith),
+      @assistant,
+      @conversation,
+      @conversation.latest_message_for_version(:latest)
+    )
+    TestClient::Gemini.new(access_token: "abc")
+  end
+
+  test "initializing client works" do
+    assert @gemini.client.present?
+  end
+
+  test "preceding_conversation_messages processes PDF documents" do
+    # Create a new conversation with a message that has a PDF document
+    assistant = assistants(:keith_claude35)
+    assistant.language_model.update!(supports_pdf: true)
+
+    conversation = Conversation.create!(
+      user: users(:keith),
+      assistant: assistant,
+      title: "PDF Test Conversation"
+    )
+
+    # Create a simple PDF file for testing
+    pdf_content = "%PDF-1.4\n1 0 obj\n<<\n/Type /Catalog\n/Pages 2 0 R\n>>\nendobj\n2 0 obj\n<<\n/Type /Pages\n/Kids [3 0 R]\n/Count 1\n>>\nendobj\n3 0 obj\n<<\n/Type /Page\n/Parent 2 0 R\n/MediaBox [0 0 612 792]\n/Contents 4 0 R\n>>\nendobj\n4 0 obj\n<<\n/Length 44\n>>\nstream\nBT\n/F1 12 Tf\n72 720 Td\n(Hello World) Tj\nET\nendstream\nendobj\nxref\n0 5\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \n0000000115 00000 n \n0000000200 00000 n \ntrailer\n<<\n/Size 5\n/Root 1 0 R\n>>\nstartxref\n294\n%%EOF"
+
+    # Create a temporary PDF file
+    test_file = Tempfile.new(["test", ".pdf"])
+    test_file.write(pdf_content)
+    test_file.rewind
+
+    # Create a message with PDF attachment
+    message = conversation.messages.create!(
+      role: "user",
+      content_text: "Please analyze this PDF",
+      assistant: assistant
+    )
+
+    # Attach the PDF file
+    message.documents.create!(
+      file: fixture_file_upload(test_file.path, "application/pdf"),
+      filename: "test.pdf"
+    )
+
+    # Create a second message to test with
+    second_message = conversation.messages.create!(
+      role: "assistant",
+      content_text: "I'll analyze the PDF for you",
+      assistant: assistant
+    )
+
+    gemini = AIBackend::Gemini.new(users(:keith), assistant, conversation, second_message)
+    messages = gemini.send(:preceding_conversation_messages)
+
+
+    # Find the message with PDF content
+    pdf_message = messages.find { |m| m[:parts].is_a?(Array) && m[:parts].any? { |p| p[:text]&.include?("PDF Document: test.pdf") } }
+
+    assert pdf_message, "Should find a message with PDF content"
+    assert_equal "user", pdf_message[:role]
+
+    # Check that the PDF content was processed (either successfully or with error message)
+    pdf_content_part = pdf_message[:parts].find { |p| p[:text]&.include?("PDF Document: test.pdf") }
+    assert pdf_content_part, "Should find PDF content part"
+    # The PDF extraction might fail with our test PDF, so we check for either success or error message
+    assert pdf_content_part[:text].include?("PDF Document: test.pdf"), "Should include PDF document reference"
+    # Since our test PDF is not valid, we expect the error message
+    assert pdf_content_part[:text].include?("Unable to extract text from this PDF"), "Should include error message for failed PDF extraction"
+
+    test_file.close
+    test_file.unlink
+  end
+
+  test "preceding_conversation_messages handles PDF extraction errors gracefully" do
+    # Create a new conversation with a message that has a corrupted PDF document
+    assistant = assistants(:keith_claude35)
+    assistant.language_model.update!(supports_pdf: true)
+
+    conversation = Conversation.create!(
+      user: users(:keith),
+      assistant: assistant,
+      title: "PDF Error Test Conversation"
+    )
+
+    # Create a corrupted PDF file
+    corrupted_pdf_content = "%PDF-1.4\ncorrupted content"
+
+    # Create a temporary PDF file
+    test_file = Tempfile.new(["test", ".pdf"])
+    test_file.write(corrupted_pdf_content)
+    test_file.rewind
+
+    # Create a message with corrupted PDF attachment
+    message = conversation.messages.create!(
+      role: "user",
+      content_text: "Please analyze this PDF",
+      assistant: assistant
+    )
+
+    # Attach the corrupted PDF file
+    message.documents.create!(
+      file: fixture_file_upload(test_file.path, "application/pdf"),
+      filename: "corrupted.pdf"
+    )
+
+    # Create a second message to test with
+    second_message = conversation.messages.create!(
+      role: "assistant",
+      content_text: "I'll try to analyze the PDF for you",
+      assistant: assistant
+    )
+
+    gemini = AIBackend::Gemini.new(users(:keith), assistant, conversation, second_message)
+    messages = gemini.send(:preceding_conversation_messages)
+
+    # Find the message with PDF content
+    pdf_message = messages.find { |m| m[:parts].is_a?(Array) && m[:parts].any? { |p| p[:text]&.include?("PDF Document: corrupted.pdf") } }
+
+    assert pdf_message, "Should find a message with PDF content"
+    assert_equal "user", pdf_message[:role]
+
+    # Check that the error message was included
+    pdf_content_part = pdf_message[:parts].find { |p| p[:text]&.include?("PDF Document: corrupted.pdf") }
+    assert pdf_content_part, "Should find PDF content part"
+    assert_includes pdf_content_part[:text], "Unable to extract text from this PDF"
+
+    test_file.close
+    test_file.unlink
+  end
+end