Add pdf upload and processing with OpenAI (#726)

mattlindsey · web-flow · commit 7bf86606bc31 · 2025-10-21T16:36:07.000-04:00
diff --git a/Gemfile b/Gemfile
@@ -49,6 +49,7 @@ gem "actioncable-enhanced-postgresql-adapter" # longer paylaods w/ postgresql ac
 gem "aws-sdk-s3", require: false
 gem "postmark-rails"
 gem "ostruct"
+gem "pdf-reader", "~> 2.11"
 
 gem "omniauth", "~> 2.1"
 gem "omniauth-google-oauth2", "~> 1.1"
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,6 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    Ascii85 (2.0.1)
     actioncable (8.0.2.1)
       actionpack (= 8.0.2.1)
       activesupport (= 8.0.2.1)
@@ -78,6 +79,7 @@ GEM
       uri (>= 0.13.1)
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
+    afm (1.0.0)
     amatch (0.4.2)
       mize
       tins (~> 1.0)
@@ -175,6 +177,7 @@ GEM
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
     hashdiff (1.2.1)
+    hashery (2.1.2)
     hashie (5.0.0)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
@@ -288,6 +291,12 @@ GEM
     parser (3.3.9.0)
       ast (~> 2.4.1)
       racc
+    pdf-reader (2.15.0)
+      Ascii85 (>= 1.0, < 3.0, != 2.0.0)
+      afm (>= 0.2.1, < 2)
+      hashery (~> 2.0)
+      ruby-rc4
+      ttfunk
     pg (1.6.2-aarch64-linux)
     pg (1.6.2-aarch64-linux-musl)
     pg (1.6.2-arm64-darwin)
@@ -417,6 +426,7 @@ GEM
       faraday (>= 1)
       faraday-multipart (>= 1)
     ruby-progressbar (1.13.0)
+    ruby-rc4 (0.1.5)
     ruby-vips (2.2.5)
       ffi (~> 1.12)
       logger
@@ -488,6 +498,8 @@ GEM
       mize (~> 0.6)
       sync
     tsort (0.2.0)
+    ttfunk (1.8.0)
+      bigdecimal (~> 3.1)
     turbo-rails (2.0.17)
       actionpack (>= 7.1.0)
       railties (>= 7.1.0)
@@ -553,6 +565,7 @@ DEPENDENCIES
   omniauth-microsoft_graph (~> 2.0)
   omniauth-rails_csrf_protection (~> 1.0.2)
   ostruct
+  pdf-reader (~> 2.11)
   pg (~> 1.1)
   postmark-rails
   puma (>= 6.0)
diff --git a/app/javascript/stimulus/image_upload_controller.js b/app/javascript/stimulus/image_upload_controller.js
@@ -35,21 +35,55 @@ export default class extends Controller {
 
     const input = this.fileTarget
     if (input.files && input.files[0]) {
+      const file = input.files[0]
       const reader = new FileReader()
       reader.onload = (e) => {
-        this.previewTarget.querySelector("img").src = e.target.result
+        const previewContainer = this.previewTarget.querySelector("[data-role='preview']")
+        const img = previewContainer.querySelector("img")
+        const fileIcon = previewContainer.querySelector("[data-role='file-icon']")
+
+        if (file.type.startsWith('image/')) {
+          // Handle image files
+          img.src = e.target.result
+          img.style.display = 'block'
+          if (fileIcon) fileIcon.style.display = 'none'
+        } else if (file.type === 'application/pdf') {
+          // Handle PDF files
+          img.style.display = 'none'
+          if (fileIcon) {
+            fileIcon.style.display = 'flex'
+          } else {
+            // Create PDF icon if it doesn't exist
+            const pdfIcon = document.createElement('div')
+            pdfIcon.setAttribute('data-role', 'file-icon')
+            pdfIcon.className = 'w-full h-full flex items-center justify-center bg-red-100 dark:bg-red-900'
+            pdfIcon.innerHTML = `
+              <svg class="w-8 h-8 text-red-600 dark:text-red-400" fill="currentColor" viewBox="0 0 20 20">
+                <path fill-rule="evenodd" d="M4 4a2 2 0 012-2h4.586A2 2 0 0112 2.586L15.414 6A2 2 0 0116 7.414V16a2 2 0 01-2 2H6a2 2 0 01-2-2V4zm2 6a1 1 0 011-1h6a1 1 0 110 2H7a1 1 0 01-1-1zm1 3a1 1 0 100 2h6a1 1 0 100-2H7z" clip-rule="evenodd"></path>
+              </svg>
+            `
+            previewContainer.appendChild(pdfIcon)
+          }
+        }
+
         this.element.classList.add("show-previews")
         this.contentTarget.focus()
         window.dispatchEvent(new CustomEvent('main-column-changed'))
       }
-      reader.readAsDataURL(input.files[0])
+      reader.readAsDataURL(file)
     }
   }
 
   previewRemove() {
     if (!this.hasPreviewTarget) return
 
-    this.previewTarget.querySelector("img").src = ''
+    const previewContainer = this.previewTarget.querySelector("[data-role='preview']")
+    const img = previewContainer.querySelector("img")
+    const fileIcon = previewContainer.querySelector("[data-role='file-icon']")
+
+    if (img) img.src = ''
+    if (fileIcon) fileIcon.style.display = 'none'
+
     this.element.classList.remove("show-previews")
     if (this.hasContentTarget) this.contentTarget.focus()
     window.dispatchEvent(new CustomEvent('main-column-changed'))
@@ -105,8 +139,11 @@ export default class extends Controller {
         const blob = item.getAsFile()
         if (!blob) return
 
-        const dataURL = await this.readPastedBlobAsDataURL(blob)
-        this.addImageToFileInput(dataURL, blob.type)
+        // Only handle images and PDFs
+        if (blob.type.startsWith('image/') || blob.type === 'application/pdf') {
+          const dataURL = await this.readPastedBlobAsDataURL(blob)
+          this.addFileToFileInput(dataURL, blob.type, blob.name || "pasted-file")
+        }
       }
     }
     this.previewUpdate()
@@ -135,13 +172,22 @@ export default class extends Controller {
     );
   }
 
-  addImageToFileInput(dataURL, fileType) {
+  addFileToFileInput(dataURL, fileType, fileName) {
     if (!this.hasFileTarget) return
 
     const fileList = new DataTransfer()
     const blob = this.dataURLtoBlob(dataURL, fileType)
+
+    // Generate appropriate filename based on file type
+    let defaultFileName = "pasted-file"
+    if (fileType.startsWith('image/')) {
+      defaultFileName = "pasted-image.png"
+    } else if (fileType === 'application/pdf') {
+      defaultFileName = "pasted-document.pdf"
+    }
+
     fileList.items.add(
-      new File([blob], "pasted-image.png", { type: fileType }),
+      new File([blob], fileName || defaultFileName, { type: fileType }),
     )
     this.fileTarget.files = fileList.files
   }
diff --git a/app/models/document.rb b/app/models/document.rb
@@ -3,10 +3,7 @@ class Document < ApplicationRecord
   belongs_to :assistant, optional: true
   belongs_to :message, optional: true
 
-  has_one_attached :file do |file|
-    file.variant :small, resize_to_limit: [650, 490], preprocessed: true
-    file.variant :large, resize_to_limit: [1200, 900], preprocessed: true
-  end
+  has_one_attached :file
 
   enum :purpose, %w[fine-tune fine-tune-results assistants assistants_output].index_by(&:to_sym)
 
@@ -20,11 +17,26 @@ class Document < ApplicationRecord
   validate :file_present
 
   def has_image?(variant = nil)
+    return false unless file.attached? && file.content_type&.start_with?("image/")
+
     if variant.present?
       return has_file_variant_processed?(variant)
     end
 
-    file.attached?
+    true
+  end
+
+  def image_variant(variant)
+    return nil unless file.attached? && file.content_type&.start_with?("image/")
+
+    case variant.to_sym
+    when :small
+      file.variant(resize_to_limit: [650, 490])
+    when :large
+      file.variant(resize_to_limit: [1200, 900])
+    else
+      file
+    end
   end
 
   def image_url(variant, fallback: nil)
@@ -42,32 +54,80 @@ def image_url(variant, fallback: nil)
   end
 
   def has_file_variant_processed?(variant)
-    r = file.attached? &&
-      variant.present? &&
-      (key = file.variant(variant.to_sym).key) &&
-      ActiveStorage::Blob.service.exist?(key)
+    return false unless file.attached? && file.content_type&.start_with?("image/")
 
+    variant_obj = image_variant(variant)
+    return false unless variant_obj
+
+    r = variant_obj.key && ActiveStorage::Blob.service.exist?(variant_obj.key)
     !!r
   end
 
   def fully_processed_url(variant)
-    file.attached? && variant.present? && file.representation(variant.to_sym).processed.url
+    return nil unless file.attached? && file.content_type&.start_with?("image/")
+
+    variant_obj = image_variant(variant)
+    return nil unless variant_obj
+
+    variant_obj.processed.url
   end
 
   def redirect_to_processed_path(variant)
-    return nil unless file.attached? && variant.present?
+    return nil unless file.attached? && file.content_type&.start_with?("image/") && variant.present?
+
+    variant_obj = image_variant(variant)
+    return nil unless variant_obj
 
     Rails.application.routes.url_helpers.rails_representation_url(
-      file.representation(variant.to_sym),
+      variant_obj,
       only_path: true
     )
   end
 
   def file_base64(variant = :large)
     return nil if !file.attached?
-    wait_for_file_variant_to_process!(variant.to_sym)
-    file_contents = file.variant(variant.to_sym).processed.download
-    base64 = Base64.strict_encode64(file_contents)
+
+    if file.content_type&.start_with?("image/")
+      variant_obj = image_variant(variant)
+      return nil unless variant_obj
+
+      wait_for_file_variant_to_process!(variant.to_sym)
+      file_contents = variant_obj.processed.download
+    else
+      # For non-image files, just return the raw file content
+      file_contents = file.download
+    end
+
+    Base64.strict_encode64(file_contents)
+  end
+
+  def has_document_pdf?
+    file.attached? && file.content_type == "application/pdf"
+  end
+
+  def extract_pdf_text
+    return nil unless has_document_pdf?
+
+    begin
+      pdf_data = file.download
+
+      pdf_reader = PDF::Reader.new(StringIO.new(pdf_data))
+
+      text_content = ""
+      pdf_reader.pages.each_with_index do |page, index|
+        page_text = page.text
+        text_content += "--- Page #{index + 1} ---\n" if pdf_reader.pages.count > 1
+        text_content += page_text + "\n\n"
+      end
+      text_content.strip
+
+    rescue PDF::Reader::MalformedPDFError => e
+      Rails.logger.error "PDF file is corrupted or malformed: #{e.message}"
+      nil
+    rescue => e
+      Rails.logger.error "Error processing PDF: #{e.message}"
+      nil
+    end
   end
 
   private
@@ -93,6 +153,11 @@ def file_present
   end
 
   def wait_for_file_variant_to_process!(variant)
-    file && file.attached? && file.variant(variant.to_sym).processed # this blocks until processing is done
+    return false unless file&.attached? && file.content_type&.start_with?("image/")
+
+    variant_obj = image_variant(variant)
+    return false unless variant_obj
+
+    variant_obj.processed # this blocks until processing is done
   end
 end
diff --git a/app/models/message/document_image.rb b/app/models/message/document_image.rb
@@ -11,9 +11,35 @@ def has_document_image?(variant = nil)
     documents.present? && documents.first.has_image?(variant)
   end
 
+  def has_document_pdf?
+    documents.present? && documents.first.file.attached? && documents.first.file.content_type == "application/pdf"
+  end
+
+  def has_documents?
+    documents.present? && documents.first.file.attached?
+  end
+
   def document_image_url(variant, fallback: nil)
     return nil unless has_document_image?
 
     documents.last.image_url(variant, fallback: fallback)
   end
+
+  def document_pdf_url
+    return nil unless has_document_pdf?
+
+    documents.last.file.url
+  end
+
+  def document_filename
+    return nil unless has_documents?
+
+    documents.last.filename
+  end
+
+  def document_content_type
+    return nil unless has_documents?
+
+    documents.last.file.content_type
+  end
 end
diff --git a/app/services/ai_backend/open_ai.rb b/app/services/ai_backend/open_ai.rb
@@ -118,16 +118,33 @@ def system_message(content)
   def preceding_conversation_messages
     @conversation.messages.for_conversation_version(@message.version).where("messages.index < ?", @message.index).collect do |message|
       if @assistant.supports_images? && message.documents.present? && message.role == "user"
-
-        content_with_images = [{ type: "text", text: message.content_text }]
-        content_with_images += message.documents.collect do |document|
-          { type: "image_url", image_url: { url: document.image_url(:large) }}
+        # Handle mixed content (images and PDFs)
+        content_with_media = [{ type: "text", text: message.content_text }]
+
+        message.documents.each do |document|
+          if document.has_image?
+            content_with_media << { type: "image_url", image_url: { url: document.image_url(:large) }}
+          elsif document.has_document_pdf?
+            # Extract text from PDF and include it in the conversation
+            pdf_text = document.extract_pdf_text
+            if pdf_text.present?
+              content_with_media << {
+                type: "text",
+                text: "\n\n[PDF Document: #{document.filename}]\n#{pdf_text}"
+              }
+            else
+              content_with_media << {
+                type: "text",
+                text: "\n[PDF Document: #{document.filename} - Unable to extract text from this PDF]"
+              }
+            end
+          end
         end
 
         {
           role: message.role,
           name: message.name_for_api,
-          content: content_with_images,
+          content: content_with_media,
         }.compact
       else
         begin
diff --git a/app/views/messages/_main_column.html.erb b/app/views/messages/_main_column.html.erb
@@ -266,7 +266,7 @@
           <%= form.hidden_field :index, value: @messages&.length %>
           <%= form.hidden_field :version, value: @version %>
           <%= form.fields_for :documents, @new_message.documents.build do |document_form| %>
-            <%= document_form.file_field :file, accept: "image/*", class: "hidden", data: { image_upload_target: "file" } %>
+            <%= document_form.file_field :file, accept: "image/*,.pdf", class: "hidden", data: { image_upload_target: "file" } %>
           <% end %>
 
           <%= form.text_area :content_text,
diff --git a/app/views/messages/_message.html.erb b/app/views/messages/_message.html.erb
diff --git a/test/controllers/messages_controller_test.rb b/test/controllers/messages_controller_test.rb
