diff --git a/Gemfile b/Gemfile index 38d6901d..88df2417 100644 --- a/Gemfile +++ b/Gemfile @@ -49,6 +49,7 @@ gem "actioncable-enhanced-postgresql-adapter" # longer paylaods w/ postgresql ac gem "aws-sdk-s3", require: false gem "postmark-rails" gem "ostruct" +gem "pdf-reader", "~> 2.11" gem "omniauth", "~> 2.1" gem "omniauth-google-oauth2", "~> 1.1" diff --git a/Gemfile.lock b/Gemfile.lock index f1594ebf..e95251d1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,7 @@ GEM remote: https://rubygems.org/ specs: + Ascii85 (2.0.1) actioncable (8.0.2.1) actionpack (= 8.0.2.1) activesupport (= 8.0.2.1) @@ -78,6 +79,7 @@ GEM uri (>= 0.13.1) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) + afm (1.0.0) amatch (0.4.2) mize tins (~> 1.0) @@ -175,6 +177,7 @@ GEM os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) hashdiff (1.2.1) + hashery (2.1.2) hashie (5.0.0) i18n (1.14.7) concurrent-ruby (~> 1.0) @@ -288,6 +291,12 @@ GEM parser (3.3.9.0) ast (~> 2.4.1) racc + pdf-reader (2.15.0) + Ascii85 (>= 1.0, < 3.0, != 2.0.0) + afm (>= 0.2.1, < 2) + hashery (~> 2.0) + ruby-rc4 + ttfunk pg (1.6.2-aarch64-linux) pg (1.6.2-aarch64-linux-musl) pg (1.6.2-arm64-darwin) @@ -417,6 +426,7 @@ GEM faraday (>= 1) faraday-multipart (>= 1) ruby-progressbar (1.13.0) + ruby-rc4 (0.1.5) ruby-vips (2.2.5) ffi (~> 1.12) logger @@ -488,6 +498,8 @@ GEM mize (~> 0.6) sync tsort (0.2.0) + ttfunk (1.8.0) + bigdecimal (~> 3.1) turbo-rails (2.0.17) actionpack (>= 7.1.0) railties (>= 7.1.0) @@ -553,6 +565,7 @@ DEPENDENCIES omniauth-microsoft_graph (~> 2.0) omniauth-rails_csrf_protection (~> 1.0.2) ostruct + pdf-reader (~> 2.11) pg (~> 1.1) postmark-rails puma (>= 6.0) diff --git a/app/javascript/stimulus/image_upload_controller.js b/app/javascript/stimulus/image_upload_controller.js index cd002fbe..7a61863f 100644 --- a/app/javascript/stimulus/image_upload_controller.js +++ b/app/javascript/stimulus/image_upload_controller.js @@ -35,21 +35,55 @@ export default class extends Controller { const input = this.fileTarget if (input.files && input.files[0]) { + const file = input.files[0] const reader = new FileReader() reader.onload = (e) => { - this.previewTarget.querySelector("img").src = e.target.result + const previewContainer = this.previewTarget.querySelector("[data-role='preview']") + const img = previewContainer.querySelector("img") + const fileIcon = previewContainer.querySelector("[data-role='file-icon']") + + if (file.type.startsWith('image/')) { + // Handle image files + img.src = e.target.result + img.style.display = 'block' + if (fileIcon) fileIcon.style.display = 'none' + } else if (file.type === 'application/pdf') { + // Handle PDF files + img.style.display = 'none' + if (fileIcon) { + fileIcon.style.display = 'flex' + } else { + // Create PDF icon if it doesn't exist + const pdfIcon = document.createElement('div') + pdfIcon.setAttribute('data-role', 'file-icon') + pdfIcon.className = 'w-full h-full flex items-center justify-center bg-red-100 dark:bg-red-900' + pdfIcon.innerHTML = ` + + + + ` + previewContainer.appendChild(pdfIcon) + } + } + this.element.classList.add("show-previews") this.contentTarget.focus() window.dispatchEvent(new CustomEvent('main-column-changed')) } - reader.readAsDataURL(input.files[0]) + reader.readAsDataURL(file) } } previewRemove() { if (!this.hasPreviewTarget) return - this.previewTarget.querySelector("img").src = '' + const previewContainer = this.previewTarget.querySelector("[data-role='preview']") + const img = previewContainer.querySelector("img") + const fileIcon = previewContainer.querySelector("[data-role='file-icon']") + + if (img) img.src = '' + if (fileIcon) fileIcon.style.display = 'none' + this.element.classList.remove("show-previews") if (this.hasContentTarget) this.contentTarget.focus() window.dispatchEvent(new CustomEvent('main-column-changed')) @@ -105,8 +139,11 @@ export default class extends Controller { const blob = item.getAsFile() if (!blob) return - const dataURL = await this.readPastedBlobAsDataURL(blob) - this.addImageToFileInput(dataURL, blob.type) + // Only handle images and PDFs + if (blob.type.startsWith('image/') || blob.type === 'application/pdf') { + const dataURL = await this.readPastedBlobAsDataURL(blob) + this.addFileToFileInput(dataURL, blob.type, blob.name || "pasted-file") + } } } this.previewUpdate() @@ -135,13 +172,22 @@ export default class extends Controller { ); } - addImageToFileInput(dataURL, fileType) { + addFileToFileInput(dataURL, fileType, fileName) { if (!this.hasFileTarget) return const fileList = new DataTransfer() const blob = this.dataURLtoBlob(dataURL, fileType) + + // Generate appropriate filename based on file type + let defaultFileName = "pasted-file" + if (fileType.startsWith('image/')) { + defaultFileName = "pasted-image.png" + } else if (fileType === 'application/pdf') { + defaultFileName = "pasted-document.pdf" + } + fileList.items.add( - new File([blob], "pasted-image.png", { type: fileType }), + new File([blob], fileName || defaultFileName, { type: fileType }), ) this.fileTarget.files = fileList.files } diff --git a/app/models/document.rb b/app/models/document.rb index 60169d72..5e8d9e1b 100644 --- a/app/models/document.rb +++ b/app/models/document.rb @@ -3,10 +3,7 @@ class Document < ApplicationRecord belongs_to :assistant, optional: true belongs_to :message, optional: true - has_one_attached :file do |file| - file.variant :small, resize_to_limit: [650, 490], preprocessed: true - file.variant :large, resize_to_limit: [1200, 900], preprocessed: true - end + has_one_attached :file enum :purpose, %w[fine-tune fine-tune-results assistants assistants_output].index_by(&:to_sym) @@ -20,11 +17,26 @@ class Document < ApplicationRecord validate :file_present def has_image?(variant = nil) + return false unless file.attached? && file.content_type&.start_with?("image/") + if variant.present? return has_file_variant_processed?(variant) end - file.attached? + true + end + + def image_variant(variant) + return nil unless file.attached? && file.content_type&.start_with?("image/") + + case variant.to_sym + when :small + file.variant(resize_to_limit: [650, 490]) + when :large + file.variant(resize_to_limit: [1200, 900]) + else + file + end end def image_url(variant, fallback: nil) @@ -42,32 +54,80 @@ def image_url(variant, fallback: nil) end def has_file_variant_processed?(variant) - r = file.attached? && - variant.present? && - (key = file.variant(variant.to_sym).key) && - ActiveStorage::Blob.service.exist?(key) + return false unless file.attached? && file.content_type&.start_with?("image/") + variant_obj = image_variant(variant) + return false unless variant_obj + + r = variant_obj.key && ActiveStorage::Blob.service.exist?(variant_obj.key) !!r end def fully_processed_url(variant) - file.attached? && variant.present? && file.representation(variant.to_sym).processed.url + return nil unless file.attached? && file.content_type&.start_with?("image/") + + variant_obj = image_variant(variant) + return nil unless variant_obj + + variant_obj.processed.url end def redirect_to_processed_path(variant) - return nil unless file.attached? && variant.present? + return nil unless file.attached? && file.content_type&.start_with?("image/") && variant.present? + + variant_obj = image_variant(variant) + return nil unless variant_obj Rails.application.routes.url_helpers.rails_representation_url( - file.representation(variant.to_sym), + variant_obj, only_path: true ) end def file_base64(variant = :large) return nil if !file.attached? - wait_for_file_variant_to_process!(variant.to_sym) - file_contents = file.variant(variant.to_sym).processed.download - base64 = Base64.strict_encode64(file_contents) + + if file.content_type&.start_with?("image/") + variant_obj = image_variant(variant) + return nil unless variant_obj + + wait_for_file_variant_to_process!(variant.to_sym) + file_contents = variant_obj.processed.download + else + # For non-image files, just return the raw file content + file_contents = file.download + end + + Base64.strict_encode64(file_contents) + end + + def has_document_pdf? + file.attached? && file.content_type == "application/pdf" + end + + def extract_pdf_text + return nil unless has_document_pdf? + + begin + pdf_data = file.download + + pdf_reader = PDF::Reader.new(StringIO.new(pdf_data)) + + text_content = "" + pdf_reader.pages.each_with_index do |page, index| + page_text = page.text + text_content += "--- Page #{index + 1} ---\n" if pdf_reader.pages.count > 1 + text_content += page_text + "\n\n" + end + text_content.strip + + rescue PDF::Reader::MalformedPDFError => e + Rails.logger.error "PDF file is corrupted or malformed: #{e.message}" + nil + rescue => e + Rails.logger.error "Error processing PDF: #{e.message}" + nil + end end private @@ -93,6 +153,11 @@ def file_present end def wait_for_file_variant_to_process!(variant) - file && file.attached? && file.variant(variant.to_sym).processed # this blocks until processing is done + return false unless file&.attached? && file.content_type&.start_with?("image/") + + variant_obj = image_variant(variant) + return false unless variant_obj + + variant_obj.processed # this blocks until processing is done end end diff --git a/app/models/message/document_image.rb b/app/models/message/document_image.rb index fb00e050..dac8bda2 100644 --- a/app/models/message/document_image.rb +++ b/app/models/message/document_image.rb @@ -11,9 +11,35 @@ def has_document_image?(variant = nil) documents.present? && documents.first.has_image?(variant) end + def has_document_pdf? + documents.present? && documents.first.file.attached? && documents.first.file.content_type == "application/pdf" + end + + def has_documents? + documents.present? && documents.first.file.attached? + end + def document_image_url(variant, fallback: nil) return nil unless has_document_image? documents.last.image_url(variant, fallback: fallback) end + + def document_pdf_url + return nil unless has_document_pdf? + + documents.last.file.url + end + + def document_filename + return nil unless has_documents? + + documents.last.filename + end + + def document_content_type + return nil unless has_documents? + + documents.last.file.content_type + end end diff --git a/app/services/ai_backend/open_ai.rb b/app/services/ai_backend/open_ai.rb index 2e646ccf..6a02159e 100644 --- a/app/services/ai_backend/open_ai.rb +++ b/app/services/ai_backend/open_ai.rb @@ -118,16 +118,33 @@ def system_message(content) def preceding_conversation_messages @conversation.messages.for_conversation_version(@message.version).where("messages.index < ?", @message.index).collect do |message| if @assistant.supports_images? && message.documents.present? && message.role == "user" - - content_with_images = [{ type: "text", text: message.content_text }] - content_with_images += message.documents.collect do |document| - { type: "image_url", image_url: { url: document.image_url(:large) }} + # Handle mixed content (images and PDFs) + content_with_media = [{ type: "text", text: message.content_text }] + + message.documents.each do |document| + if document.has_image? + content_with_media << { type: "image_url", image_url: { url: document.image_url(:large) }} + elsif document.has_document_pdf? + # Extract text from PDF and include it in the conversation + pdf_text = document.extract_pdf_text + if pdf_text.present? + content_with_media << { + type: "text", + text: "\n\n[PDF Document: #{document.filename}]\n#{pdf_text}" + } + else + content_with_media << { + type: "text", + text: "\n[PDF Document: #{document.filename} - Unable to extract text from this PDF]" + } + end + end end { role: message.role, name: message.name_for_api, - content: content_with_images, + content: content_with_media, }.compact else begin diff --git a/app/views/messages/_main_column.html.erb b/app/views/messages/_main_column.html.erb index 8e345dae..887235a4 100644 --- a/app/views/messages/_main_column.html.erb +++ b/app/views/messages/_main_column.html.erb @@ -266,7 +266,7 @@ <%= form.hidden_field :index, value: @messages&.length %> <%= form.hidden_field :version, value: @version %> <%= form.fields_for :documents, @new_message.documents.build do |document_form| %> - <%= document_form.file_field :file, accept: "image/*", class: "hidden", data: { image_upload_target: "file" } %> + <%= document_form.file_field :file, accept: "image/*,.pdf", class: "hidden", data: { image_upload_target: "file" } %> <% end %> <%= form.text_area :content_text, diff --git a/app/views/messages/_message.html.erb b/app/views/messages/_message.html.erb index b78cdafc..fc03eb03 100644 --- a/app/views/messages/_message.html.erb +++ b/app/views/messages/_message.html.erb @@ -103,6 +103,32 @@ end %> <%= spinner size: 6, class: "text-black dark:text-white" %> <% end %> + <% elsif message.has_document_pdf? %> + <%= link_to message.document_pdf_url, + class: "w-full my-2 p-4 border-2 border-gray-200 dark:border-gray-600 rounded-lg bg-gray-50 dark:bg-gray-800 hover:bg-gray-100 dark:hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-indigo-500 block", + target: "_blank", + rel: "noopener noreferrer" do %> +
+
+ + + +
+
+

+ <%= message.document_filename %> +

+

+ PDF Document - Click to open +

+
+
+ + + +
+
+ <% end %> <% end %> <%= formatted_text = format_for_display message, append_inside_tag: thinking_html(message, thinking) %> diff --git a/test/controllers/messages_controller_test.rb b/test/controllers/messages_controller_test.rb index 34a27d4b..55feff96 100644 --- a/test/controllers/messages_controller_test.rb +++ b/test/controllers/messages_controller_test.rb @@ -99,10 +99,39 @@ class MessagesControllerTest < ActionDispatch::IntegrationTest end end - (user_msg, asst_msg) = Message.last(2) + (user_msg, _asst_msg) = Message.last(2) assert_equal Document.last, user_msg.documents.first end + test "should create message with PDF attachment" do + # Create a simple PDF file for testing + pdf_content = "%PDF-1.4\n1 0 obj\n<<\n/Type /Catalog\n/Pages 2 0 R\n>>\nendobj\n2 0 obj\n<<\n/Type /Pages\n/Kids [3 0 R]\n/Count 1\n>>\nendobj\n3 0 obj\n<<\n/Type /Page\n/Parent 2 0 R\n/MediaBox [0 0 612 792]\n>>\nendobj\nxref\n0 4\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \n0000000115 00000 n \ntrailer\n<<\n/Size 4\n/Root 1 0 R\n>>\nstartxref\n174\n%%EOF" + + # Create a temporary PDF file + test_file = Tempfile.new(["test", ".pdf"]) + test_file.write(pdf_content) + test_file.rewind + + # Create a proper test file using the same pattern as the image test + pdf_file = fixture_file_upload(test_file.path, "application/pdf") + + assert_difference "Conversation.count", 1 do + assert_difference "Document.count", 1 do + assert_difference "Message.count", 2 do + post assistant_messages_url(@assistant), params: { message: { documents_attributes: {"0": {file: pdf_file}}, content_text: @message.content_text } } + end + end + end + + (user_msg, _asst_msg) = Message.last(2) + assert_equal Document.last, user_msg.documents.first + assert_equal "application/pdf", user_msg.documents.first.file.content_type + assert user_msg.has_document_pdf? + + test_file.close + test_file.unlink + end + test "should fail to create message when there is no content_text" do post assistant_messages_url(@assistant), params: { message: { content_text: nil } } assert_response :unprocessable_content