From 3d2b01c38b2f391b238c136c04e2b8d08d413eb2 Mon Sep 17 00:00:00 2001 From: Christian Genco Date: Fri, 15 Aug 2025 12:05:07 -0500 Subject: [PATCH] Add batch upload session endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements three new endpoints for batch upload operations: - upload_session/start_batch: Start multiple upload sessions at once - upload_session/finish_batch: Finish multiple uploads in single request - upload_session/finish_batch/check: Check async batch operation status Includes result classes, error handling, and test specifications. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- api_coverage.md | 5 +- lib/dropbox_api.rb | 7 ++ .../files/upload_session_finish_batch.rb | 76 +++++++++++++ .../upload_session_finish_batch_check.rb | 38 +++++++ .../files/upload_session_start_batch.rb | 44 ++++++++ .../upload_session_finish_batch_job_status.rb | 21 ++++ .../upload_session_finish_batch_result.rb | 30 +++++ ...pload_session_finish_batch_result_entry.rb | 23 ++++ .../upload_session_start_batch_result.rb | 12 ++ .../upload_session_finish_batch_check_spec.rb | 49 ++++++++ .../files/upload_session_finish_batch_spec.rb | 105 ++++++++++++++++++ .../files/upload_session_start_batch_spec.rb | 41 +++++++ 12 files changed, 449 insertions(+), 2 deletions(-) create mode 100644 lib/dropbox_api/endpoints/files/upload_session_finish_batch.rb create mode 100644 lib/dropbox_api/endpoints/files/upload_session_finish_batch_check.rb create mode 100644 lib/dropbox_api/endpoints/files/upload_session_start_batch.rb create mode 100644 lib/dropbox_api/results/upload_session_finish_batch_job_status.rb create mode 100644 lib/dropbox_api/results/upload_session_finish_batch_result.rb create mode 100644 lib/dropbox_api/results/upload_session_finish_batch_result_entry.rb create mode 100644 lib/dropbox_api/results/upload_session_start_batch_result.rb create mode 100644 spec/endpoints/files/upload_session_finish_batch_check_spec.rb create mode 100644 spec/endpoints/files/upload_session_finish_batch_spec.rb create mode 100644 spec/endpoints/files/upload_session_start_batch_spec.rb diff --git a/api_coverage.md b/api_coverage.md index 018d3c71..d8f43e2f 100644 --- a/api_coverage.md +++ b/api_coverage.md @@ -73,9 +73,10 @@ API call | Status `/upload_session/append` | alias? `/upload_session/append_v2` | 🌕 `/upload_session/finish` | 🌕 -`/upload_session/finish_batch` | 🌑 -`/upload_session/finish_batch/check` | 🌑 +`/upload_session/finish_batch` | 🌕 +`/upload_session/finish_batch/check` | 🌕 `/upload_session/start` | 🌕 +`/upload_session/start_batch` | 🌕 ## Paper diff --git a/lib/dropbox_api.rb b/lib/dropbox_api.rb index d0b16b35..8168dab9 100644 --- a/lib/dropbox_api.rb +++ b/lib/dropbox_api.rb @@ -148,6 +148,10 @@ require 'dropbox_api/results/shared_folder_members' require 'dropbox_api/results/void_result' require 'dropbox_api/results/upload_session_start' +require 'dropbox_api/results/upload_session_start_batch_result' +require 'dropbox_api/results/upload_session_finish_batch_result_entry' +require 'dropbox_api/results/upload_session_finish_batch_result' +require 'dropbox_api/results/upload_session_finish_batch_job_status' require 'dropbox_api/results/delete_batch_result_entry' require 'dropbox_api/results/delete_batch_result' @@ -194,6 +198,9 @@ require 'dropbox_api/endpoints/files/upload_session_start' require 'dropbox_api/endpoints/files/upload_session_append_v2' require 'dropbox_api/endpoints/files/upload_session_finish' +require 'dropbox_api/endpoints/files/upload_session_start_batch' +require 'dropbox_api/endpoints/files/upload_session_finish_batch' +require 'dropbox_api/endpoints/files/upload_session_finish_batch_check' require 'dropbox_api/endpoints/files/delete_batch_check' require 'dropbox_api/endpoints/files/delete_batch' require 'dropbox_api/endpoints/files/create_folder_batch_check' diff --git a/lib/dropbox_api/endpoints/files/upload_session_finish_batch.rb b/lib/dropbox_api/endpoints/files/upload_session_finish_batch.rb new file mode 100644 index 00000000..ee466f5f --- /dev/null +++ b/lib/dropbox_api/endpoints/files/upload_session_finish_batch.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true +module DropboxApi::Endpoints::Files + class UploadSessionFinishBatch < DropboxApi::Endpoints::Rpc + Method = :post + Path = '/2/files/upload_session/finish_batch_v2' + ResultType = DropboxApi::Results::UploadSessionFinishBatchResult + ErrorType = nil + + include DropboxApi::OptionsValidator + + # This route helps you commit many files at once into a user's Dropbox. + # Use {Client#upload_session_start} and {Client#upload_session_append_v2} to + # upload file contents. We recommend uploading many files in parallel to + # increase throughput. Once the file contents have been uploaded, rather + # than calling {Client#upload_session_finish}, use this route to finish all + # your upload sessions in a single request. + # + # UploadSessionStartArg.close or UploadSessionAppendArg.close needs to be + # true for the last upload_session/start or upload_session/append:2 call + # of each upload session. The maximum size of a file one can upload to an + # upload session is 350 GiB. We allow up to 1000 entries in a single request. + # + # @param entries [Array] Commit information for each file in the batch. + # Each entry should contain: + # - cursor: A DropboxApi::Metadata::UploadSessionCursor + # - commit: A DropboxApi::Metadata::CommitInfo + # @return [DropboxApi::Results::UploadSessionFinishBatchResult] Result containing + # either an async_job_id or the file metadata for each entry + # @example + # entries = [ + # { + # cursor: DropboxApi::Metadata::UploadSessionCursor.new({ + # session_id: "session123", + # offset: 1024 + # }), + # commit: DropboxApi::Metadata::CommitInfo.new({ + # path: "/file1.txt", + # mode: "add" + # }) + # }, + # { + # cursor: DropboxApi::Metadata::UploadSessionCursor.new({ + # session_id: "session456", + # offset: 2048 + # }), + # commit: DropboxApi::Metadata::CommitInfo.new({ + # path: "/file2.txt", + # mode: "add" + # }) + # } + # ] + # result = client.upload_session_finish_batch(entries) + add_endpoint :upload_session_finish_batch do |entries| + if !entries.is_a?(Array) + raise ArgumentError, "entries must be an array" + end + + if entries.empty? || entries.size > 1000 + raise ArgumentError, "entries must contain between 1 and 1000 items" + end + + formatted_entries = entries.map do |entry| + unless entry[:cursor] && entry[:commit] + raise ArgumentError, "Each entry must have :cursor and :commit" + end + + { + cursor: entry[:cursor].to_hash, + commit: entry[:commit].to_hash + } + end + + perform_request(entries: formatted_entries) + end + end +end \ No newline at end of file diff --git a/lib/dropbox_api/endpoints/files/upload_session_finish_batch_check.rb b/lib/dropbox_api/endpoints/files/upload_session_finish_batch_check.rb new file mode 100644 index 00000000..c1098812 --- /dev/null +++ b/lib/dropbox_api/endpoints/files/upload_session_finish_batch_check.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true +module DropboxApi::Endpoints::Files + class UploadSessionFinishBatchCheck < DropboxApi::Endpoints::Rpc + Method = :post + Path = '/2/files/upload_session/finish_batch/check' + ResultType = DropboxApi::Results::UploadSessionFinishBatchJobStatus + ErrorType = DropboxApi::Errors::PollError + + # Returns the status of an asynchronous job for {Client#upload_session_finish_batch}. + # If success, it returns list of result for each entry. + # + # @param async_job_id [String] Id of the asynchronous job. + # This is the value of a response returned from the method that + # launched the job. + # @return [:in_progress, Array] This could be either the `:in_progress` + # flag or a list of file metadata entries. + # @example + # # First, start a batch finish operation + # batch_result = client.upload_session_finish_batch(entries) + # + # # If async, check the status + # if batch_result.async_job_id + # job_status = client.upload_session_finish_batch_check(batch_result.async_job_id) + # + # if job_status == :in_progress + # # Job is still processing + # else + # # job_status is an array of file metadata + # job_status.each do |file_metadata| + # puts "Uploaded: #{file_metadata.path_display}" + # end + # end + # end + add_endpoint :upload_session_finish_batch_check do |async_job_id| + perform_request async_job_id: async_job_id + end + end +end \ No newline at end of file diff --git a/lib/dropbox_api/endpoints/files/upload_session_start_batch.rb b/lib/dropbox_api/endpoints/files/upload_session_start_batch.rb new file mode 100644 index 00000000..ba5d682d --- /dev/null +++ b/lib/dropbox_api/endpoints/files/upload_session_start_batch.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +module DropboxApi::Endpoints::Files + class UploadSessionStartBatch < DropboxApi::Endpoints::Rpc + Method = :post + Path = '/2/files/upload_session/start_batch' + ResultType = DropboxApi::Results::UploadSessionStartBatchResult + ErrorType = nil + + include DropboxApi::OptionsValidator + + # This route starts batch of upload_sessions. Please refer to + # {Client#upload_session_start} usage. + # + # @param num_sessions [Integer] The number of upload sessions to start. + # Must be between 1 and 1000. + # @option options session_type [String] Type of upload session you want to + # start. If not specified, default is 'sequential'. Valid values are + # 'sequential' or 'concurrent'. + # @return [DropboxApi::Results::UploadSessionStartBatchResult] Result containing + # session IDs that can be used with upload_session/append and upload_session/finish + add_endpoint :upload_session_start_batch do |num_sessions, options = {}| + validate_options([ + :session_type + ], options) + + if num_sessions < 1 || num_sessions > 1000 + raise ArgumentError, "num_sessions must be between 1 and 1000" + end + + params = { + num_sessions: num_sessions + } + + if options[:session_type] + unless %w[sequential concurrent].include?(options[:session_type]) + raise ArgumentError, "session_type must be 'sequential' or 'concurrent'" + end + params[:session_type] = options[:session_type] + end + + perform_request(params) + end + end +end \ No newline at end of file diff --git a/lib/dropbox_api/results/upload_session_finish_batch_job_status.rb b/lib/dropbox_api/results/upload_session_finish_batch_job_status.rb new file mode 100644 index 00000000..6f7527ef --- /dev/null +++ b/lib/dropbox_api/results/upload_session_finish_batch_job_status.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +module DropboxApi::Results + # Result returned by {Client#upload_session_finish_batch_check} representing + # the status of an asynchronous batch upload session finish operation. + # + # The value will be either `:in_progress` or a list of file metadata entries. + class UploadSessionFinishBatchJobStatus < DropboxApi::Results::Base + def self.new(result_data) + case result_data['.tag'] + when 'in_progress' + :in_progress + when 'complete' + result_data['entries'].map do |entry| + DropboxApi::Results::UploadSessionFinishBatchResultEntry.new(entry) + end + else + raise NotImplementedError, "Unknown result type: #{result_data['.tag']}" + end + end + end +end \ No newline at end of file diff --git a/lib/dropbox_api/results/upload_session_finish_batch_result.rb b/lib/dropbox_api/results/upload_session_finish_batch_result.rb new file mode 100644 index 00000000..2795385b --- /dev/null +++ b/lib/dropbox_api/results/upload_session_finish_batch_result.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true +module DropboxApi::Results + # Result returned by {Client#upload_session_finish_batch} that may + # either launch an asynchronous job or complete synchronously. + # + # The value will be either an async_job_id string or a list of file metadata entries. + class UploadSessionFinishBatchResult < DropboxApi::Results::Base + def self.new(result_data) + case result_data['.tag'] + when 'async_job_id' + # Return a result object that has the async_job_id + super(result_data) + when 'complete' + # Return the array of entries directly + result_data['entries'].map do |entry| + DropboxApi::Results::UploadSessionFinishBatchResultEntry.new(entry) + end + else + raise NotImplementedError, "Unknown result type: #{result_data['.tag']}" + end + end + + # Returns the async job ID if the operation is asynchronous + # + # @return [String, nil] The async job ID or nil if operation completed synchronously + def async_job_id + @data['async_job_id'] if @data + end + end +end \ No newline at end of file diff --git a/lib/dropbox_api/results/upload_session_finish_batch_result_entry.rb b/lib/dropbox_api/results/upload_session_finish_batch_result_entry.rb new file mode 100644 index 00000000..f15895b4 --- /dev/null +++ b/lib/dropbox_api/results/upload_session_finish_batch_result_entry.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +module DropboxApi::Results + # Entry in the result returned by {Client#upload_session_finish_batch} + # representing the result for each file in the batch. + class UploadSessionFinishBatchResultEntry < DropboxApi::Results::Base + def self.new(result_data) + case result_data['.tag'] + when 'success' + # Return file metadata + DropboxApi::Metadata::File.new(result_data) + when 'failure' + # Return error information + # The actual error is in result_data['failure'] + DropboxApi::Errors::UploadSessionFinishError.build( + result_data['failure']['.tag'], + result_data['failure'] + ) + else + raise NotImplementedError, "Unknown result type: #{result_data['.tag']}" + end + end + end +end \ No newline at end of file diff --git a/lib/dropbox_api/results/upload_session_start_batch_result.rb b/lib/dropbox_api/results/upload_session_start_batch_result.rb new file mode 100644 index 00000000..a78edf70 --- /dev/null +++ b/lib/dropbox_api/results/upload_session_start_batch_result.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true +module DropboxApi::Results + class UploadSessionStartBatchResult < DropboxApi::Results::Base + # Returns a list of unique identifiers for the upload sessions. + # Pass each session_id to upload_session/append:2 and upload_session/finish. + # + # @return [Array] List of session IDs + def session_ids + @data['session_ids'] + end + end +end \ No newline at end of file diff --git a/spec/endpoints/files/upload_session_finish_batch_check_spec.rb b/spec/endpoints/files/upload_session_finish_batch_check_spec.rb new file mode 100644 index 00000000..2de57a6f --- /dev/null +++ b/spec/endpoints/files/upload_session_finish_batch_check_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe DropboxApi::Client, '#upload_session_finish_batch_check' do + before :each do + @client = DropboxApi::Client.new + end + + it "returns in_progress for ongoing jobs", :cassette => "upload_session_finish_batch_check/in_progress" do + result = @client.upload_session_finish_batch_check("sample_async_job_id") + + expect(result).to eq(:in_progress) + end + + it "returns file metadata for completed jobs", :cassette => "upload_session_finish_batch_check/complete" do + result = @client.upload_session_finish_batch_check("completed_job_id") + + expect(result).to be_a(Array) + result.each do |entry| + # Each entry should be either a File metadata or an error + expect(entry).to be_a(DropboxApi::Metadata::File).or be_a(DropboxApi::Errors::UploadSessionFinishError) + end + end + + it "handles mixed success and failure results", :cassette => "upload_session_finish_batch_check/mixed" do + result = @client.upload_session_finish_batch_check("mixed_results_job_id") + + expect(result).to be_a(Array) + + # Check that we have both success and failure entries + successes = result.select { |r| r.is_a?(DropboxApi::Metadata::File) } + failures = result.select { |r| r.is_a?(DropboxApi::Errors::UploadSessionFinishError) } + + expect(successes).not_to be_empty + expect(failures).not_to be_empty + end + + it "raises PollError for invalid job id", :cassette => "upload_session_finish_batch_check/invalid_job_id" do + expect { + @client.upload_session_finish_batch_check("invalid_job_id") + }.to raise_error(DropboxApi::Errors::PollError) + end + + it "handles internal errors", :cassette => "upload_session_finish_batch_check/internal_error" do + expect { + @client.upload_session_finish_batch_check("error_job_id") + }.to raise_error(DropboxApi::Errors::InternalError) + end +end \ No newline at end of file diff --git a/spec/endpoints/files/upload_session_finish_batch_spec.rb b/spec/endpoints/files/upload_session_finish_batch_spec.rb new file mode 100644 index 00000000..c05c1547 --- /dev/null +++ b/spec/endpoints/files/upload_session_finish_batch_spec.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe DropboxApi::Client, '#upload_session_finish_batch' do + before :each do + @client = DropboxApi::Client.new + end + + it "finishes a batch of upload sessions synchronously", :cassette => "upload_session_finish_batch/sync" do + # First, create upload sessions and upload content + start_result = @client.upload_session_start_batch(2) + + cursors = [] + start_result.session_ids.each_with_index do |session_id, i| + cursor = DropboxApi::Metadata::UploadSessionCursor.new( + 'session_id' => session_id, + 'offset' => 0 + ) + content = "File content #{i}" + @client.upload_session_append_v2(cursor, content, close: true) + cursors << cursor + end + + # Now finish the batch + entries = cursors.map.with_index do |cursor, i| + { + cursor: cursor, + commit: DropboxApi::Metadata::CommitInfo.new( + path: "/test_batch_#{i}.txt", + mode: 'add' + ) + } + end + + result = @client.upload_session_finish_batch(entries) + + # Result could be synchronous (array) or async (has async_job_id) + if result.is_a?(Array) + expect(result.length).to eq(2) + result.each do |entry| + expect(entry).to be_a(DropboxApi::Metadata::File).or be_a(DropboxApi::Errors::UploadSessionFinishError) + end + else + expect(result.async_job_id).to be_a(String) + end + end + + it "returns async job id for large batches", :cassette => "upload_session_finish_batch/async" do + # Create a larger batch that will likely be processed asynchronously + entries = [] + 10.times do |i| + entries << { + cursor: DropboxApi::Metadata::UploadSessionCursor.new( + 'session_id' => "session_#{i}", + 'offset' => 1024 + ), + commit: DropboxApi::Metadata::CommitInfo.new( + path: "/async_test_#{i}.txt", + mode: 'add' + ) + } + end + + result = @client.upload_session_finish_batch(entries) + + if result.respond_to?(:async_job_id) && result.async_job_id + expect(result.async_job_id).to be_a(String) + else + # If it completed synchronously, verify the results + expect(result).to be_a(Array) + end + end + + it "raises an error for empty entries" do + expect { + @client.upload_session_finish_batch([]) + }.to raise_error(ArgumentError, /must contain between 1 and 1000 items/) + end + + it "raises an error for too many entries" do + entries = [] + 1001.times do |i| + entries << { + cursor: DropboxApi::Metadata::UploadSessionCursor.new( + 'session_id' => "session_#{i}", + 'offset' => 0 + ), + commit: DropboxApi::Metadata::CommitInfo.new( + path: "/file_#{i}.txt", + mode: 'add' + ) + } + end + + expect { + @client.upload_session_finish_batch(entries) + }.to raise_error(ArgumentError, /must contain between 1 and 1000 items/) + end + + it "raises an error for invalid entry format" do + expect { + @client.upload_session_finish_batch([{cursor: nil, commit: nil}]) + }.to raise_error(ArgumentError, /Each entry must have :cursor and :commit/) + end +end \ No newline at end of file diff --git a/spec/endpoints/files/upload_session_start_batch_spec.rb b/spec/endpoints/files/upload_session_start_batch_spec.rb new file mode 100644 index 00000000..947f1b1e --- /dev/null +++ b/spec/endpoints/files/upload_session_start_batch_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe DropboxApi::Client, '#upload_session_start_batch' do + before :each do + @client = DropboxApi::Client.new + end + + it "starts a batch of upload sessions", :cassette => "upload_session_start_batch/success" do + result = @client.upload_session_start_batch(2) + + expect(result).to be_a(DropboxApi::Results::UploadSessionStartBatchResult) + expect(result.session_ids).to be_a(Array) + expect(result.session_ids.length).to eq(2) + expect(result.session_ids.first).to be_a(String) + end + + it "starts concurrent upload sessions", :cassette => "upload_session_start_batch/concurrent" do + result = @client.upload_session_start_batch(3, session_type: 'concurrent') + + expect(result).to be_a(DropboxApi::Results::UploadSessionStartBatchResult) + expect(result.session_ids).to be_a(Array) + expect(result.session_ids.length).to eq(3) + end + + it "raises an error for invalid num_sessions" do + expect { + @client.upload_session_start_batch(0) + }.to raise_error(ArgumentError, /must be between 1 and 1000/) + + expect { + @client.upload_session_start_batch(1001) + }.to raise_error(ArgumentError, /must be between 1 and 1000/) + end + + it "raises an error for invalid session_type" do + expect { + @client.upload_session_start_batch(1, session_type: 'invalid') + }.to raise_error(ArgumentError, /must be 'sequential' or 'concurrent'/) + end +end \ No newline at end of file