Skip to content

SNOW-2882054 fix for uploading azure metadata and bump to 5.2.1 #1288

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sfc-gh-mhofman merged 7 commits into from
Dec 9, 2025
Merged

SNOW-2882054 fix for uploading azure metadata and bump to 5.2.1 #1288

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
82a6590
SNOW-2882054 upload blob and metadata in a single call
sfc-gh-mhofman Dec 5, 2025
60874ea
SNOW-2882054 bump patch version
sfc-gh-mhofman Dec 5, 2025
80f8d9c
SNOW-2882054 improved logging
sfc-gh-mhofman Dec 5, 2025
96bf899
SNOW-2882054 fixed retries for failures scenarios during blob upload
sfc-gh-mhofman Dec 5, 2025
c39bb2e
SNOW-2882054 unit tests for edge case HTTP error codes
sfc-gh-mhofman Dec 5, 2025
bfa8449
SNOW-2882054 support for known azure blob service failure HTTP codes
sfc-gh-mhofman Dec 9, 2025
dc40af9
Merge branch 'master' into SNOW-2882054-fix-for-uploading-azure-metadata
sfc-gh-mhofman Dec 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#### For the official .NET Release Notes please refer to https://docs.snowflake.com/en/release-notes/clients-drivers/dotnet

# Changelog
- v5.2.1
- Bug fix: Fix the extremely rare case where intermittent network issues during uploads to Azure Blob Storage prevent metadata updates
- v5.2.0
- Added multi-targeting support. The appropriate build is selected by NuGet based on target framework and OS.
- Fixed CRL validation to reject newly downloaded CRLs if their NextUpdate has already expired.
Expand Down
19 changes: 17 additions & 2 deletions Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ public void TestExtractBucketNameAndPath()
[TestCase(HttpStatusCode.BadRequest, ResultStatus.RENEW_TOKEN)]
[TestCase(HttpStatusCode.NotFound, ResultStatus.NOT_FOUND_FILE)]
[TestCase(HttpStatusCode.Forbidden, ResultStatus.ERROR)] // Any error that isn't the above will return ResultStatus.ERROR
[TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)]
[TestCase(HttpStatusCode.RequestTimeout, ResultStatus.ERROR)]
[TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)]
[TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.ERROR)]
public void TestGetFileHeader(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus)
{
// Arrange
Expand Down Expand Up @@ -190,6 +194,8 @@ private void AssertForGetFileHeaderTests(ResultStatus expectedResultStatus, File
[TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)]
[TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)]
public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus)
{
// Arrange
Expand All @@ -202,7 +208,7 @@ public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedR
.Returns<string>((blobName) =>
{
var mockBlobClient = new Mock<BlobClient>();
mockBlobClient.Setup(client => client.Upload(It.IsAny<Stream>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
mockBlobClient.Setup(client => client.Upload(It.IsAny<Stream>(), It.IsAny<BlobUploadOptions>(), It.IsAny<CancellationToken>()))
.Returns(() => MockAzureClient.createMockResponseForBlobContentInfo(key));

return mockBlobClient.Object;
Expand Down Expand Up @@ -234,6 +240,9 @@ public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedR
[TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)]
[TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)]
[TestCase(HttpStatusCode.TemporaryRedirect, ResultStatus.ERROR)]
public async Task TestUploadFileAsync(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus)
{
// Arrange
Expand All @@ -246,7 +255,7 @@ public async Task TestUploadFileAsync(HttpStatusCode httpStatusCode, ResultStatu
.Returns<string>((blobName) =>
{
var mockBlobClient = new Mock<BlobClient>();
mockBlobClient.Setup(client => client.UploadAsync(It.IsAny<Stream>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
mockBlobClient.Setup(client => client.UploadAsync(It.IsAny<Stream>(), It.IsAny<BlobUploadOptions>(), It.IsAny<CancellationToken>()))
.Returns(async () => await Task.Run(() => MockAzureClient.createMockResponseForBlobContentInfo(key)).ConfigureAwait(false));

return mockBlobClient.Object;
Expand Down Expand Up @@ -287,6 +296,9 @@ private void AssertForUploadFileTests(ResultStatus expectedResultStatus)
[TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)]
[TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)]
[TestCase(HttpStatusCode.NotFound, ResultStatus.ERROR)]
public void TestDownloadFile(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus)
{
// Arrange
Expand Down Expand Up @@ -334,6 +346,9 @@ public void TestDownloadFile(HttpStatusCode httpStatusCode, ResultStatus expecte
[TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)]
[TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)]
[TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)]
[TestCase(HttpStatusCode.RequestTimeout, ResultStatus.ERROR)]
public async Task TestDownloadFileAsync(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus)
{
// Arrange
Expand Down
97 changes: 87 additions & 10 deletions Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,13 @@ public FileHeader GetFileHeader(SFFileMetadata fileMetadata)
}
catch (RequestFailedException ex)
{
fileMetadata = HandleFileHeaderErr(ex, fileMetadata);
HandleFileHeaderErr(ex, fileMetadata);
return null;
}
catch (Exception ex)
{
Logger.Error("Blob client unknown get file header error: " + ex.Message);
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
return null;
}

Expand Down Expand Up @@ -139,7 +145,13 @@ public async Task<FileHeader> GetFileHeaderAsync(SFFileMetadata fileMetadata, Ca
}
catch (RequestFailedException ex)
{
fileMetadata = HandleFileHeaderErr(ex, fileMetadata);
HandleFileHeaderErr(ex, fileMetadata);
return null;
}
catch (Exception ex)
{
Logger.Error("Blob client unknown get file header error: " + ex.Message);
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
return null;
}

Expand Down Expand Up @@ -168,6 +180,11 @@ internal FileHeader HandleFileHeaderResponse(ref SFFileMetadata fileMetadata, Bl
};
}

if (fileMetadata.stageInfo.isClientSideEncrypted && encryptionMetadata == null)
{
Logger.Error("File is expected to be client-side encrypted but no encryption metadata found.");
}

return new FileHeader
{
digest = GetMetadataValueCaseInsensitive(response, "sfcdigest", false),
Expand Down Expand Up @@ -215,12 +232,22 @@ public void UploadFile(SFFileMetadata fileMetadata, Stream fileBytesStream, SFEn
{
// Issue the POST/PUT request
fileBytesStream.Position = 0;
blobClient.Upload(fileBytesStream, overwrite: true);
blobClient.SetMetadata(metadata);
var uploadOptions = new BlobUploadOptions
{
Metadata = metadata
};
blobClient.Upload(fileBytesStream, uploadOptions);
}
catch (RequestFailedException ex)
{
fileMetadata = HandleUploadFileErr(ex, fileMetadata);
Logger.Error("Blob client request upload error: " + ex.Message);
HandleUploadFileErr(ex, fileMetadata);
return;
}
catch (Exception ex)
{
Logger.Error("Blob client unknown upload error: " + ex.Message);
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
return;
}

Expand All @@ -245,12 +272,22 @@ public async Task UploadFileAsync(SFFileMetadata fileMetadata, Stream fileBytesS
{
// Issue the POST/PUT request
fileBytesStream.Position = 0;
await blobClient.UploadAsync(fileBytesStream, true, cancellationToken).ConfigureAwait(false);
blobClient.SetMetadata(metadata);
var uploadOptions = new BlobUploadOptions
{
Metadata = metadata
};
await blobClient.UploadAsync(fileBytesStream, uploadOptions, cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException ex)
{
fileMetadata = HandleUploadFileErr(ex, fileMetadata);
Logger.Error("Blob client request upload error: " + ex.Message);
HandleUploadFileErr(ex, fileMetadata);
return;
}
catch (Exception ex)
{
Logger.Error("Blob client unknown upload error: " + ex.Message);
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
return;
}

Expand Down Expand Up @@ -331,7 +368,14 @@ public void DownloadFile(SFFileMetadata fileMetadata, string fullDstPath, int ma
catch (RequestFailedException ex)
{
File.Delete(fullDstPath);
fileMetadata = HandleDownloadFileErr(ex, fileMetadata);
HandleDownloadFileErr(ex, fileMetadata);
return;
}
catch (Exception ex)
{
File.Delete(fullDstPath);
Logger.Error("Blob client unknown download error: " + ex.Message);
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
return;
}

Expand Down Expand Up @@ -364,7 +408,14 @@ public async Task DownloadFileAsync(SFFileMetadata fileMetadata, string fullDstP
catch (RequestFailedException ex)
{
File.Delete(fullDstPath);
fileMetadata = HandleDownloadFileErr(ex, fileMetadata);
HandleDownloadFileErr(ex, fileMetadata);
return;
}
catch (Exception ex)
{
File.Delete(fullDstPath);
Logger.Error("Blob client unknown download error: " + ex.Message);
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
return;
}

Expand All @@ -383,27 +434,47 @@ private SFFileMetadata HandleFileHeaderErr(RequestFailedException ex, SFFileMeta
}
else
{
Logger.Error($"Unexpected HTTP status for file header operation: {ex.Status} {ex.ErrorCode}");
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
}
return fileMetadata;
}

private SFFileMetadata HandleUploadFileErr(RequestFailedException ex, SFFileMetadata fileMetadata)
{
// 400
if (ex.Status == (int)HttpStatusCode.BadRequest)
{
fileMetadata.resultStatus = ResultStatus.RENEW_PRESIGNED_URL.ToString();
}
// 401
else if (ex.Status == (int)HttpStatusCode.Unauthorized)
{
fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString();
}
// 403, 500, 503
else if (ex.Status == (int)HttpStatusCode.Forbidden ||
ex.Status == (int)HttpStatusCode.InternalServerError ||
ex.Status == (int)HttpStatusCode.ServiceUnavailable)
{
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
}
// other possible Azure blob service error codes: 404, 409, 412, 416
else if (ex.Status == (int)HttpStatusCode.NotFound ||
ex.Status == 409 || // Conflict
ex.Status == (int)HttpStatusCode.PreconditionFailed ||
ex.Status == (int)HttpStatusCode.RequestedRangeNotSatisfiable)
{
String error = $"Unrecoverable HTTP status for file upload operation: {ex.Status} {ex.ErrorCode}";
Logger.Error(error);
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
}
else
{
String error = $"Unexpected HTTP status for file upload operation: {ex.Status} {ex.ErrorCode}";
Logger.Error(error);
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
}
return fileMetadata;
}

Expand All @@ -419,6 +490,12 @@ private SFFileMetadata HandleDownloadFileErr(RequestFailedException ex, SFFileMe
{
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
}
else
{
String error = $"Unexpected HTTP status for file download operation: {ex.Status} {ex.ErrorCode}";
Logger.Error(error);
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
}
return fileMetadata;
}
}
Expand Down
2 changes: 1 addition & 1 deletion Snowflake.Data/Snowflake.Data.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
<Company>Snowflake Computing, Inc</Company>
<Product>Snowflake Connector for .NET</Product>
<Authors>Snowflake</Authors>
<Version>5.2.0</Version>
<Version>5.2.1</Version>
<DebugType>Full</DebugType>
<LangVersion>8</LangVersion>
</PropertyGroup>
Expand Down
Loading