diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b9dc9ec7..83379fc71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ #### For the official .NET Release Notes please refer to https://docs.snowflake.com/en/release-notes/clients-drivers/dotnet # Changelog +- v5.2.1 + - Bug fix: Fix the extremely rare case where intermittent network issues during uploads to Azure Blob Storage prevent metadata updates - v5.2.0 - Added multi-targeting support. The appropriate build is selected by NuGet based on target framework and OS. - Fixed CRL validation to reject newly downloaded CRLs if their NextUpdate has already expired. diff --git a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs index 6a1581470..70e7ba033 100644 --- a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs @@ -100,6 +100,10 @@ public void TestExtractBucketNameAndPath() [TestCase(HttpStatusCode.BadRequest, ResultStatus.RENEW_TOKEN)] [TestCase(HttpStatusCode.NotFound, ResultStatus.NOT_FOUND_FILE)] [TestCase(HttpStatusCode.Forbidden, ResultStatus.ERROR)] // Any error that isn't the above will return ResultStatus.ERROR + [TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)] + [TestCase(HttpStatusCode.RequestTimeout, ResultStatus.ERROR)] + [TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)] + [TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.ERROR)] public void TestGetFileHeader(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus) { // Arrange @@ -190,6 +194,8 @@ private void AssertForGetFileHeaderTests(ResultStatus expectedResultStatus, File [TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)] [TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)] [TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)] + [TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)] + [TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)] public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus) { // Arrange @@ -202,7 +208,7 @@ public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedR .Returns((blobName) => { var mockBlobClient = new Mock(); - mockBlobClient.Setup(client => client.Upload(It.IsAny(), It.IsAny(), It.IsAny())) + mockBlobClient.Setup(client => client.Upload(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(() => MockAzureClient.createMockResponseForBlobContentInfo(key)); return mockBlobClient.Object; @@ -234,6 +240,9 @@ public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedR [TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)] [TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)] [TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)] + [TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)] + [TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)] + [TestCase(HttpStatusCode.TemporaryRedirect, ResultStatus.ERROR)] public async Task TestUploadFileAsync(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus) { // Arrange @@ -246,7 +255,7 @@ public async Task TestUploadFileAsync(HttpStatusCode httpStatusCode, ResultStatu .Returns((blobName) => { var mockBlobClient = new Mock(); - mockBlobClient.Setup(client => client.UploadAsync(It.IsAny(), It.IsAny(), It.IsAny())) + mockBlobClient.Setup(client => client.UploadAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(async () => await Task.Run(() => MockAzureClient.createMockResponseForBlobContentInfo(key)).ConfigureAwait(false)); return mockBlobClient.Object; @@ -287,6 +296,9 @@ private void AssertForUploadFileTests(ResultStatus expectedResultStatus) [TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)] [TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)] [TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)] + [TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)] + [TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)] + [TestCase(HttpStatusCode.NotFound, ResultStatus.ERROR)] public void TestDownloadFile(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus) { // Arrange @@ -334,6 +346,9 @@ public void TestDownloadFile(HttpStatusCode httpStatusCode, ResultStatus expecte [TestCase(HttpStatusCode.Forbidden, ResultStatus.NEED_RETRY)] [TestCase(HttpStatusCode.InternalServerError, ResultStatus.NEED_RETRY)] [TestCase(HttpStatusCode.ServiceUnavailable, ResultStatus.NEED_RETRY)] + [TestCase(HttpStatusCode.BadGateway, ResultStatus.ERROR)] + [TestCase(HttpStatusCode.GatewayTimeout, ResultStatus.ERROR)] + [TestCase(HttpStatusCode.RequestTimeout, ResultStatus.ERROR)] public async Task TestDownloadFileAsync(HttpStatusCode httpStatusCode, ResultStatus expectedResultStatus) { // Arrange diff --git a/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs b/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs index 43fcc5672..270e707de 100644 --- a/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs +++ b/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs @@ -110,7 +110,13 @@ public FileHeader GetFileHeader(SFFileMetadata fileMetadata) } catch (RequestFailedException ex) { - fileMetadata = HandleFileHeaderErr(ex, fileMetadata); + HandleFileHeaderErr(ex, fileMetadata); + return null; + } + catch (Exception ex) + { + Logger.Error("Blob client unknown get file header error: " + ex.Message); + fileMetadata.resultStatus = ResultStatus.ERROR.ToString(); return null; } @@ -139,7 +145,13 @@ public async Task GetFileHeaderAsync(SFFileMetadata fileMetadata, Ca } catch (RequestFailedException ex) { - fileMetadata = HandleFileHeaderErr(ex, fileMetadata); + HandleFileHeaderErr(ex, fileMetadata); + return null; + } + catch (Exception ex) + { + Logger.Error("Blob client unknown get file header error: " + ex.Message); + fileMetadata.resultStatus = ResultStatus.ERROR.ToString(); return null; } @@ -168,6 +180,11 @@ internal FileHeader HandleFileHeaderResponse(ref SFFileMetadata fileMetadata, Bl }; } + if (fileMetadata.stageInfo.isClientSideEncrypted && encryptionMetadata == null) + { + Logger.Error("File is expected to be client-side encrypted but no encryption metadata found."); + } + return new FileHeader { digest = GetMetadataValueCaseInsensitive(response, "sfcdigest", false), @@ -215,12 +232,22 @@ public void UploadFile(SFFileMetadata fileMetadata, Stream fileBytesStream, SFEn { // Issue the POST/PUT request fileBytesStream.Position = 0; - blobClient.Upload(fileBytesStream, overwrite: true); - blobClient.SetMetadata(metadata); + var uploadOptions = new BlobUploadOptions + { + Metadata = metadata + }; + blobClient.Upload(fileBytesStream, uploadOptions); } catch (RequestFailedException ex) { - fileMetadata = HandleUploadFileErr(ex, fileMetadata); + Logger.Error("Blob client request upload error: " + ex.Message); + HandleUploadFileErr(ex, fileMetadata); + return; + } + catch (Exception ex) + { + Logger.Error("Blob client unknown upload error: " + ex.Message); + fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString(); return; } @@ -245,12 +272,22 @@ public async Task UploadFileAsync(SFFileMetadata fileMetadata, Stream fileBytesS { // Issue the POST/PUT request fileBytesStream.Position = 0; - await blobClient.UploadAsync(fileBytesStream, true, cancellationToken).ConfigureAwait(false); - blobClient.SetMetadata(metadata); + var uploadOptions = new BlobUploadOptions + { + Metadata = metadata + }; + await blobClient.UploadAsync(fileBytesStream, uploadOptions, cancellationToken).ConfigureAwait(false); } catch (RequestFailedException ex) { - fileMetadata = HandleUploadFileErr(ex, fileMetadata); + Logger.Error("Blob client request upload error: " + ex.Message); + HandleUploadFileErr(ex, fileMetadata); + return; + } + catch (Exception ex) + { + Logger.Error("Blob client unknown upload error: " + ex.Message); + fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString(); return; } @@ -331,7 +368,14 @@ public void DownloadFile(SFFileMetadata fileMetadata, string fullDstPath, int ma catch (RequestFailedException ex) { File.Delete(fullDstPath); - fileMetadata = HandleDownloadFileErr(ex, fileMetadata); + HandleDownloadFileErr(ex, fileMetadata); + return; + } + catch (Exception ex) + { + File.Delete(fullDstPath); + Logger.Error("Blob client unknown download error: " + ex.Message); + fileMetadata.resultStatus = ResultStatus.ERROR.ToString(); return; } @@ -364,7 +408,14 @@ public async Task DownloadFileAsync(SFFileMetadata fileMetadata, string fullDstP catch (RequestFailedException ex) { File.Delete(fullDstPath); - fileMetadata = HandleDownloadFileErr(ex, fileMetadata); + HandleDownloadFileErr(ex, fileMetadata); + return; + } + catch (Exception ex) + { + File.Delete(fullDstPath); + Logger.Error("Blob client unknown download error: " + ex.Message); + fileMetadata.resultStatus = ResultStatus.ERROR.ToString(); return; } @@ -383,6 +434,7 @@ private SFFileMetadata HandleFileHeaderErr(RequestFailedException ex, SFFileMeta } else { + Logger.Error($"Unexpected HTTP status for file header operation: {ex.Status} {ex.ErrorCode}"); fileMetadata.resultStatus = ResultStatus.ERROR.ToString(); } return fileMetadata; @@ -390,20 +442,39 @@ private SFFileMetadata HandleFileHeaderErr(RequestFailedException ex, SFFileMeta private SFFileMetadata HandleUploadFileErr(RequestFailedException ex, SFFileMetadata fileMetadata) { + // 400 if (ex.Status == (int)HttpStatusCode.BadRequest) { fileMetadata.resultStatus = ResultStatus.RENEW_PRESIGNED_URL.ToString(); } + // 401 else if (ex.Status == (int)HttpStatusCode.Unauthorized) { fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString(); } + // 403, 500, 503 else if (ex.Status == (int)HttpStatusCode.Forbidden || ex.Status == (int)HttpStatusCode.InternalServerError || ex.Status == (int)HttpStatusCode.ServiceUnavailable) { fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString(); } + // other possible Azure blob service error codes: 404, 409, 412, 416 + else if (ex.Status == (int)HttpStatusCode.NotFound || + ex.Status == 409 || // Conflict + ex.Status == (int)HttpStatusCode.PreconditionFailed || + ex.Status == (int)HttpStatusCode.RequestedRangeNotSatisfiable) + { + String error = $"Unrecoverable HTTP status for file upload operation: {ex.Status} {ex.ErrorCode}"; + Logger.Error(error); + fileMetadata.resultStatus = ResultStatus.ERROR.ToString(); + } + else + { + String error = $"Unexpected HTTP status for file upload operation: {ex.Status} {ex.ErrorCode}"; + Logger.Error(error); + fileMetadata.resultStatus = ResultStatus.ERROR.ToString(); + } return fileMetadata; } @@ -419,6 +490,12 @@ private SFFileMetadata HandleDownloadFileErr(RequestFailedException ex, SFFileMe { fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString(); } + else + { + String error = $"Unexpected HTTP status for file download operation: {ex.Status} {ex.ErrorCode}"; + Logger.Error(error); + fileMetadata.resultStatus = ResultStatus.ERROR.ToString(); + } return fileMetadata; } } diff --git a/Snowflake.Data/Snowflake.Data.csproj b/Snowflake.Data/Snowflake.Data.csproj index c71e8c517..4592f216f 100644 --- a/Snowflake.Data/Snowflake.Data.csproj +++ b/Snowflake.Data/Snowflake.Data.csproj @@ -13,7 +13,7 @@ Snowflake Computing, Inc Snowflake Connector for .NET Snowflake - 5.2.0 + 5.2.1 Full 8