问题描述
在上一篇博文(【Azure 存储服务】.NET7.0 示例代码之上传大文件到Azure Storage Blob (一)：https://www.cnblogs.com/lulight/p/17061631.html)中，介绍了第一种分片的方式上传文件。 本文章接着介绍第二种方式，使用 Microsoft.Azure.Storage.DataMovement 库中的 TransferManager.UploadAsync 通过并发的方式来上传大文件。
问题回答
第一步：添加 Microsoft.Azure.Storage.DataMovement
dotnet add package Microsoft.Azure.Storage.DataMovement
第二步：编写示例代码
String storageConnectionString = "xxxxxxxxxxxxxxxxxxx";
CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobclient = account.CreateCloudBlobClient();
CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123");
await blobcontainer.CreateIfNotExistsAsync();
// 获取文件路径
string sourcePath = @"C:\home\bigfiles0120.zip";
CloudBlockBlob docBlob = blobcontainer.GetBlockBlobReference("bigfiles-2");
await docBlob.DeleteIfExistsAsync();
// 设置并发操作的数量
TransferManager.Configurations.ParallelOperations = 64;
// 设置单块 blob 的大小，它必须在 4MB 到 100MB 之间，并且是 4MB 的倍数，默认情况下是 4MB
TransferManager.Configurations.BlockSize = 64 * 1024 * 1024;
// 设置传输上下文并跟踪上传进度
var context = new SingleTransferContext();
UploadOptions uploadOptions = new UploadOptions
{
DestinationAccessCondition = AccessCondition.GenerateIfExistsCondition()
};
context.ProgressHandler = new Progress<TransferStatus>(progress =>
{
//显示上传进度
Console.WriteLine("Bytes uploaded: {0}", progress.BytesTransferred);
});
// 使用 Stopwatch 查看上传所需时间
var timer = System.Diagnostics.Stopwatch.StartNew();
// 上传 Blob
TransferManager.UploadAsync(sourcePath, docBlob, uploadOptions, context, CancellationToken.None).Wait();
timer.Stop();
Console.WriteLine("Time (millisecond):" + timer.ElapsedMilliseconds);
Console.WriteLine("upload success");
第一种分片方式上传和第二步并发上传的代码执行对比：
全部代码
Program.cs
// See https://aka.ms/new-console-template for more information
Console.WriteLine("Hello, World! Start to upload big files...");
//第一种上传文件方法: Microsoft.WindowsAzure.Storage
Console.WriteLine("第一种上传文件方法: Microsoft.WindowsAzure.Storage");
await UploadMethodOne.WindowsAzureStorageUpload();
//第二种上传文件方法: Microsoft.Azure.Storage.DataMovement
Console.WriteLine("第二种上传文件方法: Microsoft.Azure.Storage.DataMovement");
await UploadMethodTwo.DataMovementUploadFiletoBlob();
Console.WriteLine("End!");
UploadMethodOne.cs
using Microsoft.Azure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.RetryPolicies;
public static class UploadMethodOne
{
public static async Task WindowsAzureStorageUpload()
{
TimeSpan backOffPeriod = TimeSpan.FromSeconds(2);
int retryCount = 1;
//设置请求选项
BlobRequestOptions requestoptions = new BlobRequestOptions()
{
SingleBlobUploadThresholdInBytes = 1024 * 1024 * 10, //10MB
ParallelOperationThreadCount = 12,
RetryPolicy = new ExponentialRetry(backOffPeriod, retryCount),
};
//String storageConnectionString = System.Environment.GetEnvironmentVariable("StorageConnectionString", EnvironmentVariableTarget.User);
//Console.WriteLine("String account string : "+storageConnectionString);
String storageConnectionString = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobclient = account.CreateCloudBlobClient();
//设置客户端默认请求选项
blobclient.DefaultRequestOptions = requestoptions;
CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123");
await blobcontainer.CreateIfNotExistsAsync();
//文件路径，文件大小
string sourcePath = @"C:\home\bigfiles0120.zip";
CloudBlockBlob blockblob = blobcontainer.GetBlockBlobReference("bigfiles-1");
//设置单个块 Blob 的大小（分块方式）
blockblob.StreamWriteSizeInBytes = 1024 * 1024 * 5;
try
{
Console.WriteLine("uploading");
//使用 Stopwatch 查看上传时间
var timer = System.Diagnostics.Stopwatch.StartNew();
using (var filestream = System.IO.File.OpenRead(sourcePath))
{
await blockblob.UploadFromStreamAsync(filestream);
}
timer.Stop();
Console.WriteLine(timer.ElapsedMilliseconds);
Console.WriteLine("Upload Successful, Time:" + timer.ElapsedMilliseconds);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
}
UploadMethodTwo.cs
using Microsoft.Azure.Storage;
using Microsoft.Azure.Storage.Blob;
using Microsoft.Azure.Storage.DataMovement;
public static class UploadMethodTwo
{
public async static Task DataMovementUploadFiletoBlob()
{
String storageConnectionString = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobclient = account.CreateCloudBlobClient();
CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123");
await blobcontainer.CreateIfNotExistsAsync();
// 获取文件路径
string sourcePath = @"C:\home\bigfiles0120.zip";
CloudBlockBlob docBlob = blobcontainer.GetBlockBlobReference("bigfiles-2");
await docBlob.DeleteIfExistsAsync();
// 设置并发操作的数量
TransferManager.Configurations.ParallelOperations = 64;
// 设置单块 blob 的大小，它必须在 4MB 到 100MB 之间，并且是 4MB 的倍数，默认情况下是 4MB
TransferManager.Configurations.BlockSize = 64 * 1024 * 1024;
// 设置传输上下文并跟踪上传进度
var context = new SingleTransferContext();
UploadOptions uploadOptions = new UploadOptions
{
DestinationAccessCondition = AccessCondition.GenerateIfExistsCondition()
};
context.ProgressHandler = new Progress<TransferStatus>(progress =>
{
//显示上传进度
Console.WriteLine("Bytes uploaded: {0}", progress.BytesTransferred);
});
// 使用 Stopwatch 查看上传所需时间
var timer = System.Diagnostics.Stopwatch.StartNew();
// 上传 Blob
TransferManager.UploadAsync(sourcePath, docBlob, uploadOptions, context, CancellationToken.None).Wait();
timer.Stop();
Console.WriteLine("Time (millisecond):" + timer.ElapsedMilliseconds);
Console.WriteLine("upload success");
}
}
参考资料
上传大文件到 Azure 存储块 Blob：https://docs.azure.cn/zh-cn/articles/azure-operations-guide/storage/aog-storage-blob-howto-upload-big-file-to-storage
当在复杂的环境中面临问题，格物之道需：浊而静之徐清，安以动之徐生。 云中，恰是如此!
分类: 【Azure Developer】, 【Azure 存储服务】
标签: Storage Blob Upload, Microsoft.Azure.Storage.DataMovemen, TransferManager.UploadAsync