TransferWebFileHelper1
1 public static class TransferWebFileHelper1
2 {
3 /// <summary>
4 /// 异步网络请求
5 /// </summary>
6 public static async Task CommunicatingWithWebServerAsync(string url,string saveFilePath, bool isGet = true, string postData = "", string contentType = "application/x-www-form-urlencoded;charset=utf-8")
7 {
8 if (string.IsNullOrWhiteSpace(url))
9 throw new ArgumentNullException("url");
10 if (string.IsNullOrWhiteSpace(contentType))
11 throw new ArgumentNullException("contentType");
12 HttpWebResponse response = null;
13 Stream stream = Stream.Null;
14 FileStream outStream = null;
15 try
16 {
17 if (url.ToUpper().Contains("HTTPS:")) //解决：“访问https请求被中止: 未能创建 SSL/TLS 安全通道”
18 {
19 ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls |
20 SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
21 }
22 HttpWebRequest request = isGet
23 ? GenerateHttpWebRequest(new Uri(url))
24 : GenerateHttpWebRequest(new Uri(url), postData, contentType);
25 response = await request.GetResponseAsync() as HttpWebResponse;
26 if (CategorizeResponse(response) == ResponseCategories.Success)
27 {
28 stream = response.GetResponseStream();
29
30 outStream = new FileStream(saveFilePath, FileMode.Create);
31 int readByteCount = 0;
32 byte[] buffers = new byte[1024];
33 while (true)
34 {
35 readByteCount = stream.Read(buffers, 0, buffers.Length);
36 if (readByteCount <= 0)
37 {
38 break;
39 }
40 outStream.Write(buffers, 0, readByteCount);
41 }
42
43 stream.Close();
44 stream.Dispose();
45 outStream.Close();
46 outStream.Dispose();
47 response.Close();
48 response.Dispose();
49 }
50
51
52 }
53 catch (Exception ex)
54 {
55
56 }
57 finally
58 {
59 System.GC.Collect();//解决程序提示线程用尽崩溃的问题
60 }
61 }
62
63 public static HttpWebRequest GenerateHttpWebRequest(Uri uri)
64 {
65 HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(uri);
66 httpRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36";//部分网站会检查此项
67 return httpRequest;
68 }
69
70 public static HttpWebRequest GenerateHttpWebRequest(Uri uri,
71 string postData,
72 string contentType)
73 {
74 HttpWebRequest httpRequest = GenerateHttpWebRequest(uri);
75
76 byte[] bytes = Encoding.UTF8.GetBytes(postData);
77
78 httpRequest.ContentType = contentType;
79
80 httpRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";
81
82 httpRequest.ContentLength = postData.Length;
83
84 using (Stream requestStream = httpRequest.GetRequestStream())
85 {
86 requestStream.Write(bytes, 0, bytes.Length);
87 }
88 return httpRequest;
89 }
90
91 public static ResponseCategories CategorizeResponse(HttpWebResponse httpResponse)
92 {
93 int statusCode = (int)httpResponse.StatusCode;
94 if ((statusCode >= 100) && (statusCode <= 199))
95 {
96 return ResponseCategories.Informational;
97 }
98 else if ((statusCode >= 200) && (statusCode <= 299))
99 {
100 return ResponseCategories.Success;
101 }
102 else if ((statusCode >= 300) && (statusCode <= 399))
103 {
104 return ResponseCategories.Redirected;
105 }
106 else if ((statusCode >= 400) && (statusCode <= 499))
107 {
108 return ResponseCategories.ClientError;
109 }
110 else if ((statusCode >= 500) && (statusCode <= 599))
111 {
112 return ResponseCategories.ServerError;
113 }
114 return ResponseCategories.Unknown;
115 }
116 }
Enumerator
public class Enumerator
{
public enum HttpMethodEnum
{
Post,
Get,
Put,
Delete,
Patch
}
}
ResponseCategories
public enum ResponseCategories
{
Unknown,
Informational,
Success,
Redirected,
ClientError,
ServerError
}
Program
class Program
{
private static string[] randomArray = new[]
{
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V",
"W", "X", "Y", "Z"
};
private static string GetRandomStr()
{
Random ran = new Random();
int i = ran.Next(0, 26);
Thread.Sleep(1);//尽量让生成的字符每一个都不相同
return randomArray[i];
}
static List<DownloadFile> ReadFileUrl(string folderName)
{
List<DownloadFile> list = new List<DownloadFile>();
DirectoryInfo dir;
List<string> contents = new List<string>();
string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName);
if (Directory.Exists(path))
{
dir = new DirectoryInfo(path);
FileInfo[] files = dir.GetFiles();
contents = File.ReadAllLines(files[0].FullName, Encoding.Default).ToList();
string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
contents.ForEach(fileName =>
{
//TODO 完全限定文件名必须少于 260 个字符，并且目录名必须少于 248 个字符。
string tempFileName = fileName.Split('?')[0].Replace("\\", "").Replace(":", "").Replace("*", "").Replace("\"", "").Replace("<", "").Replace(">", "").Replace("|", "");
int index = tempFileName.LastIndexOf("/");
tempFileName = tempFileName.Substring(index + 1, tempFileName.Length - index - 1);
int tempFileNameLength = tempFileName.Length;
if ((260 - saveFolder.Length - 2) < tempFileNameLength)
{
int difference = tempFileNameLength - (260 - saveFolder.Length - 2);//差
tempFileName = tempFileName.Substring(difference, tempFileName.Length - difference);
}
tempFileName = GetRandomStr() + "_" + tempFileName;
string saveFileName = Path.Combine(saveFolder, tempFileName);
var model = new DownloadFile(fileName, folderName, saveFileName);
list.Add(model);
});
}
return list;
}
private static void CreateFolder(string folderName)
{
string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
if (!Directory.Exists(saveFolder))
{
Directory.CreateDirectory(saveFolder);
}
}
private static void Main(string[] args)
{
List<string> folders = new List<string>()
{
"demo"
};
Parallel.ForEach(folders, folder => CreateFolder(folder));
List<DownloadFile> downloadFiles = new List<DownloadFile>();
Parallel.ForEach(folders, folder =>
{
downloadFiles.AddRange(ReadFileUrl(folder));
});
int i = 0;
Task[] taskList = new Task[downloadFiles.Count];
downloadFiles.ForEach(downloadFile =>
{
try
{
Task t = TransferWebFileHelper1.CommunicatingWithWebServerAsync(downloadFile.FileName,downloadFile.SaveFileName);
taskList[i] = t;
}
catch (Exception ex)
{
}
i++;
});
Task.WaitAll(taskList);
}
}
测试
程序根目录下，新建一个文件夹，名demo。内新建一个txt文件，文件名随意。测试如下图片链接[随意]：
http://www.banktunnel.eu/tumblr.com/tabea-lara.2_files/tumblr_n4kxopXkJ41sq93cpo10_1280.jpg
http://www.pptok.com/wp-content/uploads/2012/08/xunguang-9.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_3163ee0990499b7.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_25d8b5e402d0391.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFNWIA9vCAAmNhS2C8eAAAPyvQApsb4ACY2d098.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOGIIc0VAAom_BUlna4AAPyvQCLDjYACicU104.png
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOqIWH2cAAXWIl1DvA4AAPyvQDbhzkABdY6485.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFPGIfiurAAKxaF37_vcAAPyvQGII2YAArGA075.jpg
http://www.hr.com.cn/attachments/2015/06/b16af76b691f47609f3397e8cdd9c1e2.jpg
http://www.hr.com.cn/attachments/2015/06/226fc8acee0d478c9588ded1d04616b1.jpg
http://www.hr.com.cn/attachments/2015/06/c36e39b67d4c410cb794f553a271377b.jpg
下载完成的图片存放在根目录下名demo_download的文件夹下
测试过支持10W左右的图片下载，不崩溃。