刚学c#不久，想体验一下使用c#语言来爬虫，之前是用python来爬取的。（其实就是语法不一样而已，??）
下面写了个简单例子
爬取图片
创建dotnet new console --name crawler
安装dotnet add package HtmlAgilityPack --version 1.11.23
string url = "https://www.iqiyi.com/dianying_new/i_list_paihangbang.html";
HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
request.Timeout = 30 * 1000;
request.UserAgent = @"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/81.0.4044.122 Safari/537.36";
request.ContentType = "text/html; charset=utf-8";
request.CookieContainer = new CookieContainer();
string html;
using(HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if(response.StatusCode != HttpStatusCode.OK)
{
return;
}
else
{
StreamReader sr = new StreamReader(response.GetResponseStream(),Encoding.GetEncoding("utf-8"));
html = sr.ReadToEnd();
sr.Close();
}
}
HtmlDocument document = new HtmlDocument();
document.LoadHtml(html);
string li_xpath = "//*[@id=‘widget-tab-0‘]/div[2]/div/div[1]/ul/li";
HtmlNodeCollection liNodeList = document.DocumentNode.SelectNodes(li_xpath);
foreach(var liNode in liNodeList)
{
string img_xpath = "//*/a/img";
HtmlDocument imgDocument = new HtmlDocument();
imgDocument.LoadHtml(liNode.OuterHtml);
HtmlNode imgNode = imgDocument.DocumentNode.SelectSingleNode(img_xpath);
if (imgNode.Attributes["src"] != null)
{
string imgUrl = imgNode.Attributes["src"].Value;
}
}