using System; using System.Collections.Generic; using System.ComponentModel.Design; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using CDPWIB.DAL; using CDPWIB.Data; using CommonUtility; using HtmlAgilityPack; using MongoDB.Driver; using MongoDB.Driver.Builders; using MongoDB.Driver.Linq; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using WebKit; namespace CDPWIB.WebCollection { internal class QiDianCol : INovalCollect { private int Source = Convert.ToInt32(NovalSource.QiDian); private readonly MongoCollection<NovalTempBase> Novalcol = MongoConnectionFactory.GetMongoCollction<NovalTempBase>("Noval", typeof (NovalTempBase)); public void GetNovalTypeTemp() { try { var typecol = MongoConnectionFactory.GetMongoCollction<NovalTypeTemp>("Noval", typeof (NovalTypeTemp)); var subcol = MongoConnectionFactory.GetMongoCollction<NovalSubType>("Noval", typeof (NovalSubType)); // 大类 http://www.qidian.com/Javascript/qidian.bookstore.js?t=20130917 string typeshtml = NetHelper.HttpGet("http://www.qidian.com/Javascript/qidian.bookstore.js?t=20130917") .Replace("/", "") .Replace(" ", "") .Replace("\r", "") .Replace("\n", "") .Replace("\t", "") .Replace("|", "") .Replace(" ", ""); ; string subtypes = NetHelper.HttpGet("http://script.cmfu.com/script/BookStore.js ") .Replace(" ", "") .Replace("\r", "") .Replace("\n", "") .Replace("\t", "") .Replace("|", "") .Replace(" ", ""); ; Match mtype = Regex.Match(typeshtml, "CategoryArr:(.*?)]]", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline); string typesstring = mtype.Groups[1].Value + "]]"; JArray typearr = (JArray) JsonConvert.DeserializeObject(typesstring); //JsonTextWriter Match msubtype = Regex.Match(subtypes, "SubCategoryArr=(.*?);", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline); string subtypesstring = msubtype.Groups[1].Value; JArray subarr = (JArray) JsonConvert.DeserializeObject(subtypesstring); List<NovalTypeTemp> lstypes = new List<NovalTypeTemp>(10); //CategoryArr: [["全部", "-1"], ["玄幻", "21"], ["奇幻", "1"], ["武侠", "2"], ["仙侠", "22"], ["都市", "4"], ["历史", "5"], ["军事", "6"], ["游戏", "7"] for (int i = 0; i < typearr.Count; i++) { if (typearr[i][1].ToString() != "-1") { NovalTypeTemp type = new NovalTypeTemp() { WebNum = typearr[i][1].ToString().ToInt(), Name = typearr[i][0].ToString(), Source = Source }; lstypes.Add(type); } } IMongoQuery query = Query<NovalTypeTemp>.EQ(p => p.Source, Source); typecol.Remove(query); typecol.InsertBatch(lstypes); List<NovalSubType> subtypels = new List<NovalSubType>(300); foreach (var NovalTypeTemp in lstypes) { for (int i = 0; i < subarr.Count; i++) { var obj = subarr[i]; if (obj[0].ToString() == NovalTypeTemp.WebNum.ToString()) { NovalSubType subtype = new NovalSubType() { Name = obj[2].ToString(), ParentWebNum = NovalTypeTemp.WebNum, WebNum = obj[1].ToString().ToInt(), Source = Source }; subtypels.Add(subtype); } } } query = Query<NovalSubType>.EQ(p => p.Source, Source); subcol.Remove(query); subcol.InsertBatch(subtypels); } catch (Exception ex) { throw; } } /// <summary> /// 根据点击数页面查小说 /// </summary> public void GetNovals() { //取1到10页 //得到月点击排行小说。 string sourcehtml = string.Empty; HtmlDocument htmldocc = new HtmlDocument(); List<NovalTempBase> qdls = new List<NovalTempBase>(500); for (int j = 1; j < 11; j++) { sourcehtml = NetHelper.HttpGet("http://top.qidian.com/Book/TopDetail.aspx?TopType&Time=2&PageIndex=" + j); ; htmldocc.LoadHtml(sourcehtml); var doc = htmldocc.GetElementbyId("textlist"); //string tablehtml = "<table>" + doc.InnerHtml + "</table>"; // htmldocc.LoadHtml(tablehtml); //一页50列 for (int i = 2; i < 52; i++) { var trdoc = doc.SelectSingleNode("tr[" + i + "]"); //这里的下标，从1算起 var tdtype = trdoc.SelectSingleNode("td[2]/a"); var tdbook = trdoc.SelectSingleNode("td[3]/a[1]"); var tdclick = trdoc.SelectSingleNode("td[4]"); var tdauth = trdoc.SelectSingleNode("td[5]/a"); Match typematch = Regex.Match(tdtype.OuterHtml, "ChannelId=(\\d*?)&SubCategoryId=(\\d*?)‘"); Match bookmatck = Regex.Match(tdbook.OuterHtml, "Book/(\\d*?).aspx"); Match authmatch = Regex.Match(tdauth.OuterHtml, "id=(\\d*?)\""); int authid = authmatch.Groups[1].Value.ToInt(); int type = typematch.Groups[1].Value.ToInt(); int subtype = typematch.Groups[2].Value.ToInt(); int booknum = bookmatck.Groups[1].Value.ToInt(); string bookname = tdbook.InnerText.Trim(); //http://image.cmfu.com/books/3127618/3127618.jpg string titleimg = "http://image.cmfu.com/books/" + booknum + "/" + booknum + ".jpg"; bool exist= qdls.Exists(p => p.SourceWebNum == booknum); if (!exist) { NovalTempBase qidian = new NovalTempBase() { AuthName = tdauth.InnerText.Trim(), AuthId = authid, SubType = subtype, TitleImg = titleimg, Title = bookname, TotalClick = tdclick.InnerText.ToInt(), TotalComment = 0, Type = type, SourceWebNum = booknum, Source = Source }; qdls.Add(qidian); } } } PublicMethod.InsertAndUpdateNovalTmp(qdls,Source); } //public void GetNovalsByType() //{ //} /// <summary> /// 得到小说章节 ，个别来源，带分卷。 /// </summary> public void GetNovalChapers() { //http://sight.qq.com/book/chapterpage?uin=0&g_tk=5381&callback=_Callback&pagesize=100&pageno=2&bid=16043&_r=0.6934567329008132 var novalcol = MongoConnectionFactory.GetMongoCollction<NovalTempBase>("Noval", typeof (NovalTempBase)); var books = novalcol.AsQueryable().Where(p=>p.Source==Source).ToList(); foreach (var infoQidian in books) { GetSingleNovalChapers(infoQidian.SourceWebNum); } } public void GetSingleNovalChapers(int novalwebnum) { IMongoQuery q2 = Query<NovalVolumeTemp>.EQ(p => p.Source, Source); IMongoQuery q1 = Query<NovalVolumeTemp>.EQ(p => p.NovalWebNum, novalwebnum); IMongoQuery[] qarray = { q1, q2 }; IMongoQuery query = Query.And(qarray); var chaptercol = MongoConnectionFactory.GetMongoCollction<NovalChapterTemp>("Noval", typeof(NovalChapterTemp)); var volumecol = MongoConnectionFactory.GetMongoCollction<NovalVolumeTemp>("Noval", typeof (NovalVolumeTemp)); List<NovalChapterTemp> lschapters = new List<NovalChapterTemp>(1000); List<NovalVolumeTemp> lsvolumes = new List<NovalVolumeTemp>(10); int chapterorder = 1; int volumeorder = 1; HtmlDocument htmldocc = new HtmlDocument(); //http://read.qidian.com/BookReader/3127618.aspx string sourcehtml = string.Empty; string url = "http://read.qidian.com/BookReader/" + novalwebnum + ".aspx"; try { sourcehtml = NetHelper.HttpGet(url); //目录主页 htmldocc.LoadHtml(sourcehtml); var doc = htmldocc.GetElementbyId("content"); int i = 1; var topdoc = doc.SelectSingleNode("div[" + i + "]"); while (topdoc != null) { var topa = topdoc.SelectSingleNode("div/a"); //如果是vip章节，没有这个A标签。 int topnum; //分卷信息 if (topa != null) { string topahtml = topa.OuterHtml; //href="http://www.qidian.com/BookReader/vol,107580,486625.aspx" Match m = Regex.Match(topahtml, ",(\\d*?).aspx"); topnum = m.Groups[1].Value.ToInt(); var topaname = topdoc.SelectSingleNode("div/b"); string topname = topaname.InnerText.Trim(); topname = topname.Replace(" ", "").Split(‘;‘)[1]; //if(topname=="作品相关") NovalVolumeTemp volume = new NovalVolumeTemp() { Sort = volumeorder, WebNum = topnum, Name = topname, NovalWebNum = novalwebnum, Source = Source }; lsvolumes.Add(volume); volumeorder++; } else { topnum = 0; } var contextdoc = doc.SelectSingleNode("div[" + (i + 1) + "]"); var chaperas = contextdoc.SelectNodes("div/ul/li/a"); //<a itemprop=‘url‘ href="http://read.qidian.com/BookReader/107580,20901221.aspx" title=‘凡人修仙传
字数：84 更新时间：2008-08-01 07:54:48‘><span itemprop=‘headline‘>呵呵！终于上架了！</span></a> //,(\d*?).aspx string chaptername = string.Empty; //章节信息 int chapterwebnum = 0; for (int x = 0; x < chaperas.Count; x++) { var chapera = chaperas[x]; chaptername = chapera.InnerText.Trim(); Match chapmatchwebnum = Regex.Match(chapera.OuterHtml, ",(\\d*?).aspx"); chapterwebnum = chapmatchwebnum.Groups[1].Value.ToInt(); NovalChapterTemp chapter = new NovalChapterTemp() { Name = chaptername, Sort = chapterorder, WebNum = chapterwebnum, VolumeId = topnum , NovalWebNum = novalwebnum, Source = Source }; lschapters.Add(chapter); chapterorder++; } i += 2; topdoc = doc.SelectSingleNode("div[" + i + "]"); } volumecol.Remove(query); volumecol.InsertBatch(lsvolumes); PublicMethod.InsertChapterTempToSQL(lschapters, Source, novalwebnum); } catch (Exception ex) { return; } } public void GetNovalCilckComment() { var novalcol = MongoConnectionFactory.GetMongoCollction<NovalTempBase>("Noval", typeof(NovalTempBase)); var books = novalcol.AsQueryable().Where(p => p.Source == Source).ToList(); string sourcehtml = string.Empty; string url = string.Empty; HtmlDocument htmldocc = new HtmlDocument(); foreach (var novalTempBase in books) { //http://www.qidian.com/Book/3106580.aspx url = "http://www.qidian.com/Book/" + novalTempBase.SourceWebNum + ".aspx"; sourcehtml = NetHelper.HttpGet(url); htmldocc.LoadHtml(sourcehtml); var cliclickdiv = htmldocc.GetElementbyId("contentdiv"); // /div/div/div[1]/table/tbody/tr/td[1] var clickcount = cliclickdiv.SelectSingleNode("div/div[1]/table/tr/td[1]") .InnerText.Replace("总点击", "") .Replace("：", "").Trim(); int click = Convert.ToInt32(clickcount); // string urlcom = "http://forum.qidian.com/NewForum/List.aspx?BookId=3106580"; ////http://forum.qidian.com/NewForum/List.aspx?BookId=3106580 // // http://c.pingba.qidian.com/BookComment.aspx?BookId=3106580 // url = "http://c.pingba.qidian.com/BookComment.aspx?" + novalTempBase.SourceWebNum; // sourcehtml = NetHelper.HttpGet(url); // htmldocc.LoadHtml(sourcehtml); novalTempBase.TotalClick = click; novalcol.Save(novalTempBase); } //目录主页 } } }