本案例是爬的一部小说，爬取并存在本地
使用了动态浏览器头部信息，和 动态ip代理, 这2个方式可以有效的伪装自己，不至于被反爬，然后拉黑，之前已有记录，浏览器头部信息，也记录的有，
app.js
import request from 'request';
import userAgents from './common/userAgent.js';
import Promise from 'bluebird';
import cheerio from 'cheerio';//类似jquery写法
import fs from 'fs';
const Iconv = require('iconv').Iconv; const iconv = new Iconv('GBK', 'UTF-8'); const URL = 'http://www.qb5200.org/xiaoshuo/62/62493/';
let pageUrl = `${URL}.html`; //章节存放变量，初始化是第一章地址
//这里只做测试，所以用变量存，而实际应用中，应该使用数据缓存
const expiryTime = * * ;// 过期间隔时间，毫秒
let ips = null; //代理ip
let time = null;// 存储代理IP的时间，判断是否过期，如果过期重新请求
let pageNumber = ; //页码
//let info = '';//爬取到的内容存放变量
/**
* 请求免费代理，可做缓存，这里就存在变量中，只做测试
*/
const getProxyList = () => {
return new Promise((resolve, reject) => {
const nowDate = Date.now();
if( nowDate - time < expiryTime ){
resolve(ips);
return;
}
const apiURL = 'http://www.66ip.cn/mo.php?sxb=&tqsl=100&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=http%3A%2F%2Fwww.66ip.cn%2F%3Fsxb%3D%26tqsl%3D100%26ports%255B%255D2%3D%26ktip%3D%26sxa%3D%26radio%3Dradio%26submit%3D%25CC%25E1%2B%2B%25C8%25A1';
const options = { method: 'GET', url: apiURL, gzip: true, encoding: null,
headers: {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4',
'User-Agent': 'Mozilla/8.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',
'referer': 'http://www.66ip.cn/'
},
};
request(options, (error, response, body)=>{
try {
if(Buffer.isBuffer(body)){
const ret = body.toString().match(/\d{,}\.\d{,}\.\d{,}\.\d{,}:\d{,}/g);
ips = ret;
time = Date.now();
resolve(ret);
}
} catch (e) {
console.log(e);
}
});
})
}
//爬取小说
async function getPageIndex(ipList,url){
return new Promise((resolve, reject) => {
let userAgent = userAgents[parseInt(Math.random() * userAgents.length)];
let ip = ipList[parseInt(Math.random() * ips.length)];
let useIp = `http://${ip}`;const options = { method: 'GET', url: url, gzip: true, encoding: null,
headers: {
'User-Agent': userAgent, //动态设置浏览器头部信息
},
proxy: useIp, //动态设置代理ip
timeout:
};
request( options , (error, response, body)=>{
//429 Too Many Requests 这个问题貌似是服务器问题
if (error || body.toString().indexOf('429 Too Many Requests') >= ) {
console.log(`爬取页面失败，${error}，正在重新寻找代理ip... ×`);
resolve(undefined);
return;
}else{
console.log('爬取页面成功， √');
}
resolve(body)
})
})
}
//获取内容
async function getChapter(body){
return new Promise((resolve, reject) => {
let result = iconv.convert(new Buffer(body, 'binary')).toString();
let $ = cheerio.load(result);
let title = $('#title').text();
title = `第${pageNumber}章 ${title}`;
let content = $('#content').text();
let list = $('#footlink>a');
console.log(title);
//这里是因为有时候拿不到下一页的按钮，就自己退出了，所以做下处理
if(list.length === ){
resolve(undefined);
return;
}
list.each(function(i,item){
let $this = $(this);
if($this.text() === '下一页'){
let path = $this.attr('href');
if( path.indexOf('http') >= ){
path = null;
}else{
path = URL + path ;
}
resolve({
content: `${title}\n ${content}\n\n\n\n`,
path: path
});
}
}) })
}
//写入到本地
async function Write(info){
return new Promise((resolve, reject) => {
const decodedText = new Buffer(info);
fs.readFile('九天剑道.txt',function(err,_buffer){
if(err){//读取失败，则直接创建
fs.writeFile('九天剑道.txt',decodedText,function(err){
if(err){
console.log(err);
}else{
console.log('写入成功---!');
resolve(true);
}
})
}
if(Buffer.isBuffer(_buffer)){//判断是否是Buffer对象
fs.writeFile('九天剑道.txt',new Buffer(_buffer.toString() + info),function(err){
if(err){
console.log(err);
}else{
console.log('写入成功!');
resolve(true);
}
})
}
}) });
}
//启动方法
async function startFun(){
const ipList = await getProxyList();//获取代理ip
const body = await getPageIndex( ipList, pageUrl );//爬取主页面
if(!body){
startFun();
return;
}
const item = await getChapter(body);//获取爬到的内容
if(!item){
startFun();
return;
}
const info = item.content;
//判断是否有下一页，全部爬完之后写入到本地，生产文件
//如果章节过多，应该加上一点处理方式，这里不赘述，我想应该是这样的
if(item.path){
// if(pageNumber%2===0){
const isWrite = await Write(info);
// info = '';
// }
pageNumber++;
pageUrl = item.path;
startFun();
}
}
//启动方法
startFun();
package.json
{
"name": "reptile",
"version": "1.0.0",
"description": "",
"main": "app.js",
"scripts": {
"start": "babel-node app.js"
},
"author": "",
"license": "ISC",
"dependencies": {
"bluebird": "^3.5.1",
"cheerio": "^1.0.0-rc.2",
"fs": "^0.0.1-security",
"iconv": "^2.3.0",
"request": "^2.87.0"
},
"devDependencies": {
"babel-cli": "^6.26.0",
"babel-preset-es2015": "^6.24.1",
"babel-preset-stage-2": "^6.24.1"
}
}
es6的支持之前也记录的有，就是代理ip的成功率不怎么高，有点慢,
我这里用的是边爬边写入，爬完一次写入 应该也是可以的，这里就不做验证了