MinerHtmlThread.java 爬取页面线程

package com.iteye.injavawetrust.miner;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

import org.jsoup.Connection;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

/**

 * 爬取页面线程

 * @author InJavaWetrust

 *

 */

public class MinerHtmlThread extends Thread {

	private static final Log LOG = LogFactory.getLog(MinerHtmlThread.class);

	private MinerConfig config = null;

	public MinerHtmlThread(MinerConfig config) {

		this.config = config;

	}

	@Override

	public void run() {

		while (!MinerMonitorThread.done) {

			minerHtml();

		}

	}

	public synchronized void minerHtml() {

		MinerUrl minerUrl = MinerQueue.unVisitedPoll(); // 待访问出队列。

		try {

			//判断当前页面爬取深度

			if(null == minerUrl || MinerUtil.isBlank(minerUrl.getUrl()) || minerUrl.getDepth() > config.getMaxDepth()) {

				return;

			}

			//判断爬取页面URL是否包含http

			if("http".contains(minerUrl.getUrl())) {

				LOG.info("MinerHtmlThread当前爬取URL[" + minerUrl.getUrl() + "]没有http");

				return;

			}

			LOG.info("MinerHtmlThread当前爬取页面[" + minerUrl.getUrl() + "]爬取深度[" + minerUrl.getDepth() + "] 当前线程 [" + Thread.currentThread().getName() + "]");

			Connection conn = Jsoup.connect(minerUrl.getUrl());

			conn.header("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13");//配置模拟浏览器

			Document doc = conn.get();

			String page = doc.html();

			Html html = new Html();

			html.setUrl(minerUrl.getUrl());

			html.setHtml(page);

			html.setDepth(minerUrl.getDepth());

			// 添加到存储队列

			MinerQueue.addStore(html);

			// 已经爬取的页面 添加到等待提取URL的分析页面队列

			MinerQueue.addWaitingMisering(html); 

		} catch(Exception e) {

			LOG.info("MinerHtmlThread爬取页面失败 URL [" + minerUrl.getUrl() + "]");

			LOG.info("MinerHtmlThreadError info [" + e.getMessage() + "]");

		}

	}

}

返回列表