crawler4j是用Java实现的开源网络爬虫。提供了简单易用的接口，可以在几分钟内创建一个多线程网络爬虫。下面实例结合jsoup，采集搜狐新闻网（http://news.sohu.com/）新闻标题信息。
所有的过程仅需两步完成：
第一步：建立采集程序核心部分
/**
* @date 2016年8月20日 上午11:52:13
* @version
* @since JDK 1.8
*/
public class MyCrawler extends WebCrawler { //链接地址过滤//
private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|gif|jpg" + "|png|mp3|mp3|zip|gz))$"); @Override
public boolean shouldVisit(Page referringPage, WebURL url) {
String href = url.getURL().toLowerCase();
return !FILTERS.matcher(href).matches() && href.startsWith("http://news.sohu.com/");
} /**
* This function is called when a page is fetched and ready to be processed
* by your program.
*/
@Override
public void visit(Page page) {
String url = page.getWebURL().getURL();
logger.info("URL: " + url); if (page.getParseData() instanceof HtmlParseData) {
HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
String text = htmlParseData.getText();
String html = htmlParseData.getHtml();
Set<WebURL> links = htmlParseData.getOutgoingUrls(); logger.debug("Text length: " + text.length());
logger.debug("Html length: " + html.length());
logger.debug("Number of outgoing links: " + links.size());
logger.info("Title: " + htmlParseData.getTitle()); }
} }
第二步：建立采集程序控制部分
/**
* @date 2016年8月20日 上午11:55:56
* @version
* @since JDK 1.8
*/
public class MyController { /**
* @param args
* @since JDK 1.8
*/
public static void main(String[] args) {
// TODO Auto-generated method stub //本地嵌入式数据库，采用berkeley DB
String crawlStorageFolder = "data/crawl/root";
int numberOfCrawlers = 3; CrawlConfig config = new CrawlConfig();
config.setCrawlStorageFolder(crawlStorageFolder); /*
* Instantiate the controller for this crawl.
*/
PageFetcher pageFetcher = new PageFetcher(config);
RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
CrawlController controller;
try {
controller = new CrawlController(config, pageFetcher, robotstxtServer);
controller.addSeed("http://news.sohu.com/");
controller.start(MyCrawler.class, numberOfCrawlers);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
} } }
采集结果展示：