package WebCrawler;

import java.io.File;

import java.util.ArrayList;

import java.util.LinkedList;

import java.util.Queue;

public class MyCrawler {

    private static final String SAVEPATH = "C:"+File.separator+"downloadURL";

    public void crawl(ArrayList<URL> urls, int depth) {

        //初始化队列

        Queue<URL> q = new LinkedList<URL>();

        ArrayList<URL> visited = new ArrayList<URL>();

        q.addAll(urls);

        while (!q.isEmpty()) {

            URL head = q.poll();    //出列

            if(head.getDepth() > depth){

                break;

            }

            visited.add(head);

            String page = HtmlParserTool.getPage(head.toString());

            String charset = HtmlParserTool.getCharset(page);

            String urlFullPath = SAVEPATH+File.separator+head.toString().replaceAll("[?:<>*|////]","_")+".html";

            HtmlParserTool.writeToDisk(urlFullPath, page, charset);    //保存到磁盘

            ArrayList<String> toVisit = HtmlParserTool.extractLinks(page);

            for (String s : toVisit) {

                if (!visited.contains(s)) {

                    //visited.add(s);

                    q.add(new URL(s, head.getDepth()+1));

                }

            }

        }

    }

    public static void main(String[] args) throws Exception {

        ArrayList<URL> urls = new ArrayList<URL>();

        urls.add(new URL("http://www.baidu.com"));

        new MyCrawler().crawl(urls,1);

    }

}

package WebCrawler;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileOutputStream;

import java.io.InputStreamReader;

import java.io.OutputStream;

import java.util.ArrayList;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;

import org.htmlparser.Parser;

import org.htmlparser.Tag;

import org.htmlparser.tags.LinkTag;

import org.htmlparser.visitors.NodeVisitor;

public class HtmlParserTool {

    //判断字符串是否是一个网址

    private static boolean isValidUrl(String url) {

        if (url.startsWith("http") | url.startsWith("https")) {

            return true;

        } else {

            return false;

        }

    }

    //获取网页包含的超链接

    public static ArrayList<String> extractLinks(String content){

        ArrayList<String> links = new ArrayList<String>();

        Parser parser = null;

        NodeVisitor visitor = null;

        try {

            parser = new Parser(content);

            visitor = new NodeVisitor() {

                @Override

                public void visitTag(Tag tag) {

                    if(tag instanceof LinkTag) {

                        LinkTag link = (LinkTag)tag;

                        String linkString = link.getLink();

                        if(isValidUrl(linkString) && !links.contains(linkString)) {

                            links.add(linkString);

                        }

                    }

                }

            };

            parser.visitAllNodesWith(visitor);

        } catch (Exception e) {

            e.printStackTrace();

        }

        return links;

    }

    //获取字符集

    public static String getCharset(String content) {

        int startIdx = content.indexOf("charset");

        int endIdx = content.indexOf("\"", startIdx+9);

        String charset = content.substring(startIdx+9, endIdx);

        return charset;

    }

    //获取网页内容

    public static String getPage(String url) {

        CloseableHttpClient client = HttpClients.createDefault();

        HttpGet request = new HttpGet(url);

        String content="";

        try {

            CloseableHttpResponse response = client.execute(request);

            //System.out.println("Response Code: " + response.getStatusLine().getStatusCode());

            BufferedReader rd = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));

            String line = "";

            while ((line = rd.readLine()) != null) {

                content = content + line + "\n";

            }

            response.close();

            client.close();

            String charset = getCharset(content);

            if(charset != null) {

                content = new String(content.getBytes(),charset);

            }

        } catch (Exception e) {

            e.printStackTrace();

        }

        return content;

    }

    //将网页内容写至磁盘

    public static void writeToDisk(String path, String content, String charset){

        try {

            File file = new File(path);

            OutputStream o = new FileOutputStream(file);

            o.write(content.getBytes(charset));

            o.close();

        } catch (Exception e) {

            e.printStackTrace();

        }

    }

}

package WebCrawler;

public class URL {

    private String url;

    private int depth;

    public URL(String url) {

        this.url = url;

        this.depth = 1;

    }

    public URL(String url, int depth) {

        this.url = url;

        this.depth = depth;

    }

    public String toString() {

        return this.url;

    }

    public String getUrl() {

        return url;

    }

    public void setUrl(String url) {

        this.url = url;

    }

    public int getDepth() {

        return depth;

    }

    public void setDepth(int depth) {

        this.depth = depth;

    }

}