最近GFW为了刷存在感，搞得大家是头晕眼花，修改hosts 几乎成了每日必备工作。
索性写了一个小程序，给办公室的同事们分享，其中有个内容 就是抓取网络上的hosts，废了一些周折。
我是在一个博客上抓取的。但是这位朋友的博客应该是在做防盗链，但他的方式比较简单就是5位数的一个整形随机数。这里折腾一下就ok了。
要命的是他这个链接的流类型 居然是gzip。这个郁闷好久，一直以为是编码格式导致解析不出来结果，后来发现是gzip搞的。
主要的一段代码做个记录吧。
/**
* 网络工具类 用于抓取http://serve.netsh.org上的hosts数据
*
* @author tone
*/
public class NetUtil { private final static String ENCODING = "UTF-8";
private final static String GZIPCODING = "gzip";
private final static String HOST = "http://serve.netsh.org/pub/hosts.php";
private final static String COOKIE = "hostspasscode=%s; Hm_lvt_e26a7cd6079c926259ded8f19369bf0b=1421846509,1421846927,1421847015,1421849633; Hm_lpvt_e26a7cd6079c926259ded8f19369bf0b=1421849633";
private final static String OFF = "off";
private final static String ON = "on";
private final static int RANDOM = 100000;
private static String hostspasscode = null;
private static NetUtil instance; public static NetUtil getInstance() {
if (instance == null) {
instance = new NetUtil();
}
return instance;
} private NetUtil() {
hostspasscode = createRandomCookies();
} /**
* 获取html内容
*
* @param gs
* @param wk
* @param twttr
* @param fb
* @param flkr
* @param dpbx
* @param odrvB
* @param yt
* @param nohl
* @return
*/
public String getHtmlInfo(boolean gs, boolean wk, boolean twttr, boolean fb,
boolean flkr, boolean dpbx, boolean odrv,
boolean yt, boolean nohl) throws Exception {
HttpURLConnection conn = null; String result = ""; //String cookie = "hostspasscode="+hostspasscode+"; Hm_lvt_e26a7cd6079c926259ded8f19369bf0b=1421846509,1421846927,1421847015,1421849633; Hm_lpvt_e26a7cd6079c926259ded8f19369bf0b=1421849633";
String cookie = String.format(COOKIE, hostspasscode); //URL url = new URL("http://serve.netsh.org/pub/hosts.php?passcode=13008&gs=on&wk=on&twttr=on&fb=on&flkr=on&dpbx=on&odrv=on&yt=on&nolh=on");
URL url = new URL(createUrl(hostspasscode, gs, wk, twttr, fb, flkr, dpbx, odrv, yt, nohl));
//System.out.println(cookie);
// System.out.println(url.toString()); conn = (HttpURLConnection) url.openConnection(); conn.setConnectTimeout(5 * 1000);
conn.setDoOutput(true);
//get方式提交
conn.setRequestMethod("GET");
//凭借请求头文件
conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
conn.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("Cookie", cookie);
conn.setRequestProperty("Host", "serve.netsh.org");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:35.0) Gecko/20100101 Firefox/35.0"); // conn.setRequestProperty("Referer", "http://serve.netsh.org/pub/gethosts.php");
// conn.setRequestProperty("X-Requested-With", "XMLHttpRequest"); conn.connect(); String encoding = conn.getContentEncoding(); result = readStream(conn.getInputStream(), encoding);
//测试进度条显示
// result = readStream(new FileInputStream(new File("/home/tone/Resident.Evil.Damnation.2012.1080p.BluRay.x264.DTS-WiKi.mkv")), "11"); conn.disconnect();
if (nohl) {
result=getLocalHost()+result;
} return result;
} /**
* 读取将InputStream中的字节读以字符的形式取到字符串中，如果encoding是gzip，那么需要先有GZIPInputStream进行封装
*
* @param inputStream InputStream字节流
* @param encoding 编码格式
* @return String类型的形式
* @throws IOException IO异常
*/
private String readStream(InputStream inputStream, String encoding) throws Exception {
StringBuffer buffer = new StringBuffer();
ProgressMonitorInputStream pmis = null; InputStreamReader inputStreamReader = null;
GZIPInputStream gZIPInputStream = null;
if (GZIPCODING.equals(encoding)) {
gZIPInputStream = new GZIPInputStream(inputStream);
inputStreamReader = new InputStreamReader(ProgressUtil.getMonitorInputStream(gZIPInputStream, "获取网络数据"), ENCODING); } else { inputStreamReader = new InputStreamReader(ProgressUtil.getMonitorInputStream(inputStream, "获取网络数据"), ENCODING);
} char[] c = new char[1024]; int lenI;
while ((lenI = inputStreamReader.read(c)) != -1) { buffer.append(new String(c, 0, lenI)); }
if (inputStream != null) {
inputStream.close();
}
if (gZIPInputStream != null) {
gZIPInputStream.close();
}
if (pmis!=null) {
gZIPInputStream.close();
} return buffer.toString(); } /**
* 生成随机Cookies数组
*
* @return 五位随机数字
*/
private String createRandomCookies() { return String.valueOf(Math.random() * RANDOM).substring(0, 5); } /**
* 生成链接字符串
*
* @param hostspasscode
* @param gs
* @param wk
* @param twttr
* @param fb
* @param flkr
* @param dpbx
* @param odrvB
* @param yt
* @param nohl
* @return
*/
private String createUrl(String hostspasscode, boolean gs, boolean wk, boolean twttr, boolean fb,
boolean flkr, boolean dpbx, boolean odrv,
boolean yt, boolean nohl) {
StringBuffer buffer = new StringBuffer();
buffer.append(HOST);
buffer.append("?passcode=" + hostspasscode);
if (gs) {
buffer.append("&gs=" + ON);
} else {
buffer.append("&gs=" + OFF);
}
if (wk) {
buffer.append("&wk=" + ON);
} else {
buffer.append("&wk=" + OFF);
}
if (twttr) {
buffer.append("&twttr=" + ON);
} else {
buffer.append("&twttr=" + OFF);
}
if (fb) {
buffer.append("&fb=" + ON);
} else {
buffer.append("&fb=" + OFF);
}
if (flkr) {
buffer.append("&flkr=" + ON);
} else {
buffer.append("&flkr=" + OFF);
}
if (dpbx) {
buffer.append("&dpbx=" + ON);
} else {
buffer.append("&dpbx=" + OFF);
}
if (odrv) {
buffer.append("&odrv=" + ON);
} else {
buffer.append("&odrv=" + OFF);
}
if (yt) {
buffer.append("&yt=" + ON);
} else {
buffer.append("&yt=" + OFF);
}
if (nohl) {
buffer.append("&nohl=" + ON);
} else {
buffer.append("&nohl=" + OFF);
}
return buffer.toString();
} private String getLocalHost() throws Exception { StringBuffer buffer=new StringBuffer();
String hostName=OSUtil.getInstance().getLocalhostName();
buffer.append("#LOCALHOST begin"+"\n");
buffer.append("127.0.0.1\tlocalhost"+"\n");
if (hostName!=null&&!"".equals(hostName)) {
buffer.append("127.0.1.1\t"+hostName+"\n");
} buffer.append("#LOCALHOST end"+"\n");
return buffer.toString(); } }