新上线了一个自己的小服务
每天都有很多来扫描服务器的
很烦人
就想着吧这些IP地址都记下来
然后去读apache的access_log
太多了实在麻烦
写了一个小工具类。写小记一下吧
需要说一下
1 IPAddress.class中读取IP地址的归属地我找了4个API（有些是抓包来的：）
其中IP138返回的相对最好，不过需要付费
IP126方式暂时不受限制，以后不知道
2 hosts.deny是我自己弄的黑名单，主要是放在http配置中，用来将这些IP直接403干掉
3 Apache/2.4.6 (CentOS)，如果你的日志格式不一样就自己调整下函数吧
附代码：
HttpdLogTest.class
package com.sys.util;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import net.sf.json.JSONObject;
/**
* 用于分析Apache Httpd的访问日志，主要是统计IP访问次数。
* 其中有白名单和黑名单的过滤功能。白名单在构造函数初始化时自己定义，黑名单是读了我的Apache hosts.deny文件
* 如果读取到非名单中的新IP地址，调用了外部的IPAddress.get来获取IP的归属地
* 提供了accessLog用于分析成功访问的日志，errorLog是错误日志，printlogByKeyword是按关键字查询日志
*
* @author qing 2021-03-20
*
*/
public class HttpdLogTest {
//黑名单文件本地存储位置
public static String hostsDenyPath = "D:\\hosts.deny";
//访问日志本地路径
public static String access_log_path = "C:\\Users\\Administrator\\AppData\\Roaming\\Private Shell\\Temp\\access_log";
//错误日志本地路径
public static String error_log_path = "C:\\Users\\Administrator\\AppData\\Roaming\\Private Shell\\Temp\\error_log";
//是否调用查询IP地址
public static boolean isfondIpAddress = true;
//是否过滤意见被403的请求
public static boolean filter403 = false;
//白名单
Set<String> whiteList = null;
//黑名单
Set<String> blacklist = null;
public HttpdLogTest() {
// 初始化时加载白名单和黑名单
initWhiteList();
initBlackList();
}
public static void main(String[] args) {
HttpdLogTest httpLogTest = new HttpdLogTest();
//System.out.println("===============accessLog===============");
httpLogTest.accessLog();
//System.out.println("===============accessLog end===============");
//System.out.println("===============errorLog===============");
//httpLogTest.errorLog();
//System.out.println("===============errorLog end===============");
//httpLogTest.printlogByKeyword(access_log_path,"1.1.1.1");
}
//加载白名单
void initWhiteList(){
whiteList = new HashSet<>();
whiteList.add("::1");//内部IP
}
//读取本地的黑名单
void initBlackList(){
blacklist = new HashSet<>();
try(FileInputStream fin = new FileInputStream(hostsDenyPath);
InputStreamReader reader = new InputStreamReader(fin);
BufferedReader buffReader = new BufferedReader(reader);) {
String strTmp = "";
while ((strTmp = buffReader.readLine()) != null) {
//本地黑名单hosts.Deny格式 8.8.8.8 -
if(strTmp.endsWith("-")){
blacklist.add(strTmp.replace("-", "").trim());
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 获取IP地址之前判断是否需要联网获取
* @param ip
* @return
*/
String getIpAddress(String ip){
if(isfondIpAddress){
return IPAddress.get(ip);
}
return "false";
}
//解析错误日志
public void errorLog() {
String splitChar = "\\[client ";// 分割关键字
try (FileInputStream fin = new FileInputStream(error_log_path);
InputStreamReader reader = new InputStreamReader(fin);
BufferedReader buffReader = new BufferedReader(reader);){
String strTmp = "";
HashMap<String, Integer> res = new HashMap<String, Integer>();
Integer idx = 1;
//采用每次读一行的方式，因为日志文件可能比较大。一次读完会卡顿
while ((strTmp = buffReader.readLine()) != null) {
String arr[] = strTmp.split(splitChar);
if (arr.length > 1) {
String ip = arr[1].trim().substring(0,arr[1].trim().indexOf(":"));
if (!res.containsKey(ip)) {
res.put(ip, 1);
} else {
res.put(ip, res.get(ip) + 1);
}
}
idx++;
}
System.out.println("读取完毕：" + idx + "行");
sortMapAndPrint(res);
} catch (IOException e) {
e.printStackTrace();
}
}
//解析访问记录
public void accessLog() {
String splitChar = "- -";// 分割关键字
try (FileInputStream fin = new FileInputStream(access_log_path);
InputStreamReader reader = new InputStreamReader(fin);
BufferedReader buffReader = new BufferedReader(reader);){
String strTmp = null;
HashMap<String, Integer> res = new HashMap<String, Integer>();
Integer idx = 0;
while ((strTmp = buffReader.readLine()) != null) {
idx++;
//过滤掉已经被拦截的403请求信息
if(filter403 && strTmp.indexOf(" 403 ")>0){continue;}
//只查看访问成功的
//if(strTmp.indexOf(" 200 ")<0){continue;}
String arr[] = strTmp.split(splitChar);
if (arr.length > 0) {
String ip = arr[0].trim();
if (!res.containsKey(ip)) {
res.put(ip, 1);
} else {
res.put(ip, res.get(ip) + 1);
}
}
}
System.out.println("读取完毕：" + idx);
sortMapAndPrint(res);
} catch (IOException e) {
e.printStackTrace();
}
}
//Map 按value倒序 并且打印
void sortMapAndPrint(HashMap<String, Integer> map){
List<Entry<String, Integer>> list = new ArrayList<Entry<String, Integer>>(map.entrySet());
// Map按value倒序
Collections.sort(list,new Comparator<Map.Entry<String, Integer>>() {
public int compare(Map.Entry<String, Integer> o1,
Map.Entry<String, Integer> o2) {
return (o2.getValue() - o1.getValue());
}
});
//安全访问的IP
List<JSONObject> safeList = new ArrayList<JSONObject>();
//新访问的IP
List<JSONObject> newList = new ArrayList<JSONObject>();
//已经被拦截的IP
List<JSONObject> filterList = new ArrayList<JSONObject>();
for (int i = 0; i < list.size(); i++) {
String ip = list.get(i).getKey();
Integer num = list.get(i).getValue();
JSONObject obj = new JSONObject();
obj.put("ip", ip);
obj.put("num", num);
if(whiteList.contains(ip)){
safeList.add(obj);
}else if(blacklist.contains(ip)){
filterList.add(obj);
}else{
newList.add(obj);
}
}
//IP地址只有新发现的才去获取，这里没采用异步线程访问，可能会出现卡顿的现象，不过用来测试足够了
for (int i = 0; i < newList.size(); i++) {
JSONObject obj = newList.get(i);
System.out.println("[NEW]\t" + obj.get("ip") + "\t" +obj.get("num")+"\t"+getIpAddress(obj.get("ip").toString()) );
}
for (int i = 0; i < safeList.size(); i++) {
JSONObject obj = safeList.get(i);
System.out.println("[安 全]\t" + obj.get("ip") + "\t" +obj.get("num") );
}
for (int i = 0; i < filterList.size(); i++) {
JSONObject obj = filterList.get(i);
System.out.println("[拦 截]\t" + obj.get("ip") + "\t" +obj.get("num") );
}
}
/**
* 按关键字查询日志后打印日志
* @param ipaddress
*/
public void printlogByKeyword(String logPath,String keyword){
//Java7的try-with-resources可以优雅关闭文件，异常时自动关闭文件；详细解读https://*.com/a/12665271
try(FileInputStream fin = new FileInputStream(logPath);
InputStreamReader reader = new InputStreamReader(fin);
BufferedReader buffReader = new BufferedReader(reader);) {
String strTmp = "";
Integer idx = 1;
while ((strTmp = buffReader.readLine()) != null) {
if(strTmp.indexOf(keyword)>=0){
System.out.println((idx++) + " : "+strTmp);
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
IPAddress.class
package com.sys.util;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
public class IPAddress {
//适用的IPapi类型
/**
* IP138（付费）：https://api.ip138.com/ip/?ip=1.1.1.1
* IPIP(请求过多会错误)：http://freeapi.ipip.net/1.1.1.1
* IP126（敞开用）：http://ip.ws.126.net/ipquery?ip=1.1.1.1
* IP5ME(感觉不太可靠) https://q.ip5.me/?s=123.58.4.233
*
*/
public enum ApiType
{
IP138,IP126,IPIP,IP5ME;
}
public static void main(String[] args) {
System.out.println(IPAddress.get("123.58.4.233",ApiType.IP126));
System.out.println(IPAddress.get("123.58.4.233",ApiType.IPIP));
System.out.println(IPAddress.get("123.58.4.233",ApiType.IP138));
System.out.println(IPAddress.get("123.58.4.233",ApiType.IP5ME));
}
/**
* 读取IP地址的API，默认适用IP126接口
* @param ip
* @return
*/
public static String get(String ip){
return get(ip,ApiType.IP126);
}
/**
*
* @param apiType
* @param ip
* @return
*/
public static String get(String ip,ApiType apiType){
switch (apiType) {
case IP126:return getIP126(ip);
case IPIP: return getIPIP(ip);
case IP138:return getIP138(ip);
case IP5ME:return getIP5ME(ip);
}
return "NOT API";
}
/**
* https://q.ip5.me/?s=123.58.4.233
* 这个估计改版就不能用了
*/
public static String getIP5ME(String ip) {
HttpURLConnection con = null;
BufferedReader buffer = null;
try {
URL url = new URL("https://q.ip5.me/?s=" + ip);
// 得到连接对象
con = (HttpURLConnection) url.openConnection();
// 设置请求类型
con.setRequestMethod("GET");
// 设置请求需要返回的数据类型和字符集类型
con.setRequestProperty("content-type", "application/x-www-form-urlencoded; charset=gb2312");
con.setRequestProperty("user-agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36");
// 允许写出
con.setDoOutput(true);
// 允许读入
con.setDoInput(true);
// 不使用缓存
con.setUseCaches(false);
// 得到响应码
int responseCode = con.getResponseCode();
//System.out.println(responseCode);
if (responseCode == HttpURLConnection.HTTP_OK) {
// 得到响应流
InputStream inputStream = con.getInputStream();
String line;
buffer = new BufferedReader(new InputStreamReader(inputStream,"gb2312"));
String startKey = "<td><div id=\"ip_pos\" style=\"color:#FF0000\">";
String endKey = "</div></td>";
while ((line = buffer.readLine()) != null) {
if(line.startsWith(startKey)&&line.endsWith(endKey)){
return line.substring(startKey.length(),line.indexOf(endKey));
}
//resultBuffer.append(line);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return "ERROR";
}
/**
* http://ip.ws.126.net/ipquery?ip=1.1.1.1
*
*/
public static String getIP126(String ip) {
HttpURLConnection con = null;
BufferedReader buffer = null;
StringBuffer resultBuffer = null;
try {
URL url = new URL("http://ip.ws.126.net/ipquery?ip=" + ip);
// 得到连接对象
con = (HttpURLConnection) url.openConnection();
// 设置请求类型
con.setRequestMethod("GET");
// 设置请求需要返回的数据类型和字符集类型
con.setRequestProperty("Host", "ip.ws.126.net");
con.setRequestProperty("accept", "*/*");
con.setRequestProperty("connection", "Keep-Alive");
con.setRequestProperty(
"user-agent",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36");
con.setRequestProperty("Content-Type",
"application/json;charset=utf-8");
// 允许写出
con.setDoOutput(true);
// 允许读入
con.setDoInput(true);
// 不使用缓存
con.setUseCaches(false);
// 得到响应码
int responseCode = con.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) {
// 得到响应流
InputStream inputStream = con.getInputStream();
// 将响应流转换成字符串
resultBuffer = new StringBuffer();
String line;
buffer = new BufferedReader(new InputStreamReader(inputStream,"GBK"));
while ((line = buffer.readLine()) != null) {
resultBuffer.append(line);
}
if(resultBuffer.toString().indexOf("{")>=0){
JSONObject addArr = JSONObject.fromObject(resultBuffer.toString().substring(resultBuffer.toString().indexOf("{")));
return addArr.getString("province")+","+addArr.getString("city");
}else{
return "NOT";
}
}
} catch (Exception e) {
e.printStackTrace();
}
return "ERROR";
}
/**
* http://freeapi.ipip.net/1.1.1.1
*
*/
public static String getIPIP(String ip) {
HttpURLConnection con = null;
BufferedReader buffer = null;
StringBuffer resultBuffer = null;
try {
URL url = new URL("http://freeapi.ipip.net/" + ip);
// 得到连接对象
con = (HttpURLConnection) url.openConnection();
// 设置请求类型
con.setRequestMethod("GET");
// 设置请求需要返回的数据类型和字符集类型
con.setRequestProperty("Host", "freeapi.ipip.net");
con.setRequestProperty("accept", "*/*");
con.setRequestProperty("connection", "Keep-Alive");
con.setRequestProperty(
"user-agent",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36");
con.setRequestProperty("Content-Type",
"application/json;charset=utf-8");
// 允许写出
con.setDoOutput(true);
// 允许读入
con.setDoInput(true);
// 不使用缓存
con.setUseCaches(false);
// 得到响应码
int responseCode = con.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) {
// 得到响应流
InputStream inputStream = con.getInputStream();
// 将响应流转换成字符串
resultBuffer = new StringBuffer();
String line;
buffer = new BufferedReader(new InputStreamReader(inputStream,
"UTF-8"));
while ((line = buffer.readLine()) != null) {
resultBuffer.append(line);
}
JSONArray addArr = JSONArray
.fromObject(resultBuffer.toString());
String address = "";
for (int i = 0; i < addArr.size(); i++) {
address += addArr.get(i).toString();
}
return address;
}
} catch (Exception e) {
e.printStackTrace();
}
return "ERROR";
}
/**
* https://api.ip138.com/ip/?ip=1.1.1.1
*
*/
public static String getIP138(String ip) {
String urlString="https://api.ip138.com/ip/?ip="+ip+"&datatype=jsonp";
String token="去IP138申请token";
try {
URL url = new URL(urlString);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(5 * 1000);
conn.setReadTimeout(5 * 1000);
conn.setDoInput(true);
conn.setDoOutput(true);
conn.setUseCaches(false);
conn.setInstanceFollowRedirects(false);
conn.setRequestMethod("GET");
conn.setRequestProperty("token",token);
int responseCode = conn.getResponseCode();
if (responseCode == 200) {
StringBuilder builder = new StringBuilder();
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(),"utf-8"));
for (String s = br.readLine(); s != null; s = br.readLine()) {
builder.append(s);
}
br.close();
return ((JSONArray)(JSONObject.fromObject(builder.toString()).get("data"))).join(",").replace("\"", "");
}else{
return "error:"+responseCode;
}
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
}
hosts.deny
# HTTP Server Blacklist
# qing
## 2021-03-22
193.112.94.225 -
123.207.151.66 -
195.3.147.56 -
202.164.138.92 -
124.74.40.6 -
202.164.138.92 -
123.160.221.52 -
223.104.96.188 -
107.161.50.66 -
172.104.242.173 -
197.232.1.182 -
194.183.10.237 -
192.241.226.191 -
185.36.81.52 -输出效果：