filebeat.inputs:

• type: log

  to do

  enabled: true

  to do

  paths:
  • /app/httpd/logs/access_log

filebeat.config.modules:
path: ${path.config}/modules.d/*.yml

reload.enabled: false

setup.template.settings:
index.number_of_shards: 3

setup.kibana:

output.logstash:

to do The Logstash hosts

hosts: [“localhost:5045”]

processors:

• add_host_metadata: ~
• add_cloud_metadata: ~

Sample Logstash configuration for creating a simple

Beats -> Logstash -> Elasticsearch pipeline.

input {
# to do
beats {
port => 5045
}
}
filter {
# to do
grok {
match => { “message” => “%{HTTPD_COMMONLOG}” }
}
}
output {
csv {
path => “/home/output/httpd-outfile.csv”
fields => [“verb”, “httpversion”]
csv_options => {“col_sep” => " "}
}

stdout{
    codec => rubydebug
}

}

#!/usr/bin/env python

-- coding: utf-8 --

import time
import urllib, time, os, base64, json
import re, sys
from lxml import etree

from urllib.request import urlopen
from urllib import request

def getPage(base_url):
try:
req = request.Request(base_url)

    page = urllib.request.urlopen(req)  # 5
    content = page.read().decode("utf-8")  # 2
    re_tag = re.compile('\<[\S\s]+?\>', re.I)
    re_cr = re.compile('\s{1,}', re.I)
    re_script = re.compile('\<script[\S\s]+?\</script\>', re.I)  # Script
    re_style = re.compile('\<style[\S\s]+?\</style\>', re.I)  # style
    content = re_script.sub('', content)  # 去掉SCRIPT  2
    content = re_style.sub('', content)  # 去掉style  2
    content = re_tag.sub('', content)  # 去除所有尖括号内的HTML代码 2
    contentStr = re_cr.sub('', content)  # 去掉换行  2

    # answer two
    writefile("/home/output/crawler_result.csv", contentStr)  # 2

except Exception as e:  # 1
    print("Failed to read from %s." % base_url)
    print(sys.exc_info())
    return False

def writefile(filename, content):
try:
fp = open(filename, ‘a’) # 5
fp.write(content + “\n”) # 5
fp.close() # 5
except:
return False

now = time.strftime(’%Y-%m-%d %X’, time.localtime(time.time()))

try:
# 5
url = ‘http://117.73.9.229:9090/’
getPage(url)

except Exception as e:
info = ‘%s\nError: %s’ % (now, e)
writefile(‘Error.log’, info)
print(info)
time.sleep(1)