#Software: Microsoft Internet Information Services 6.0

#Version: 1.0

#Date: -- ::

#Fields: date time s-sitename s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) sc-status sc-substatus sc-win32-status

-- :: W3SVC1 :da8:::: GET /skin6/film_sort.asp id=  - :da8::f07:ac50:d2b:f22d:5dec Mozilla/5.0+(Windows+NT+6.1;+Trident/7.0;+rv:11.0)+like+Gecko

-- :: W3SVC1 :da8:::: GET /news.asp -  - :da8::f07:ac50:d2b:f22d:5dec Mozilla/5.0+(Windows+NT+6.1;+Trident/7.0;+rv:11.0)+like+Gecko

-- :: W3SVC1 :da8:::: GET /UploadFile/.jpg -  - :da8::f07:ac50:d2b:f22d:5dec Mozilla/5.0+(Windows+NT+6.1;+Trident/7.0;+rv:11.0)+like+Gecko   

CREATE EXTERNAL  TABLE IF NOT EXISTS exmovielog

(

    log_date            TIMESTAMP,

    s_sitename            STRING,

    s_ip                STRING,

    cs_method            STRING,

    cs_uri_stem            STRING,

    cs_uri_query            STRING,

    s_port                INT,

    cs_username            STRING,

    c_ip                STRING,

    user_agen            STRING,

    sc_status            INT,

    sc_substatus            INT,

    sc_win32_status            INT

)

ROW FORMAT DELIMITED FIELDS TERMINATED BY '`'

LOCATION '/movielog';

agent1.sources = source1

agent1.channels = channel1

agent1.sinks = sink1

# Each channel's type is defined.

agent1.channels.channel1.type = file

agent1.channels.channel1.checkpointDir = /home/hadoop_admin/flumeTemp/fchannel/spool/checkpoint

agent1.channels.channel1.dataDirs = /home/hadoop_admin/flumeTemp/fchannel/spool/data

agent1.channels.channel1.capacity = 

# For each one of the sources, the type is defined

agent1.sources.source1.type = spooldir

agent1.sources.source1.inputCharset = GBK

agent1.sources.source1.spoolDir =/home/hadoop_admin/movielog

agent1.sources.source1.fileHeader = true

agent1.sources.source1.deletePolicy = immediate

agent1.sources.source1.batchSize =

agent1.sources.source1.channels = channel1

# remove the line that starts with '#'

agent1.sources.source1.interceptors = i1  search-replace1 search-replace2 search-replace3

agent1.sources.source1.interceptors.i1.type = regex_filter

agent1.sources.source1.interceptors.i1.regex = ^[^#].*$

# the default value of this configuration is flase

# agent1.sources.source1.interceptors.i1.excludeEvents = true

# agent1.sources.source1.interceptors.i1.regex = ^#

# connect the date and time to be a timestamp

agent1.sources.source1.interceptors.search-replace1.type = search_replace

agent1.sources.source1.interceptors.search-replace1.searchPattern = (\\d\\d\\d\\d-\\d\\d-\\d\\d)\\s(\\d\\d:\\d\\d:\\d\\d)

agent1.sources.source1.interceptors.search-replace1.replaceString = $1T$

# change the split char

agent1.sources.source1.interceptors.search-replace2.type = search_replace

agent1.sources.source1.interceptors.search-replace2.searchPattern = \\s

agent1.sources.source1.interceptors.search-replace2.replaceString = `

agent1.sources.source1.interceptors.search-replace3.type = search_replace

agent1.sources.source1.interceptors.search-replace3.searchPattern = (\\d\\d\\d\\d-\\d\\d-\\d\\d)T(\\d\\d:\\d\\d:\\d\\d)

agent1.sources.source1.interceptors.search-replace3.replaceString = $ $

# Each sink's type must be defined

agent1.sinks.sink1.type = hdfs

agent1.sinks.sink1.channel = channel1

agent1.sinks.sink1.hdfs.path = hdfs://master:9000/movielog

agent1.sinks.sink1.hdfs.writeFormat = Text

agent1.sinks.sink1.hdfs.fileType = DataStream

agent1.sinks.sink1.hdfs.rollInterval =

agent1.sinks.sink1.hdfs.idleTimeout =

agent1.sinks.sink1.hdfs.rollCount =

agent1.sinks.sink1.hdfs.rollSize =

agent1.sinks.sink1.hdfs.batchSize =

agent1.sinks.sink1.hdfs.callTimeout =