NGUSERNAME [a-zA-Z\.\@\-\+_%]+

NGUSER %{NGUSERNAME}

NGINXACCESS %{IPORHOST:server_name} %{IPORHOST:server_ip} %{IPORHOST:client_ip} %{NUMBER:client_port} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response}  (?:%{NUMBER:body_bytes_sent}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent} %{NUMBER:request_time} (?:%{NUMBER:upstream_response_time}|-)

SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:

SYSLOGPAMSESSION %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})?

CRON_ACTION [A-Z ]+

CRONLOG %{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\)

SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}

# IETF  syslog() format (see http://www.rfc-editor.org/info/rfc5424)

SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}>

SYSLOG5424SD \[%{DATA}\]+

SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(?:%{WORD:syslog5424_app}|-) +(?:%{WORD:syslog5424_proc}|-) +(?:%{WORD:syslog5424_msgid}|-) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)

SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}

input {

  beats {

    port =>

    host => "10.80.2.181"

  }

}

filter {

  if [type] == "51-nginxaccesslog" {

  grok {

    patterns_dir => ["./patterns"]

    match => { "message" => "%{IPORHOST:server_name} %{IPORHOST:server_ip} %{IPORHOST:clientip} %{NUMBER:clientport} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:status} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{NUMBER:request_time:float} (?:%{NUMBER:upstream_time:float}|-)" }

    remove_field => ["message"]

  }

    date {

      match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z"]

    }

  } else if [type] == "51-nginxerrorlog" {

  grok {

    patterns_dir => ["./patterns"]

    match => { "message" => "%{DATESTAMP} %{SYSLOG5424SD:nginx_error_level} %{GREEDYDATA:nginx_error_msg}"}

    remove_field => ["message"]

  }

    date {

      match => [ "timestamp", "YYYY/MMM/dd HH:mm:ss"]

    }

  } else if [type] == "51-phperrorlog" {

  grok {

    patterns_dir => ["./patterns"]

    match => { "message" => "%{SYSLOG5424SD} (?:%{DATA:php_error_level}\:) %{GREEDYDATA:error_msg}" }

    remove_field => ["message"]

  }

    date {

      match => [ "timestamp", "dd-MMM-YYYY HH:mm:ss Z"]

    }

  }

}

output {

  if "_grokparsefailure" in [tags] {

    file { path => "/var/log/logstash/grokparsefailure-%{[type]}-%{+YYYY.MM.dd}.log" }

  }

  elasticsearch {

    hosts => ["10.80.2.83:9200","10.80.2.84:9200"]

    sniffing => true

    manage_template => false

    template_overwrite => true

    index => "%{[type]}-%{+YYYY.MM.dd}"

    document_type => "%{[type]}"

  }

}

#!/bin/sh

# Init script for logstash

# Maintained by Elasticsearch

# Generated by pleaserun.

# Implemented based on LSB Core 3.1:

#   * Sections: 20.2, 20.3

#

### BEGIN INIT INFO

# Provides:          logstash

# Required-Start:    $remote_fs $syslog

# Required-Stop:     $remote_fs $syslog

# Default-Start:

# Default-Stop:

# Short-Description:

# Description:        Starts Logstash as a daemon.

### END INIT INFO

PATH=/sbin:/usr/sbin:/bin:/usr/bin

export PATH

if [ `id -u` -ne  ]; then

   echo "You need root privileges to run this script"

   exit

fi

name=logstash

pidfile="/usr/local/logstash/$name.pid"

LS_USER=nobody

LS_GROUP=nobody

LS_HOME=/usr/local/logstash

#LS_HOME=/home/logstash

LS_HEAP_SIZE="12g"

LS_LOG_DIR=/data/logstash/log

LS_LOG_FILE="${LS_LOG_DIR}/$name.log"

LS_CONF_DIR=/usr/local/logstash/conf.d

LS_OPEN_FILES=

LS_NICE=-

LS_THREADS=

KILL_ON_STOP_TIMEOUT=${KILL_ON_STOP_TIMEOUT-} #default value is zero to this variable but could be updated by user request

LS_OPTS=""

[ -r /etc/default/$name ] && . /etc/default/$name

[ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name

program=/usr/local/logstash/bin/logstash

args="agent -f ${LS_CONF_DIR} -w ${LS_THREADS} -l ${LS_LOG_FILE} ${LS_OPTS}"

quiet() {

  "$@" > /dev/null >&

  return $?

}

start() {

  LS_JAVA_OPTS="${LS_JAVA_OPTS} -Djava.io.tmpdir=${LS_HOME}"

  HOME=${LS_HOME}

  export PATH HOME LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGING LS_GC_LOG_FILE

  # chown doesn't grab the suplimental groups when setting the user:group - so we have to do it for it.

  # Boy, I hope we're root here.

  SGROUPS=$(id -Gn "$LS_USER" | tr " " "," | sed 's/,$//'; echo '')

  if [ ! -z $SGROUPS ]

  then

    EXTRA_GROUPS="--groups $SGROUPS"

  fi

  # set ulimit as (root, presumably) first, before we drop privileges

  ulimit -n ${LS_OPEN_FILES}

  # Run the program!

  nice -n ${LS_NICE} chroot --userspec $LS_USER:$LS_GROUP $EXTRA_GROUPS / sh -c "

    cd $LS_HOME

    ulimit -n ${LS_OPEN_FILES}

    exec \"$program\" $args

  " > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &

  # Generate the pidfile from here. If we instead made the forked process

  # generate it there will be a race condition between the pidfile writing

  # and a process possibly asking for status.

  echo $! > $pidfile

  echo "$name started."

  return

}

stop() {

  # Try a few times to kill TERM the program

  if status ; then

    pid=`cat "$pidfile"`

    echo "Killing $name (pid $pid) with SIGTERM"

    kill -TERM $pid

    # Wait for it to exit.

    for i in          ; do

      echo "Waiting $name (pid $pid) to die..."

      status || break

      sleep

    done

    if status ; then

      if [ $KILL_ON_STOP_TIMEOUT -eq  ] ; then

        echo "Timeout reached. Killing $name (pid $pid) with SIGKILL. This may result in data loss."

        kill -KILL $pid

        echo "$name killed with SIGKILL."

      else

        echo "$name stop failed; still running."

        return  # stop timed out and not forced

      fi

    else

      echo "$name stopped."

    fi

  fi

}

status() {

  if [ -f "$pidfile" ] ; then

    pid=`cat "$pidfile"`

    if kill - $pid > /dev/null > /dev/null ; then

      # process by this pid is running.

      # It may not be our pid, but that's what you get with just pidfiles.

      # TODO(sissel): Check if this process seems to be the same as the one we

      # expect. It'd be nice to use flock here, but flock uses fork, not exec,

      # so it makes it quite awkward to use in this case.

      return

    else

      return  # program is dead but pid file exists

    fi

  else

    return  # program is not running

  fi

}

reload() {

  if status ; then

    kill -HUP `cat "$pidfile"`

  fi

}

force_stop() {

  if status ; then

    stop

    status && kill -KILL `cat "$pidfile"`

  fi

}

configtest() {

  # Check if a config file exists

  if [ ! "$(ls -A ${LS_CONF_DIR}/* 2> /dev/null)" ]; then

    echo "There aren't any configuration files in ${LS_CONF_DIR}"

    return

  fi

  HOME=${LS_HOME}

  export PATH HOME

  test_args="--configtest -f ${LS_CONF_DIR} ${LS_OPTS}"

  $program ${test_args}

  [ $? -eq  ] && return

  # Program not configured

  return

}

case "$1" in

  start)

    status

    code=$?

    if [ $code -eq  ]; then

      echo "$name is already running"

    else

      start

      code=$?

    fi

    exit $code

    ;;

  stop) stop ;;

  force-stop) force_stop ;;

  status)

    status

    code=$?

    if [ $code -eq  ] ; then

      echo "$name is running"

    else

      echo "$name is not running"

    fi

    exit $code

    ;;

  reload) reload ;;

  restart)

    quiet configtest

    RET=$?

    if [ ${RET} -ne  ]; then

      echo "Configuration error. Not restarting. Re-run with configtest parameter for details"

      exit ${RET}

    fi

    stop && start

    ;;

  configtest)

    configtest

    exit $?

    ;;

  *)

    echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|reload|restart|configtest}" >&

    exit

  ;;

esac

exit $?