Mercurial > logstash
changeset 0:df4952a2fb06
initial version
| author | Carl Byington <carl@five-ten-sg.com> |
|---|---|
| date | Fri, 01 Mar 2013 14:58:09 -0800 |
| parents | |
| children | 59fe08a2fcbe |
| files | Makefile apache.pattern logstash.conf logstash.rc logstash.spec sendmail.pattern sources |
| diffstat | 7 files changed, 313 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Makefile Fri Mar 01 14:58:09 2013 -0800 @@ -0,0 +1,5 @@ +all: + [ -d builder ] && rm -rf builder || /bin/true + mkdir builder + rpmbuild --define "_sourcedir $(shell pwd)" --define "_builddir $(shell pwd)/builder" --define "_srcrpmdir $(shell pwd)" --define "_rpmdir $(shell pwd)" --define "_source_filedigest_algorithm md5" --define "_binary_filedigest_algorithm md5" -ba logstash.spec + #rm -rf builder
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/apache.pattern Fri Mar 01 14:58:09 2013 -0800 @@ -0,0 +1,2 @@ +APACHE_LOG_LEVEL (?:emerg|alert|crit|error|warn|notice|info|debug) +APACHE_ERROR_LOG \[%{DATESTAMP_OTHER:timestamp}\] \[%{APACHE_LOG_LEVEL:level}\] %{GREEDYDATA:message}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/logstash.conf Fri Mar 01 14:58:09 2013 -0800 @@ -0,0 +1,68 @@ +# +# the rpm install already set the following acl entries +# +# setfacl -m u:logstash:rx /var/log/httpd +# setfacl -m u:logstash:r /var/log/messages +# setfacl -m u:logstash:r /var/log/maillog +# +# you need to allow user logstash to read any input files specified here + +input { + file { + type => "sendmail" + path => "/var/log/maillog" + } + file { + type => "linux-syslog" + path => "/var/log/messages" + } +# file { +# type => "apache-access" +# path => "/var/log/httpd/*access*_log" +# } +# file { +# type => "apache-error" +# path => "/var/log/httpd/*error*_log" +# } +} + +filter { + grok { + type => "sendmail" + pattern => "%{SENDMAIL}" + patterns_dir => "/var/lib/logstash/data/patterns" + } + + grok { + type => "linux-syslog" + pattern => "%{SYSLOGBASE}" + } +# date { +# # do we need this? the above picks up SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +# type => "linux-syslog" +# timestamp => ["MMM dd HH:mm:ss","MMM d HH:mm:ss"] +# } + + grok { + type => "apache-access" + pattern => "%{COMBINEDAPACHELOG}" + } + date { + type => "apache-access" + # Try to pull the timestamp from the 'timestamp' field (parsed above with + # grok). The apache time format looks like: "18/Aug/2011:05:44:34 -0700" + timestamp => "dd/MMM/yyyy:HH:mm:ss Z" + } + + grok { + type => "apache-error" + patterns_dir => "/var/lib/logstash/data/patterns" + pattern => "%{APACHE_ERROR_LOG}" + } +} + +output { + elasticsearch { + embedded => true + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/logstash.rc Fri Mar 01 14:58:09 2013 -0800 @@ -0,0 +1,96 @@ +#!/bin/bash +# +# /etc/rc.d/init.d/logstash +# +# Starts Logstash as a daemon +# +# chkconfig: 2345 20 80 +# description: Starts Logstash as a daemon +# pidfile: /var/run/logstash.pid + +### BEGIN INIT INFO +# Provides: logstash +# Required-Start: $local_fs $remote_fs +# Required-Stop: $local_fs $remote_fs +# Default-Start: 2 3 4 5 +# Default-Stop: S 0 1 6 +# Short-Description: Logstash +# Description: Starts Logstash as a daemon. +# Modified originally from https://gist.github.com/2228905#file_logstash.sh + +### END INIT INFO + +# Amount of memory for Java +#JAVAMEM=256M + +export HOME=/var/lib/logstash +DESC="Logstash Daemon" +JAVA=$(which java) +CONFIGFILE=/etc/logstash/logstash.conf +LOGFILE=/var/log/logstash/logstash.log +JARNAME=/usr/local/bin/logstash.jar +ARGS="-jar ${JARNAME} agent -vvv --config ${CONFIGFILE} --log ${LOGFILE} -- web --backend elasticsearch://127.0.0.1/?local" +SCRIPTNAME=/etc/rc.d/init.d/logstash +PIDFILE=/var/run/logstash.pid +base=logstash + +# Exit if java is not installed +if [ ! -x "$JAVA" ]; then + echo "Couldn't find $JAVA" + exit 99 +fi + +. /etc/init.d/functions + +# +# Function that starts the daemon/service +# +do_start() { + cd $HOME + pid=$(su logstash -c 'echo -e "'"$JAVA $ARGS"' </dev/null >'"$LOGFILE"' 2>&1 & \n echo \$!" | bash') + echo $pid >$PIDFILE + [ -n "$pid" ] && success $"$base startup" || failure $"$base startup" +} + + +# +# Function that stops the daemon/service +# +do_stop() { + killproc -p $PIDFILE logstash +} + + +case "$1" in + start) + echo -n "Starting $DESC: " + do_start + touch /var/lock/subsys/$base + ;; + stop) + echo -n "Stopping $DESC: " + do_stop + rm /var/lock/subsys/$base 2>/dev/null + rm $PIDFILE 2>/dev/null + ;; + restart) + echo -n "Restarting $DESC: " + do_stop + do_start + ;; + reload) + echo -n "Reloading $DESC: " + pid=$(cat $PIDFILE) + [ -n "$pid" ] && pkill -HUP -u logstash -P $pid + ;; + status) + status -p $PIDFILE + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart}" >&2 + exit 3 + ;; +esac + +echo +exit 0
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/logstash.spec Fri Mar 01 14:58:09 2013 -0800 @@ -0,0 +1,97 @@ +# prevent brp repack jar files +%define __os_install_post %{nil} + +%define _bindir /usr/local/bin + +Summary: A tool for managing your logs +Name: logstash +Version: 1.1.9 +Release: 0 +License: new BSD +Group: Applications/Productivity +URL: http://logstash.net/ +BuildArch: noarch +Source0: https://logstash.objects.dreamhost.com/release/%{name}-%{version}-monolithic.jar +Source1: logstash.rc +Source2: %{name}.conf +Source3: apache.pattern +Source4: sendmail.pattern +Requires: httpd java-1.7.0-openjdk +Requires(pre): /usr/sbin/useradd +Requires(pre): /usr/bin/getent +Requires(postun): /usr/sbin/userdel +Requires(post,preun): /sbin/chkconfig +Requires(post,preun): /sbin/service +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) + + +%description +logstash tool for managing your logs + + +%prep +cp -p %SOURCE0 . +cp -p %SOURCE1 . +cp -p %SOURCE2 . +cp -p %SOURCE3 . +cp -p %SOURCE4 . + + +%build + + +%install +rm -rf $RPM_BUILD_ROOT +mkdir -p $RPM_BUILD_ROOT/var/log/%{name} +install -D -m 640 apache.pattern $RPM_BUILD_ROOT/var/lib/%{name}/data/patterns/apache +install -D -m 640 sendmail.pattern $RPM_BUILD_ROOT/var/lib/%{name}/data/patterns/sendmail +install -D -m 755 %{name}.rc $RPM_BUILD_ROOT/etc/rc.d/init.d/%{name} +install -D -m 750 %{name}-%{version}-monolithic.jar $RPM_BUILD_ROOT/%{_bindir}/%{name}.jar +install -D -m 640 %{name}.conf $RPM_BUILD_ROOT/etc/%{name}/%{name}.conf + + +%pre +/usr/bin/getent passwd %{name} >/dev/null || /usr/sbin/useradd -r -d /var/lib/%{name} -M -c "%{name} pseudo-user" %{name} >/dev/null + + +%post +/sbin/chkconfig --add %{name} +setfacl -m u:logstash:rx /var/log/httpd +setfacl -m u:logstash:r /var/log/messages +setfacl -m u:logstash:r /var/log/maillog + + +%preun +[ $1 = 0 ] && /sbin/service %{name} stop || : +[ $1 = 0 ] && /sbin/chkconfig --del %{name} || : + + +%postun +[ $1 = 0 ] && setfacl -x u:logstash /var/log/httpd || : +[ $1 = 0 ] && setfacl -x u:logstash /var/log/messages || : +[ $1 = 0 ] && setfacl -x u:logstash /var/log/maillog || : +[ $1 = 0 ] && userdel %{name} || : + + +%clean +rm -rf $RPM_BUILD_ROOT + + +%files +%defattr(-,root,root,-) +%attr(0750,%{name},root) %{_bindir}/* +%config(noreplace) %attr(0750,%{name},root) /etc/%{name} +%config(noreplace) %attr(0640,%{name},root) /etc/%{name}/%{name}.conf +/etc/rc.d/init.d/%{name} +%dir %attr(0750,%{name},root) /var/log/%{name} +%dir %attr(0750,%{name},root) /var/lib/%{name} +%dir %attr(0750,%{name},root) /var/lib/%{name}/data +%dir %attr(0750,%{name},root) /var/lib/%{name}/data/patterns +%config(noreplace) %attr(0640,%{name},root) /var/lib/%{name}/data/patterns/sendmail +%config(noreplace) %attr(0640,%{name},root) /var/lib/%{name}/data/patterns/apache + + +%changelog +* Fri Feb 29 2013 <carl@five-ten-sg.com> - 1.1.9-0 +- Initial build. +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sendmail.pattern Fri Mar 01 14:58:09 2013 -0800 @@ -0,0 +1,40 @@ +# https://raw.github.com/augieschwer/grok-patterns/master/sendmail.grok +# + +EMAIL %{LOGIN}@%{IPORHOST} +DSN [0-9][.][0-9][.][0-9] + +# Match a relay that gives us a QID in the return status. +SENDMAIL_TO_1 %{SYSLOGBASE} %{QID:qid}: to=<%{EMAIL:to}>, (%{WORD}=%{DATA},)+ relay=%{IPORHOST:relay} \[%{IP}\], dsn=%{DSN:dsn}, stat=%{DATA:status} \(%{QID:qid} %{GREEDYDATA:status_message}\) + +# Match a relay that does NOT give us a QID in the return status. +SENDMAIL_TO_2 %{SYSLOGBASE} %{QID:qid}: to=<%{EMAIL:to}>, (%{WORD}=%{DATA},)+ relay=%{IPORHOST:relay} \[%{IP}\], dsn=%{DSN:dsn}, stat=%{DATA:status} \(%{GREEDYDATA:status_message}\) + +# Match a message with no relay IP address or status message. +SENDMAIL_TO_3 %{SYSLOGBASE} %{QID:qid}: to=<%{EMAIL:to}>, (%{WORD}=%{DATA},)+ relay=%{IPORHOST:relay}, dsn=%{DSN:dsn}, stat=%{GREEDYDATA:status} + +# Match a message with no relay info at all. +SENDMAIL_TO_4 %{SYSLOGBASE} %{QID:qid}: to=<%{EMAIL:to}>, (%{WORD}=%{DATA},)+ stat=%{GREEDYDATA:status} + +### TODO - match multiple recipients in To: field. +#SENDMAIL_TO_5 %{SYSLOGBASE} %{QID:qid}: to=(<%{EMAIL:to}>,)+ (%{WORD}=%{DATA},)+ %{GREEDYDATA:status} + +SENDMAIL_TO (%{SENDMAIL_TO_1}|%{SENDMAIL_TO_2}|%{SENDMAIL_TO_3}|%{SENDMAIL_TO_4}) + +SENDMAIL_FROM %{SYSLOGBASE} %{QID:qid}: from=<%{EMAIL:from}>, (%{WORD}=%{DATA},)+ relay=%{IPORHOST:relay} \[%{IP}\] + +SENDMAIL_OTHER_1 %{SYSLOGBASE} %{QID:qid}: %{GREEDYDATA:message} +SENDMAIL_OTHER_2 %{SYSLOGBASE} STARTTLS=(client|server), relay=(\[)?%{IPORHOST:relay}(\])?%{GREEDYDATA:message} +SENDMAIL_OTHER_3 %{SYSLOGBASE} STARTTLS: %{GREEDYDATA:message} +SENDMAIL_OTHER_4 %{SYSLOGBASE} ruleset=tls_server, arg1=SOFTWARE, relay=%{IPORHOST:relay}, %{GREEDYDATA:message} +SENDMAIL_OTHER_5 %{SYSLOGBASE} STARTTLS=client, error: %{GREEDYDATA:message} + +SENDMAIL_RELAY %{SYSLOGBASE} ruleset=check_relay, arg1=(\[)?%{IPORHOST}(\])?, arg2=%{IP:ip}, relay=(\[)?%{IPORHOST:relay}(\])??%{GREEDYDATA:message} + +SENDMAIL_AUTH_1 %{SYSLOGBASE} AUTH=server, relay=%{IPORHOST:relay} \[%{IP}\]( \(may be forged\))?, authid=%{LOGIN:user}(@%{IPORHOST})?, %{GREEDYDATA:message} +SENDMAIL_AUTH_2 %{SYSLOGBASE} AUTH=server, relay=\[%{IP}\], authid=%{LOGIN:user}(@%{IPORHOST})?, %{GREEDYDATA:message} +SENDMAIL_AUTH (%{SENDMAIL_AUTH_1}|%{SENDMAIL_AUTH_2}) + +SENDMAIL_OTHER (%{SENDMAIL_OTHER_1}|%{SENDMAIL_OTHER_2}|%{SENDMAIL_OTHER_3}|%{SENDMAIL_OTHER_4}|%{SENDMAIL_OTHER_5}) + +SENDMAIL (%{SENDMAIL_TO}|%{SENDMAIL_FROM}|%{SENDMAIL_OTHER}|%{SENDMAIL_AUTH}|%{SENDMAIL_RELAY})