Mercurial > logstash
changeset 33:0faebb0b0fa4
update to kibana 3, logstash 1.2.1, es 0.90.5
author | Carl Byington <carl@five-ten-sg.com> |
---|---|
date | Mon, 23 Sep 2013 11:50:21 -0700 |
parents | 082f7a76fa48 |
children | 8ed811f9a0bd |
files | Kibana-external-config-patch Kibana-no-git-patch Kibana.rc Kibana.spec Kibana.sysconfig Makefile elasticsearch.spec kibana.conf kibana.spec logstash.Makefile logstash.spec |
diffstat | 11 files changed, 105 insertions(+), 348 deletions(-) [+] |
line wrap: on
line diff
--- a/Kibana-external-config-patch Wed May 29 11:37:41 2013 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,93 +0,0 @@ ---- KibanaConfig.rb 2013-05-02 15:22:03.011877891 -0700 -+++ KibanaConfig.new.rb 2013-05-02 15:26:57.419447970 -0700 -@@ -3,26 +3,14 @@ - # Your elastic search server(s). This may be set as an array for round robin - # load balancing - # Elasticsearch = ["elasticsearch1:9200","elasticsearch2:9200"] -- Elasticsearch = "localhost:9200" -+ Elasticsearch = ENV['KIBANA_ES'] ? ENV['KIBANA_ES'] : "localhost:9200" - - #Set the Net::HTTP read/open timeouts for the connection to the ES backend - ElasticsearchTimeout = 500 - -- # The port Kibana should listen on -- KibanaPort = 5601 -- -- # The adress ip Kibana should listen on. Comment out or set to -- # 0.0.0.0 to listen on all interfaces. -- KibanaHost = '127.0.0.1' -- -- # Below is an example showing how to configure the same variables -- # using environment variables, which can be set in an init script -- # es_ip = ENV['ES_IP'] ? ENV['ES_IP'] : '127.0.0.1' -- # es_port = ENV['ES_PORT'] ? ENV['ES_PORT'] : 9200 -- # Elasticsearch = "#{es_ip}:#{es_port}" -- # KibanaPort = ENV['KIBANA_PORT'] ? ENV['KIBANA_PORT'] : 5601 -- # KibanaHost = ENV['KIBANA_HOST'] ? ENV['KIBANA_HOST'] : 'localhost' -- -+ # The port and adress ip Kibana should listen on. -+ KibanaPort = ENV['KIBANA_PORT'] ? ENV['KIBANA_PORT'] : 5601 -+ KibanaHost = ENV['KIBANA_HOST'] ? ENV['KIBANA_HOST'] : 'localhost' - - # The record type as defined in your logstash configuration. - # Seperate multiple types with a comma, no spaces. Leave blank -@@ -44,19 +32,19 @@ - # Do not use isoUtcDatetime or the "UTC:" prefix described in the above - # article, as timezone correction is already performed by the "Timezone" - # config variable. -- # Time_format = 'isoDateTime' -+ # Time_format = 'isoDateTime' - Time_format = 'mm/dd HH:MM:ss' - - # Change which fields are shown by default. Must be set as an array - # Default_fields = ['@fields.vhost','@fields.response','@fields.request'] - Default_fields = ['@message'] - -- # If set to true, Kibana will use the Highlight feature of Elasticsearch to -+ # If set to true, Kibana will use the Highlight feature of Elasticsearch to - # display highlighted search results -- Highlight_results = true -+ Highlight_results = false - -- # A field needs to be specified for the highlight feature. By default, -- # Elasticsearch doesn't allow highlighting on _all because the field has to -+ # A field needs to be specified for the highlight feature. By default, -+ # Elasticsearch doesn't allow highlighting on _all because the field has to - # be either stored or part of the _source field. - Highlighted_field = "@message" - -@@ -99,18 +87,18 @@ - # indexing - Smart_index = true - -- # You can define your custom pattern here for index names if you -- # use something other than daily indexing. Pattern needs to have -- # date formatting like '%Y.%m.%d'. Will accept an array of smart -- # indexes. -- # Smart_index_pattern = ['logstash-web-%Y.%m.%d', 'logstash-mail-%Y.%m.%d'] -+ # You can define your custom pattern here for index names if you -+ # use something other than daily indexing. Pattern needs to have -+ # date formatting like '%Y.%m.%d'. Will accept an array of smart -+ # indexes. -+ # Smart_index_pattern = ['logstash-web-%Y.%m.%d', 'logstash-mail-%Y.%m.%d'] - # Smart_index_pattern = 'logstash-%Y.%m.%d' - # here is an example of how to set the pattern using an environment variable - # Smart_index_pattern = ENV['SMART_INDEX'] ? ENV['SMART_INDEX'] : 'logstash-%Y.%m.%d' - Smart_index_pattern = 'logstash-%Y.%m.%d' -- -+ - # Number of seconds between each index. 86400 = 1 day. -- Smart_index_step = 86400 -+ Smart_index_step = 86400 - - # ElasticSearch has a default limit on URL size for REST calls, - # so Kibana will fall back to _all if a search spans too many -@@ -120,7 +108,7 @@ - - # Elasticsearch has an internal mechanism called "faceting" for performing - # analysis that we use for the "Stats" and "Terms" modes. However, on large -- # data sets/queries facetting can cause ES to crash if there isn't enough -+ # data sets/queries facetting can cause ES to crash if there isn't enough - # memory available. It is suggested that you limit the number of indices that - # Kibana will use for the "Stats" and "Terms" to prevent ES crashes. For very - # large data sets and undersized ES clusers, a limit of 1 is not unreasonable.
--- a/Kibana-no-git-patch Wed May 29 11:37:41 2013 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,16 +0,0 @@ ---- kibana.gemspec 2013-04-25 01:59:24.000000000 -0700 -+++ kibana.new.gemspec 2013-05-06 23:04:47.780673300 -0700 -@@ -9,11 +9,11 @@ - gem.homepage = "http://rashidkpc.github.com/Kibana/" - gem.license = "MIT License" - -- gem.files = `git ls-files`.split("\n") -+ gem.files = `find * -type f`.split("\n") - gem.name = "kibana" - gem.require_paths = ["lib"] - gem.version = Kibana::VERSION -- gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } -+ gem.executables = `find bin/* -type f`.split("\n").map{ |f| File.basename(f) } - - gem.add_runtime_dependency 'sinatra' - gem.add_runtime_dependency 'json'
--- a/Kibana.rc Wed May 29 11:37:41 2013 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,92 +0,0 @@ -#!/bin/bash -# -# /etc/rc.d/init.d/Kibana -# -# Starts Kibana as a daemon -# -# chkconfig: 2345 20 80 -# description: Starts Kibana as a daemon -# pidfile: /var/run/Kibana.pid - -### BEGIN INIT INFO -# Provides: Kibana -# Required-Start: $local_fs $remote_fs -# Required-Stop: $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: S 0 1 6 -# Short-Description: Kibana -# Description: Starts Kibana as a daemon. - -### END INIT INFO - -. /etc/sysconfig/Kibana - -DESC="Kibana Daemon" -LOGFILE=/var/log/Kibana/Kibana.log -PIDFILE=/var/run/Kibana.pid -base=Kibana - -. /etc/init.d/functions - -# -# Function that starts the daemon/service -# -do_start() { - cd /home/kibana - pid=$( - exec sudo -u kibana /bin/bash - <<EOF - export KIBANA_HOST=$KIBANA_HOST - export KIBANA_PORT=$KIBANA_PORT - export KIBANA_ES=$KIBANA_ES - export KIBANA_CONFIG=$KIBANA_CONFIG - kibana >>$LOGFILE 2>&1 & - echo \$! -EOF - ) - echo $pid >$PIDFILE - [ -n "$pid" ] && success $"$base startup" || failure $"$base startup" -} - - -# -# Function that stops the daemon/service -# -do_stop() { - killproc -p $PIDFILE Kibana -} - - -case "$1" in - start) - echo -n "Starting $DESC: " - do_start - touch /var/lock/subsys/$base - ;; - stop) - echo -n "Stopping $DESC: " - do_stop - rm /var/lock/subsys/$base 2>/dev/null - rm $PIDFILE 2>/dev/null - ;; - restart) - echo -n "Restarting $DESC: " - do_stop - do_start - ;; - reload) - echo -n "Reloading $DESC: " - pid=$(cat $PIDFILE) - [ -n "$pid" ] && pkill -HUP -u Kibana -P $pid - ;; - status) - echo -n "$base " - status -p $PIDFILE - ;; - *) - echo "Usage: service $base {start|stop|status|restart}" >&2 - exit 3 - ;; -esac - -echo -exit 0
--- a/Kibana.spec Wed May 29 11:37:41 2013 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,117 +0,0 @@ -# prevent brp repack jar files -%define __os_install_post %{nil} - -%global commit df8ae2344433e5072a8dfe77082acfd1c9b6ed62 -%global shortcommit %(c=%{commit}; echo ${c:0:8}) -%global name2 kibana -%global gem_dir %(ruby -rubygems -e 'puts Gem::dir' 2>/dev/null) -%global gem_name kibana -%global gem_instdir %{gem_dir}/gems/%{gem_name}-%{version} - -Name: Kibana -Version: 0.2.0 -Release: 1%{?dist} -Summary: Kibana is a user friendly way to view, search and visualize your log data - -Group: System Environment/Daemons -License: MIT -URL: http://kibana.org/ -BuildArch: noarch -Source0: https://nodeload.github.com/rashidkpc/Kibana/archive/%{commit}/%{name}-%{version}-%{shortcommit}.tar.gz -Source1: Kibana-external-config-patch -Source2: Kibana-no-git-patch -Source10: %{name}.rc -Source13: %{name}.sysconfig -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -Autoreq: 0 - -BuildRequires: ruby -BuildRequires: ruby-devel -BuildRequires: rubygems -Provides: rubygem(%{gem_name}) = %{version} - -Requires: ruby -Requires: rubygems -Requires: rubygem(sinatra) -Requires: rubygem(json) -Requires: rubygem(fastercsv) -Requires: rubygem(daemons) -Requires: rubygem(tzinfo) -Requires: rubygem(thin) -Requires: elasticsearch logstash -Requires(pre): /usr/sbin/useradd -Requires(pre): /usr/bin/getent -Requires(postun): /usr/sbin/userdel -Requires(post,preun): /sbin/chkconfig -Requires(post,preun): /sbin/service - -%description -Kibana is a highly scalable interface for Logstash and ElasticSearch -that allows you to efficiently search, graph, analyze and otherwise -make sense of a mountain of logs. - - -%prep -%setup -qn %{name}-%{commit} -patch <%SOURCE1 -patch <%SOURCE2 - - -%build -mkdir -p ./%{gem_dir} -gem build %{gem_name}.gemspec -gem install --local --install-dir ./%{gem_dir} -V --force --no-rdoc --no-ri %{gem_name}-%{version}.gem - - -%install -rm -rf $RPM_BUILD_ROOT -mkdir -p $RPM_BUILD_ROOT/%{gem_dir} -mkdir -p $RPM_BUILD_ROOT/%{_prefix} -cp -pa ./%{gem_dir}/* $RPM_BUILD_ROOT/%{gem_dir} -mv -f $RPM_BUILD_ROOT/%{gem_dir}/bin $RPM_BUILD_ROOT/%{_prefix} -install -d $RPM_BUILD_ROOT/var/log/%{name} -install -D -m 755 %SOURCE10 $RPM_BUILD_ROOT/etc/rc.d/init.d/%{name} -install -D -m 640 %SOURCE13 $RPM_BUILD_ROOT/etc/sysconfig/%{name} -mv -f %{name}Config.rb $RPM_BUILD_ROOT/etc/sysconfig/%{name}Config.rb - - -%pre -/usr/bin/getent passwd %{name2} >/dev/null || /usr/sbin/useradd -r -m -c "%{name} pseudo-user" %{name2} >/dev/null - - -%post -/sbin/chkconfig --add %{name} - - -%preun -[ $1 = 0 ] && /sbin/service %{name} stop || : -[ $1 = 0 ] && /sbin/chkconfig --del %{name} || : - - -%postun -[ $1 = 0 ] && userdel %{name2} || : - - -%clean -rm -rf $RPM_BUILD_ROOT - - -%files -%defattr(-, root, root, -) -%{_bindir}/%{gem_name} -%{gem_dir}/cache/%{gem_name}-%{version}.gem -%{gem_dir}/specifications/%{gem_name}-%{version}.gemspec -%{gem_instdir} -%config(noreplace) %attr(0640,%{name2},root) /etc/sysconfig/%{name} -%config(noreplace) %attr(0640,%{name2},root) /etc/sysconfig/%{name}Config.rb -%dir %attr(0750,%{name2},root) /var/log/%{name} -/etc/rc.d/init.d/%{name} - - -%changelog -* Mon May 06 2013 <carl@five-ten-sg.com> - 0.2.0-1 -- switch to building standard ruby gem, avoid jruby - -* Fri Apr 25 2013 <carl@five-ten-sg.com> - 0.2.0-0 -- Initial build. -
--- a/Kibana.sysconfig Wed May 29 11:37:41 2013 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,14 +0,0 @@ -# used by /etc/rc.d/init.d/Kibana - -# kibana listening on all interfaces -KIBANA_HOST=0.0.0.0 - -# pretend to be logstash embedded web interface -KIBANA_PORT=9292 - -# kibana connection to elasticsearch -KIBANA_ES="localhost:9200" - -# kibana main config -KIBANA_CONFIG=/etc/sysconfig/KibanaConfig.rb -
--- a/Makefile Wed May 29 11:37:41 2013 -0700 +++ b/Makefile Mon Sep 23 11:50:21 2013 -0700 @@ -2,20 +2,18 @@ dist=.f20 endif -gv=1.1.13 +gv=1.2.1 f=logstash-$(gv)-monolithic.jar g=logstash-$(gv).tar.gz -hv=0.90.0 +hv=0.90.5 h=elasticsearch-$(hv).tar.gz i=GraphTastic.tar.gz icc=06a620d011d08bd2c1099c0d0810c053bf105400 jv=1.7.3 j=jruby-complete-$(jv).jar k=GeoLite-City-2013-01-18.dat.gz -lcc=df8ae2344433e5072a8dfe77082acfd1c9b6ed62 -lcs=df8ae234 -lv=0.2.0 -l=Kibana-$(lv)-$(lcs).tar.gz +lv=3.0.0 +l=kibana-latest.tar.gz download: [ -f $(f) ] || curl https://logstash.objects.dreamhost.com/release/$(f) >$(f) @@ -24,7 +22,7 @@ [ -f $(i) ] || curl https://nodeload.github.com/NickPadilla/GraphTastic/tar.gz/$(icc) >$(i) [ -f $(j) ] || curl http://repository.codehaus.org/org/jruby/jruby-complete/$(jv)/$(j) >$(j) [ -f $(k) ] || curl http://logstash.objects.dreamhost.com/maxmind/$(k) >$(k) - [ -f $(l) ] || curl https://nodeload.github.com/rashidkpc/Kibana/tar.gz/$(lcc) >$(l) + [ -f $(l) ] || curl https://download.elasticsearch.org/kibana/kibana/$(l) >$(l) logstash.manifest: # build the classpath in a manifest file @@ -35,7 +33,7 @@ srpm: logstash.manifest # build the source rpms - rpmbuild --define "dist $(dist)" --define "_sourcedir $(shell pwd)" --define "_srcrpmdir $(shell pwd)" --define "_rpmdir $(shell pwd)" -bs Kibana.spec + rpmbuild --define "dist $(dist)" --define "_sourcedir $(shell pwd)" --define "_srcrpmdir $(shell pwd)" --define "_rpmdir $(shell pwd)" -bs kibana.spec rpmbuild --define "dist $(dist)" --define "_sourcedir $(shell pwd)" --define "_srcrpmdir $(shell pwd)" --define "_rpmdir $(shell pwd)" -bs jruby.spec rpmbuild --define "dist $(dist)" --define "_sourcedir $(shell pwd)" --define "_srcrpmdir $(shell pwd)" --define "_rpmdir $(shell pwd)" -bs GraphTastic-client.spec rpmbuild --define "dist $(dist)" --define "_sourcedir $(shell pwd)" --define "_srcrpmdir $(shell pwd)" --define "_rpmdir $(shell pwd)" -bs elasticsearch.spec @@ -45,7 +43,7 @@ # kill off maven repository cache rm -rf ~/.m2 # build binary rpms - rpmbuild --define "dist $(dist)" --define "_rpmdir $(shell pwd)" --rebuild Kibana-$(lv)-1$(dist).src.rpm + rpmbuild --define "dist $(dist)" --define "_rpmdir $(shell pwd)" --rebuild kibana-$(lv)-0$(dist).src.rpm rpmbuild --define "dist $(dist)" --define "_rpmdir $(shell pwd)" --rebuild jruby-$(jv)-0$(dist).src.rpm rpmbuild --define "dist $(dist)" --define "_rpmdir $(shell pwd)" --rebuild GraphTastic-client-0.0.0-0$(dist).src.rpm rpmbuild --define "dist $(dist)" --define "_rpmdir $(shell pwd)" --rebuild elasticsearch-$(hv)-0$(dist).src.rpm
--- a/elasticsearch.spec Wed May 29 11:37:41 2013 -0700 +++ b/elasticsearch.spec Mon Sep 23 11:50:21 2013 -0700 @@ -4,7 +4,7 @@ Summary: elasticsearch is a scalable full text search engine Name: elasticsearch -Version: 0.90.0 +Version: 0.90.5 Release: 0%{?dist} Group: System Environment/Daemons License: Apache 2.0 @@ -59,6 +59,9 @@ %changelog +* Sun Sep 22 2013 <carl@five-ten-sg.com> - 0.90.5-0 +- update to 0.90.5 + * Fri May 10 2013 <carl@five-ten-sg.com> - 0.90.0-0 - update to 0.90.0
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/kibana.conf Mon Sep 23 11:50:21 2013 -0700 @@ -0,0 +1,14 @@ +# +# This configuration file maps the kibana source tree +# into the URL space. By default these results are only accessible +# from the local host. +# +Alias /kibana /var/www/kibana + +<Location /kibana> + Order deny,allow + Deny from all + Allow from 127.0.0.1 + Allow from ::1 + # Allow from .example.com +</Location>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/kibana.spec Mon Sep 23 11:50:21 2013 -0700 @@ -0,0 +1,69 @@ + +Name: kibana +Version: 3.0.0 +Release: 0%{?dist} +Summary: Kibana is a user friendly way to view, search and visualize your log data + +Group: Applications/Internet +License: ASL 2.0 +URL: http://www.kibana.org/ +BuildArch: noarch +Source0: https://download.elasticsearch.org/kibana/kibana/kibana-latest.tar.gz +Source1: kibana.conf +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +Requires: httpd + +%description +Kibana is a highly scalable interface for Logstash and ElasticSearch +that allows you to efficiently search, graph, analyze and otherwise +make sense of a mountain of logs. + + +%prep +%setup -qn %{name}-latest + + +%build +mv -f app/dashboards/logstash.json app/dashboards/default.json + + +%install +rm -rf $RPM_BUILD_ROOT +mkdir -p $RPM_BUILD_ROOT/var/www/kibana +mv * $RPM_BUILD_ROOT/var/www/kibana +mkdir -p $RPM_BUILD_ROOT/etc/httpd/conf.d +cp %SOURCE1 $RPM_BUILD_ROOT/etc/httpd/conf.d + + +%pre + + +%post + + +%preun + + +%postun + + +%clean +rm -rf $RPM_BUILD_ROOT + + +%files +%defattr(-, root, root, -) +/var/www/%{name} +/etc/httpd/conf.d/%{name}.conf + + +%changelog +* Mon Sep 23 2013 <carl@five-ten-sg.com> - 3.0.0-0 +- update to 3.0.0, pure html/css/javascript + +* Mon May 06 2013 <carl@five-ten-sg.com> - 0.2.0-1 +- switch to building standard ruby gem, avoid jruby + +* Fri Apr 25 2013 <carl@five-ten-sg.com> - 0.2.0-0 +- Initial build. +
--- a/logstash.Makefile Wed May 29 11:37:41 2013 -0700 +++ b/logstash.Makefile Mon Sep 23 11:50:21 2013 -0700 @@ -16,8 +16,9 @@ all: # compile-grammer - touch lib/logstash/config/grammar.rl # force rebuild - make -C lib/logstash/config grammar.rb + # use the prebuild one, since we don't have tt command from treetop (yet?) + #touch lib/logstash/config/grammar.treetop # force rebuild + #make -C lib/logstash/config grammar.rb # compile-runner mkdir -p build/ruby cd lib; $(JRUBYC) -t ../build/ruby logstash/runner.rb @@ -39,8 +40,11 @@ # build-monolith mkdir -p build/monolith/openssl mkdir -p build/monolith/jopenssl + mkdir -p build/monolith/META-INF/services # unpack all jars find $$PWD/vendor/bundle -name '*.jar' | (cd build/monolith; xargs -n1 jar xf) + # Merge all service file in all 3rdparty jars + find $$PWD/vendor/bundle $$PWD/vendor/jar -name '*.jar' | xargs $(JRUBY_CMD) extract_services.rb -o build/monolith/META-INF/services # copy openssl/lib/shared folders/files to root of jar - need this for openssl to work with JRuby cp -r vendor/bundle/jruby/1.9/gems/jruby-openss*/lib/shared/openssl/* build/monolith/openssl cp -r vendor/bundle/jruby/1.9/gems/jruby-openss*/lib/shared/jopenssl/* build/monolith/jopenssl @@ -63,8 +67,6 @@ -C build/monolith . \ -C vendor/bundle/jruby/1.9 gems \ -C vendor/bundle/jruby/1.9 specifications \ - -C lib logstash/web/public \ -C lib logstash/certs \ - -C lib logstash/web/views \ patterns
--- a/logstash.spec Wed May 29 11:37:41 2013 -0700 +++ b/logstash.spec Mon Sep 23 11:50:21 2013 -0700 @@ -3,7 +3,7 @@ Summary: logstash is a tool for managing events and logs. Name: logstash -Version: 1.1.13 +Version: 1.2.1 Release: 0%{?dist} Group: System Environment/Daemons @@ -26,7 +26,7 @@ Source22: dnsbl.pattern BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -BuildRequires: ragel jruby java-devel +BuildRequires: jruby java-devel Requires: logrotate httpd java-devel python-pip python-argparse python-ordereddict Requires: jruby elasticsearch GraphTastic-client @@ -117,6 +117,9 @@ %changelog +* Sun Sep 22 2013 <carl@five-ten-sg.com> - 1.2.1-0 +- update to newer logstash + * Fri May 10 2013 <carl@five-ten-sg.com> - 1.1.12-0 - update to newer logstash