#!/usr/bin/perl -w # # Web tool to search inside syslog's files # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # # author Jean-Louis Bergamo # copyright 2006-2007 # ## $Id: GoogLog_cgi.pl,v 1.18 2006/12/21 13:42:14 jlb Exp $ ## $Source: /cvsroot/dev/Linux/GoogLog/GoogLog_cgi.pl,v $ # # standard module use strict; use Time::HiRes qw( usleep ualarm gettimeofday tv_interval); use Carp; # CPAN module use Config::Tiny; use Compress::Zlib ; use CGI; use HTML::Template; =head1 NAME GoogLog - CGI part (the research part) of GoogLog =head1 VERSION $Revision: 1.18 $ =cut # versionning '$Revision: 1.18 $ ' =~ /\s([\d.]+)/; my $VERSION = $1; =head1 SYNOPSIS GoogLog is a easy and simple web tool to search inside your syslog files. If you want to search inside your syslog files from a web interface, this tool is made for you. Install it in the cgi-bin directory of your web server, and configure some variable below. The goal of Googlog is to be simple to install and use. so there's not a lot of functionnality, but, from my point of vue, it is simple to install and use and fast to search into syslog's files. =head1 DOWNLOAD Go here L to download the latest version. =head1 DEMO Follow this link to see an online demo of GoogLog (on very few restrictedt syslog files): L =head1 INSTALL =head2 PERL Modules Required PERL modules =over =item Config::Tiny =item Compress::Zlib =item HTML::Template =back Under debian and debian based distrib : apt-get install libhtml-template-perl libcompress-zlib-perl libconfig-tiny-perl =head2 SYSLOG server I recommend to use syslog-ng (actually google works only with syslog-ng) and to add this sort of generic config in the syslog-ng config file (on your syslog server of course). Modify to fit your needs but, keep in mind that the structure of the directory must always be like this : /LOGDIR/Category/YYYY-MM-DD/HH-name.log # log generique source net { udp(); }; destination d_generique { file("/var/log/net/$FACILITY/$YEAR-$MONTH-$DAY/$HOUR-$FACILITY.log" owner(root) group(adm) perm(0644) dir_perm(0755) create_dirs(yes) dir_group(adm)); }; log { source(net) ; destination(d_generique); }; =head2 GoogLog GoogLog consist of 3 files that need to be installed into the same directory into your cgi-bin directory (/usr/lib/cgi-bin/ on GNU/Debian): =over =item GoogLog_cgi.pl This file. this is the main program. It has to have the execution bits enabled to the user that run you web server (or the user you configured into the virtualhost if you use suexec) =item index.html The template file used to render the result. put it with the B. =item googlog.ini Configuration file for GoogLog. you will put here where to find logfile to search in. see description below to know what to put in it. =back =head1 PARAMETERS =head2 CONFIG FILE Googlog use a config file (placed in the same directory of the Googlog CGI) that contains these attributes : =head3 general directives =over =item debug Active debug or not =item logdir Where are the logs =back =head3 Directive for categories Each specific categories start with [nameofcategories] and understand this specific attributes : =over =item subdir For wich subdir this category is for =item line_around How many lines do we search around the previous found pattern for related search. =back =head2 CGI This is what GoogLog_cgi.pl understand in a GET request. =over =item action action can be : list : list available categories to search (not used now) search : do the real search related : do the related search (it means try to use some related grep to find related line) =item date Format of date : YYYY-MM-DD =item hours Which hours =item pattern What we're looking for in log =item cat Which category =item host On which host do we need to do related search =item process On which process do we need to do related search (if not more pertinent) =item file Which file. used by related search =back =cut # global var # where is the config file my $CONFFILE='googlog.ini'; my $Config; if (-r $CONFFILE){ # Create a config $Config = Config::Tiny->new(); # Open the config $Config = Config::Tiny->read($CONFFILE); }else{ warn "Can't read $CONFFILE\n"; } # Configuration. you have to edit these (at least the CONFFILE). my $LOGDIR=$Config->{_}->{'logdir'}||'/var/log/net'; # END Configuration # useful var my $DEBUG=$Config->{_}->{'debug'}||0; my $cgi=new CGI; my $t0 = [gettimeofday]; my @HOURS=( {HNAME=>'00',HSEL=>'0'}, {HNAME=>'01',HSEL=>'0'}, {HNAME=>'02',HSEL=>'0'}, {HNAME=>'03',HSEL=>'0'}, {HNAME=>'04',HSEL=>'0'}, {HNAME=>'05',HSEL=>'0'}, {HNAME=>'06',HSEL=>'0'}, {HNAME=>'07',HSEL=>'0'}, {HNAME=>'08',HSEL=>'0'}, {HNAME=>'09',HSEL=>'0'}, {HNAME=>'10',HSEL=>'0'}, {HNAME=>'11',HSEL=>'0'}, {HNAME=>'12',HSEL=>'0'}, {HNAME=>'13',HSEL=>'0'}, {HNAME=>'14',HSEL=>'0'}, {HNAME=>'15',HSEL=>'0'}, {HNAME=>'16',HSEL=>'0'}, {HNAME=>'17',HSEL=>'0'}, {HNAME=>'18',HSEL=>'0'}, {HNAME=>'19',HSEL=>'0'}, {HNAME=>'20',HSEL=>'0'}, {HNAME=>'21',HSEL=>'0'}, {HNAME=>'22',HSEL=>'0'}, {HNAME=>'23',HSEL=>'0'} ); my $action; unless ($action = $cgi->param('action')){ #default page my ($hour,$mday,$mon,$year)=(localtime(time))[2..5]; my $today=(1900+$year)."-".(($mon+1)<10?"0".($mon+1):($mon+1))."-".($mday<10?"0".$mday:$mday); # open the html template my $template = HTML::Template->new(filename => 'index.html'); # fill in some parameters $template->param(FORM => $cgi->url(-full)); $template->param(ACTION => 'search'); #$template->param(DATE => $today); $template->param(DATE => &list_date($today)); $template->param(CATEGORY => &list_category()); $HOURS[$hour]->{HSEL}=1; $template->param(HOURS => \@HOURS); my $elapsed=tv_interval ( $t0, [gettimeofday]); $template->param(ELAPSED => $elapsed); $template->param(VERSION => $VERSION); # send the obligatory Content-Type and print the template output print "Content-Type: text/html\n\n", $template->output; exit(0); } # what do we have to do if ($action eq 'list'){ # only send available categories print "Content-Type: text/plain\n\n"; my $cat=&list_category(); foreach (0 .. scalar (@$cat)){ print $cat->[$_]->{name},"\n"; } exit(0); }elsif($action eq 'search'){ # do the real search my @res=(); my $file; my $lastline; my $line; my $pattern; my $cat; my $line_number=0; my @hours; my $hours_cgi; my @files; unless ($pattern = $cgi->param('pattern')){ print "Content-Type: text/plain\n\n"; print "pattern missing\n"; exit(0); } unless ($cat = $cgi->param('cat')){ print "Content-Type: text/plain\n\n"; print "category missing\n"; exit(0); } my $date = $cgi->param('date'); unless ($date =~ /\d{4}\-\d{2}-\d{2}/i){ print "Content-Type: text/plain\n\n"; print "format invalide pour la date : $date\n"; exit(0); } if (@hours= $cgi->param('hours')){ $hours_cgi=join("&hours=",@hours); @files=&list_files($Config->{$cat}->{'subdir'},$date,@hours); }elsif ($file=$cgi->param('file')){ if (&check_filename($file,$cat,$date)) { warn "Good filename : $file\n" if $DEBUG; @files=($file); }else { print "Content-Type: text/plain\n\n"; print "Bad filename : $file\n"; exit(0); } }else{ print "Content-Type: text/plain\n\n"; print "file AND hours missing\n"; exit(0); } warn "Files : ".join(',',@files) if $DEBUG; foreach $file (@files) { warn "=>Fichier : $file\n" if $DEBUG; if ($file =~/gz$/i){ # compressed file my $gz; unless ($gz = gzopen($file, "rb")){ warn "Cannot open $file: $gzerrno\n" ; next; } while ($gz->gzreadline($_) > 0) { my ($host,$processus)=(split(/\s+/,$_,6))[3,4]; if (/$pattern/o){ my @res_line=split(/\s+/,$_); my $res_line; foreach (@res_line){ my $pat=$_; $pat=~s/(\[|\]|\\|\|)/\\$1/g; my $escapedstr=$cgi->escapeHTML($_); $escapedstr=~s/\^I/
/g; $escapedstr=~s/($pattern)/$1<\/b>/g; $res_line.="url(-full)."?action=search&cat=$cat&date=$date&pattern=$pat&hours=$hours_cgi\">$escapedstr "; } push(@res,{ res_nb_line=>$line_number, res_line=>$res_line, related_url=>$cgi->url(-full)."?action=related&file=$file&cat=$cat&date=$date&pattern=$pattern&line_number=$line_number&hours=$hours_cgi&host=$host&processus=$processus" } ); } $line_number++; } if ($gzerrno != Z_STREAM_END){ warn "Error reading from $file: $gzerrno\n"; #exit(0); } $gz->gzclose() ; }else{ # uncompressed file unless (open(FILE ,$file)){ warn "Cannot open $file: $!\n" ; next; } while () { my ($host,$processus)=(split(/\s+/,$_,6))[3,4]; if (/$pattern/o){ my @res_line=split(/\s+/,$_); my $res_line; foreach (@res_line){ my $pat=$_; $pat=~s/(\[|\]|\\|\|)/\\$1/g; my $escapedstr=$cgi->escapeHTML($_); $escapedstr=~s/\^I/
/g; $escapedstr=~s/($pattern)/$1<\/b>/g; $res_line.="url(-full)."?action=search&cat=$cat&date=$date&pattern=$pat&hours=$hours_cgi\">$escapedstr "; } push(@res,{ res_nb_line=>$line_number, res_line=>$res_line, related_url=>$cgi->url(-full)."?action=related&file=$file&cat=$cat&date=$date&pattern=$pattern&line_number=$line_number&hours=$hours_cgi&host=$host&processus=$processus" } ); } $line_number++; } close(FILE) ; } } # open the html template my $template = HTML::Template->new(filename => 'index.html'); # fill in some parameters $template->param(FORM => $cgi->url(-full)); $template->param(ACTION => 'search'); $template->param(PATTERN => $pattern); #$template->param(DATE => $date); $template->param(DATE => &list_date($date)); $template->param(CATEGORY => &list_category($cat)); foreach my $hour (@hours){ $HOURS[$hour]->{HSEL}=1; } $template->param(HOURS => \@HOURS); $template->param(RESULTS => scalar @res); $template->param(RES_LOOP => \@res); my $elapsed=tv_interval ( $t0, [gettimeofday]); $template->param(ELAPSED => $elapsed); $template->param(VERSION => $VERSION); $template->param(LINENB => $line_number); # send the obligatory Content-Type and print the template output print "Content-Type: text/html\n\n", $template->output; exit(0); }elsif($action eq 'related'){ # RELATED search #print "Content-Type: text/plain\n\n"; #print "related\n"; my $pattern; my $cat; my $file; # we will put file content in this array my @file; my $host; my $processus; my @res; my $line_number; my @hours; my $hours_cgi; unless ($pattern = $cgi->param('pattern')){ print "Content-Type: text/plain\n\n"; print "pattern missing\n"; exit(0); } unless (defined($line_number = $cgi->param('line_number'))){ print "Content-Type: text/plain\n\n"; print "line_number missing\n"; exit(0); } unless ($host = $cgi->param('host')){ print "Content-Type: text/plain\n\n"; print "host missing\n"; exit(0); } unless ($processus = $cgi->param('processus')){ print "Content-Type: text/plain\n\n"; print "processus missing\n"; exit(0); } unless ($cat = $cgi->param('cat')){ print "Content-Type: text/plain\n\n"; print "category missing\n"; exit(0); } unless ($file=$cgi->param('file')){ print "Content-Type: text/plain\n\n"; print "file missing\n"; exit(0); } my $date = $cgi->param('date'); unless ($date =~ /\d{4}\-\d{2}-\d{2}/i){ print "Content-Type: text/plain\n\n"; print "format invalide pour la date : $date\n"; exit(0); } if (@hours= $cgi->param('hours')){ $hours_cgi=join("&hours=",@hours); } foreach ($cgi->param()){ warn "$_ => ",$cgi->param($_),"\n" if $DEBUG; } my $line_around=$Config->{$cat}->{'line_around'}||10; if (&check_filename($file,$cat,$date)) { warn "Good filename : $file\n" if $DEBUG; }else { print "Content-Type: text/plain\n\n"; print "Bad filename : $file\n"; exit(0); } warn "=>Fichier : $file\n" if $DEBUG; my $line=0; my $tmp; my $end = $line_number + $line_around; my $start = $line_number - $line_around; my $found_line; warn "END: $line_number + $line_around = $end\n" if $DEBUG; if ($file =~/gz$/i){ # compressed file my $gz; unless ($gz = gzopen($file, "rb")){ warn "Cannot open $file: $gzerrno\n" ; next; } while ($line <= $end && $gz->gzreadline($tmp) > 0){ push(@file,"$line:$tmp") if ($line >= $start); $found_line = $tmp if $line==$line_number; warn"LINE: $line\n" if $DEBUG; $line++; } #if ($gzerrno != Z_STREAM_END){ # warn "Error reading from $file: $gzerrno\n"; # exit(0); #} $gz->gzclose() ; }else{ # uncompressed file unless (open(FILE ,$file)){ warn "Cannot open $file: $!\n" ; next; } while ($line <= $end && ($tmp=)) { push(@file,"$line:$tmp") if ($line >= $start); $found_line = $tmp if $line==$line_number; warn"LINE: $line\n" if $DEBUG; $line++; } close(FILE) ; } # do a global grep warn "HOST : $host\n" if $DEBUG; foreach (@file){ warn "$_"; } # my @tab=grep(/\s$host\s/o,@file); # foreach (@tab){ # warn "$_"; # } # foreach (grep(/\s$host\s/o,@file)){ # my ($line,$msg)=split(/:/,$_,2); # push (@res, { # rel_nb_line=>$line, # rel_line=>$msg # } # ); # } # search @res=&googlog_grep({ host=>$host, processus=>$processus, found_line=>$found_line, pattern=>$pattern, file=>$file, cat=>$cat, date=>$date, hours=>$hours_cgi, },\@file); # open the html template my $template = HTML::Template->new(filename => 'index.html'); # fill in some parameters $template->param(FORM => $cgi->url(-full)); $template->param(ACTION => 'search'); $template->param(PATTERN => $pattern); #$template->param(DATE => $date); $template->param(DATE => &list_date($date)); $template->param(CATEGORY => &list_category($cat)); if ($file =~ /\/([\d]+)-[^\/]*$/){ $HOURS[$1]->{HSEL}=1; } $template->param(HOURS => \@HOURS); $template->param(RELATED => scalar @res); $template->param(REL_LOOP => \@res); my $elapsed=tv_interval ( $t0, [gettimeofday]); $template->param(ELAPSED => $elapsed); $template->param(VERSION => $VERSION); # send the obligatory Content-Type and print the template output print "Content-Type: text/html\n\n", $template->output; exit(0); } # #=head1 FUNCTIONS # #=head2 Googlog_grep # #Do the grep inside lines extracted from log file # #=cut # sub googlog_grep{ my($args,$tab)=@_; my @res; my $packagebase='GoogLog'; my $procname='Default'; my $pattern; my $host=$args->{'host'}; my $processus=$args->{'processus'}; my $mod; my %modinside=( 'exim'=>1, ); if ($processus =~/^(.*)\[/){ $procname=$1; } $mod=$packagebase."::".$procname; unless(defined $modinside{$procname}){ unless (eval "use $mod") { warn "couldn't load $mod: $@"; $mod=$packagebase."::Default"; } } warn "use ".$mod."::GrepRelated function\n" if $DEBUG; no strict 'refs'; @res = &{ $mod . "::GrepRelated"}($args,$tab); return @res; } # #=head2 check_filename # #Check if the filename contains LOGDIR, category and not something like .. # #return false or true # #=cut # sub check_filename{ my ($filename,$cat,$date)=@_; unless ( $filename =~ /^$LOGDIR[\/]+$cat[\/]+$date[\/]+.*$/) { # bad filename return undef; } if ($filename =~ /\.\./g) { # try to go to parent dir return undef; } # good filename return 1; } # #=head2 list_category # #Not Yet Used # #=cut # sub list_category{ my ($cat_sel)=@_; my @res; # only send available categories foreach (sort keys %$Config){ if (defined $Config->{$_}->{subdir} && -d $LOGDIR.'/'.$Config->{$_}->{subdir}){ if (defined $cat_sel && $_ eq $cat_sel){ push(@res,{name=>$_,desc=>$_,sel=>1}); }else{ push(@res,{name=>$_,desc=>$_}); } } } return(\@res); } # #=head2 list_date # #Used to have all the date available from syslog files # #=cut # sub list_date{ my ($date_sel)=@_; my @res; my %dates; # only send available categories foreach (sort keys %$Config){ if (defined $Config->{$_}->{subdir} && -d $LOGDIR.'/'.$Config->{$_}->{subdir}){ foreach my $dir (glob($LOGDIR.'/'.$Config->{$_}->{subdir}.'/*')){ if ($dir =~ /\/(\d{4}-\d{2}-\d{2})[\/]*$/){ $dates{$1}=1; } } } } foreach (sort keys %dates){ if (defined $date_sel && $_ eq $date_sel){ push(@res,{dname=>$_,dsel=>1}); }else{ push(@res,{dname=>$_,dsel=>0}); } } return(\@res); } # #=head2 list_files # #return a list of files in where we need to search # #=cut # sub list_files{ my ($cat,$date,@hours)=@_; my @files=(); my $hour; foreach $hour (@hours){ warn "Glob : $LOGDIR/$cat/$date/$hour*" if $DEBUG; push(@files,glob($LOGDIR.'/'.$cat.'/'.$date.'/'.$hour.'*')); } return(@files); } 1; package GoogLog::Default; # # Default search function for related search # use strict; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); use Exporter; $VERSION = 1.00; # Or higher @ISA = qw(Exporter); @EXPORT = qw(GrepRelated); # Symbols to autoexport (:DEFAULT tag) @EXPORT_OK = qw(GrepRelated); # Symbols to export on request %EXPORT_TAGS = ( # Define names for sets of symbols Functions => [qw(GrepRelated)], ); ######################## # your code goes here ######################## sub GrepRelated{ my($args,$tab)=@_; my @res; my $pattern; my $this_pack = __PACKAGE__; my $patternorig=$args->{'pattern'}; my $host=$args->{'host'}; my $processus=$args->{'processus'}; $processus =~ s/\[/\\\[/g; $processus =~ s/\]/\\\]/g; $pattern="\\s+$host\\s+$processus\\s+"; warn "$this_pack PATTERN = $pattern\n" if $DEBUG; foreach (grep(/$pattern/o,@$tab)){ my ($line,$msg)=split(/:/,$_,2); # split msg line into small links my @res_line=split(/\s+/,$msg); my $res_line; foreach (@res_line){ my $pat=$_; $pat=~s/(\[|\]|\\|\|)/\\$1/g; my $escapedstr=$cgi->escapeHTML($_); $escapedstr=~s/\^I/
/g; $escapedstr=~s/($patternorig)/$1<\/b>/g; $res_line.="url(-full)."?action=search&cat=$args->{cat}&date=$args->{date}&pattern=$pat&file=$args->{file}&hours=$args->{hours}\">$escapedstr "; } push (@res, { rel_nb_line=>$line, rel_line=>$res_line } ); } return @res; } 1; # this should be your last line # EXIM "plugins" package GoogLog::exim; # # exim search function for related search # use strict; use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION); use Exporter; $VERSION = 1.00; # Or higher @ISA = qw(Exporter); @EXPORT = qw(GrepRelated); # Symbols to autoexport (:DEFAULT tag) @EXPORT_OK = qw(GrepRelated); # Symbols to export on request %EXPORT_TAGS = ( # Define names for sets of symbols Functions => [qw(GrepRelated)], ); ######################## # your code goes here ######################## sub GrepRelated{ my($args,$tab)=@_; my $pattern; my @res; my $this_pack = __PACKAGE__; my $patternorig=$args->{'pattern'}; my $host=$args->{'host'}; my $processus=$args->{'processus'}; $processus =~ s/\[/\\\[/g; $processus =~ s/\]/\\\]/g; if ($args->{'found_line'} =~ /\s+([^\s]+)\s+(<=|=>|->)\s+/oi) { # ID found so grep on it $pattern=$1; }else { # no ID found so only grep on host and processus $pattern="\\s+$host\\s+$processus\\s+"; } warn "$this_pack PATTERN = $pattern\n" if $DEBUG; foreach (grep(/$pattern/o,@$tab)){ my ($line,$msg)=split(/:/,$_,2); # split msg line into small links my @res_line=split(/\s+/,$msg); my $res_line; foreach (@res_line){ my $pat=$_; $pat=~s/(\[|\]|\\|\|)/\\$1/g; my $escapedstr=$cgi->escapeHTML($_); $escapedstr=~s/\^I/
/g; $escapedstr=~s/($patternorig)/$1<\/b>/g; $res_line.="url(-full)."?action=search&cat=$args->{cat}&date=$args->{date}&pattern=$pat&file=$args->{file}&hours=$args->{hours}\">$escapedstr "; } push (@res, { rel_nb_line=>$line, rel_line=>$res_line } ); } return @res; } 1; # this should be your last line =head1 HISTORY Extract from CVS commit : $Log: GoogLog_cgi.pl,v $ Revision 1.18 2006/12/21 13:42:14 jlb add some documentation Revision 1.17 2006/12/21 13:22:50 jlb replace date text field with a select field Revision 1.16 2006/12/08 16:26:58 jlb substitute ^I by
in syslog line, because some program syslog mutltiline with ^I instead of \n Revision 1.15 2006/12/08 15:30:09 jlb cosmetic update : - result are put in a table (to be well align) - nbsp replaced by space (to be able to have broken line) Revision 1.14 2006/12/07 16:20:47 jlb Do some documentation work Revision 1.13 2006/12/04 10:57:37 jlb use CVS version as the main versionning Revision 1.12 2006/12/01 16:01:22 jlb add some cosmetic features Revision 1.11 2006/12/01 14:38:50 jlb add escapeHTML function for some string that contains bad caracters Revision 1.10 2006/12/01 14:12:25 jlb Little bug fix on the default date presentation Revision 1.9 2006/11/30 17:08:38 jlb bug fix in the generation of file list Revision 1.8 2006/11/30 16:43:51 jlb - start using CSS - simplify template (only used index.html - add URL-click-search feature (can make search on every part of log found) Revision 1.7 2006/10/19 16:39:21 jlb start of cosmetic design Revision 1.6 2006/10/18 21:29:09 jlb bug fix when file is compressed and we want to see related lines Revision 1.5 2006/10/18 21:25:56 jlb starting the plugin support Revision 1.4 2006/10/16 16:19:30 jlb irelated link starting to work Revision 1.3 2006/10/15 20:16:06 jlb work on related lines Revision 1.2 2006/10/12 16:23:27 jlb little design starting the related link (TBC) Revision 1.1 2006/10/11 23:46:33 jlb first functionnal version TODO : return more lines from log files =head1 LICENCE This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. =head1 AUTHOR Jean-Louis Bergamo copyright 2006-2007