#!/usr/local/bin/perl
eval 'exec perl -S $0 "$@"'
    if 0;

###
# Project:     pflogstats
# Program:     pflogstats.pl
# Description: Main program
#
# Based on:    pflogsumm.pl - Produce summaries of Postfix/VMailer MTA in logfile -
#               Copyright (C) 1998-2002 by James S. Seymour (jseymour@LinxNet.com)
#               Release 1.0.4.
#               http://jimsun.LinxNet.com/postfix_contrib.html
#
# All not overtaken code:
#              Copyright (C) 2002-2005 by Dr. Peter Bieringer <pbieringer@aerasec.de>
#               ftp://ftp.aerasec.de/pub/linux/postfix/pflogsumm/
#
# License:     GNU GPL v2
# CVS:         $Id: pflogstats.pl,v 1.44 2005/05/02 13:39:32 peter Exp $
#
# See also following files: CREDITS ChangeLog TODO LICENSE README INSTALL
###


# Debug values
#                                    | 0x1000 : add verp_mung match token to address
#                                    | 0x2000 : show check of net for skip
#                                    | 0x0002 : show qid
#                                    | 0x0004 : show log line

# ***** Sorry for some coding confusion, this is a developer version *****

# Minium required Perl version
require 5.00503;

use strict;

# Global used Perl modules
use Time::Local;
use locale;
use Getopt::Long;


## Name and version
use vars qw{$release $progName};
our $release = "1.1.1";
our $progName = "pflogstats.pl";
our $copyright = "(P) & (C) by Dr. Peter Bieringer, AERAsec - based on pflogsumm.pl by James S. Seymour";


## Defines before module loader
# Extend module search path !!!! REVIEW before using !!!!

# 1st: look into current directory
push @INC, ".";

# 2nd: look into /usr/local/lib/perl5/site_perl/5.8.8/Pflogstats
push @INC, "/usr/local/lib/perl5/site_perl/5.8.8/Pflogstats";

# 3rd: look into /usr/lib/pflogstats
push @INC, "/usr/lib/pflogstats";

## Define global variables

# option handling
use vars qw{%options %opts %types};

# Default value
$types{'default'} = 0;

# Temp defines here
$types{'test_verp_mung'} = 0;

# module hooks
use vars qw{%hooks};

# module info
use vars qw{%moduleinfo};

# Format options
use vars qw{@opt_format %format};
# @opt_format; # from option parsing
# %format; # used format

my @opt_types;
my $isoDateTime;


# Preset options
%options = (
    "e"                  => \$opts{'e'},
    "iso_date_time"      => \$isoDateTime,
    "m"                  => \$opts{'m'},
    "uucp_mung"          => \$opts{'m'},
    "q"                  => \$opts{'q'},
    "u=i"                => \$opts{'u'},
  ## extra
    "show_users"         => \$opts{'show_users'},
);


## Module loader

# General
require "pflogstats-common-support.pm";

# Optional common modules
require "pflogstats-common-profiling.pm";

# Optional intermediate XML storage
require "pflogstats-common-intermediatexml.pm";

# Features/Enhancements
require "pflogstats-statistics-accounting.pm";
require "pflogstats-statistics-antivirus.pm";
require "pflogstats-statistics-accpopimap.pm";
require "pflogstats-statistics-uce.pm";
require "pflogstats-statistics-reject.pm";
require "pflogstats-extensions-addressmapping.pm";
require "pflogstats-extensions-verpmung.pm";
require "pflogstats-extensions-networking.pm";
require "pflogstats-statistics-spamassassin.pm";


## Hook 'early_begin'
for my $p_hook (keys %{$hooks{'early_begin'}}) {
	&{$hooks{'early_begin'}->{$p_hook}};
};

## Print options (debug)
#for my $key (keys %options) {
#	print $key . "\n";
#};
#exit 0;

## Print types (debug)
#for my $type (keys %types) {
#	print $type . "\n";
#};
#exit 0;

# Number formats
use vars qw{%numberformat};

# Time range of logdata
use vars qw{$timemin $timemax};
my ($time);


###
#
#
###
# ATTENTION: not updated:
# Usage:
#    pflogstats.pl -[eq] [-d <today|yesterday>] [-h <cnt>] [-u <cnt>]
#        [--verp_mung[=<n>]] [--verbose_msg_detail] [--iso_date_time]
#        [-m|--uucp_mung] [-i|--ignore_case] [--smtpd_stats] [--mailq]
#        [--problems_first] [--rej_add_from] [--no_bounce_detail]
#        [--no_deferral_detail] [--no_reject_detail] [file1 [filen]]
#
#    pflogstats.pl -[help|version]
#
# Options:
#
#    -e             extended (extreme? excessive?) detail - emit detailed
#                   reports.  At present, this includes only a per-message
#                   report, sorted by sender domain, then user-in-domain,
#                   then by queue i.d.
#
#                   WARNING: the data built to generate this report can
#                   quickly consume very large amounts of memory if a lot
#                   of log entries are processed!
#
#    --iso_date_time
#
#                   For summaries that contain date or time information, use
#                   ISO 8601 standard formats (CCYY-MM-DD and HH:MM), rather
#                   than "Mon DD CCYY" and "HHMM".
#
#    -m             modify (mung?) UUCP-style bang-paths
#    --uucp_mung
#
#                   This is for use when you have a mix of Internet-style
#                   domain addresses and UUCP-style bang-paths in the log.
#                   Upstream UUCP feeds sometimes mung Internet domain
#                   style address into bang-paths.  This option can
#                   sometimes undo the "damage".  For example:
#                   "somehost.dom!username@foo" (where "foo" is the next
#                   host upstream and "somehost.dom" was whence the email
#                   originated) will get converted to
#                   "foo!username@somehost.dom".  This also affects the
#                   extended detail report (-e), to help ensure that by-
#                    domain-by-name sorting is more accurate.
#
#
#    -q             quiet - don't print headings for empty reports (note:
#                   headings for warning, fatal, and "master" messages will
#                   always be printed.)
#
#
#    --verbose_msg_detail
#
#                   For the message deferral, bounce and reject summaries:
#                   display the full "reason", rather than a truncated one.
#                   Note: this can result in quite long lines in the report.
#
#    If no file(s) specified, reads from stdin.  Output is to stdout.
#
# Typical usage:
#    Produce a report of previous day's activities:
#        pflogsumm.pl -d yesterday /var/log/syslog
#    A report of prior week's activities (after logs rotated):
#        pflogsumm.pl /var/log/syslog.1
#    What's happened so far today:
#        pflogsumm.pl -d today /var/log/syslog
#
# Debug/developent options:
#   -t|--type test_verp_mung
#     Test addresses given on stdin for verp_mung rewriting
#
# Notes:
#
#    -------------------------------------------------------------
#    IMPORTANT: Pflogsumm makes no attempt to catch/parse non-
#               postfix/vmailer daemon log entries.  (I.e.: Unless
#               it has "postfix/" or "vmailer/" in the log entry,
#               it will be ignored.)
#    -------------------------------------------------------------
#
#    The "-c <cnt>" option is gone.  Use "--top <cnt>" and/or "-u <cnt>"
#    instead.
#
#    For display purposes: integer values are munged into "kilo" and
#    "mega" notation as they exceed certain values.  I chose the
#    admittedly arbitrary boundaries of 512k and 512m as the points
#    at which to do this--my thinking being 512x was the largest
#    number (of digits) that most folks can comfortably grok
#    at-a-glance.  These are "computer" "k" and "m", not 1000 and
#    1,000,000.  You can easily change all of this with some
#    constants near the beginning of the program.
#
#    "Items-per-day" reports are not generated for single-day
#    reports.  For multiple-day reports: "Items-per-hour" numbers
#    are daily averages (reflected in the report headings).
#
#    It's important that the logs are presented to pflogsumm in
#    chronological order so that message sizes are available when
#    needed.


#### Reviewed main options ####
my @mainhelptext;

# Help
push @mainhelptext, "    [--help|-h|-?]            Display help/usage message";
$options{'help|h|?'}  = \$opts{'help'};

# Type
my $temp = "    [--type|-t <type>]        Do type of statistics (more than one can be specified)\n                               Default: 'default' (pflogsumm-like-only)\n                               More examples:";
$options{"type|t=s"}  = \@opt_types;
foreach my $type (keys %types ) {
	$temp .= " '" . $type . "'";	

};
push @mainhelptext, $temp;

# Time range
push @mainhelptext, "    [--range|-r <timerange>]  Timerange of collecting data\n                               Default: all data\n                               More examples: 'yesterday' 'today'  'Mar 1'";
$options{"range|r=s"}  = \$opts{'range'};

# Table sort
push @mainhelptext, "    [--sort <sorttype>]       Sort output\n                               Default: 'alpha'\n                               More examples: 'maxmin' 'minmax'";
$options{"sort=s"}  = \$opts{'sort_type'};

# Ignore case
push @mainhelptext, "    [--ignore_case|-i]        Handle complete email address in a case-insensitive manner\n                               Default: lower-cases only the host and domain parts\n                               If used, entire email address will be lower-cased";
$options{"ignore_case|i"}  = \$opts{'i'};

# Version
push @mainhelptext, "    [--version]               Displays version of main program and modules";
$options{"version"}  = \$opts{'version'};

# Variables and constants used throughout pflogsumm
use vars qw(
    $progName
    $usageMsg
    $divByOneKAt $divByOneMegAt $oneK $oneMeg
    @monthNames %monthNums $thisYr $thisMon
    $msgCntI $msgSizeI $msgDfrsI $msgDlyAvgI $msgDlyMaxI
	%numberformat @opt_format %format
);

# Some constants used by display routines.  I arbitrarily chose to
# display in kilobytes and megabytes at the 512k and 512m boundaries,
# respectively.  Season to taste.
$divByOneKAt   = 524288;	# 512k
$divByOneMegAt = 536870912;	# 512m
$oneK          = 1024;		# 1k
$oneMeg        = 1048576;	# 1m

# Constants used throughout pflogsumm
@monthNames = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
%monthNums = qw(
    Jan  0 Feb  1 Mar  2 Apr  3 May  4 Jun  5
    Jul  6 Aug  7 Sep  8 Oct  9 Nov 10 Dec 11);
($thisMon, $thisYr) = (localtime(time()))[4,5];
$thisYr += 1900;


#
# Variables used only in main loop
#
# Per-user data
my (%recipUser, $recipUserCnt);
my (%sendgUser, $sendgUserCnt);
# Per-domain data
my (%recipDom, $recipDomCnt);	# recipient domain data
my (%sendgDom, $sendgDomCnt);	# sending domain data
# Indexes for arrays in above
$msgCntI    = 0;	# message count
$msgSizeI   = 1;	# total messages size
$msgDfrsI   = 2;	# number of defers
$msgDlyAvgI = 3;	# total of delays (used for averaging)
$msgDlyMaxI = 4;	# max delay

my (
    $cmd, $qid, $addr, $size, $relay, $status, $delay,
    $dateStr,
    %panics, %fatals, %warnings, %masterMsgs,
    %msgSizes,
    %deferred, %bounced,
    %noMsgSize, %msgDetail,
    $msgsRcvd, $msgsDlvrd, $sizeRcvd, $sizeDlvrd,
    $msgMonStr, $msgMon, $msgDay, $msgTimeStr, $msgHr, $msgMin, $msgSec,
    $msgYr,
    $revMsgDateStr, $dayCnt, %msgsPerDay,
    %rejects, $msgsRjctd,
    %rcvdMsg, $msgsFwdd, $msgsBncd,
    $msgsDfrdCnt, $msgsDfrd, %msgDfrdFlgs,
    %connTime, %smtpdPerDay, %smtpdPerDom, $smtpdConnCnt, $smtpdTotTime,
    %smtpMsgs
);
$dayCnt = $smtpdConnCnt = $smtpdTotTime = 0;

# time range of log data
my %timerange;

# Messages received and delivered per hour
my @rcvPerHr = qw(0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0);
my @dlvPerHr = @rcvPerHr;
my @dfrPerHr = @rcvPerHr;	# defers per hour
my @bncPerHr = @rcvPerHr;	# bounces per hour
my @rejPerHr = @rcvPerHr;	# rejects per hour
my $lastMsgDay = 0;

# "doubly-sub-scripted array: cnt, total and max time per-hour
# Gag - some things, Perl doesn't do well :-(
my @smtpdPerHr;
$smtpdPerHr[0]  = [0,0,0]; $smtpdPerHr[1]  = [0,0,0]; $smtpdPerHr[2]  = [0,0,0];
$smtpdPerHr[3]  = [0,0,0]; $smtpdPerHr[4]  = [0,0,0]; $smtpdPerHr[5]  = [0,0,0];
$smtpdPerHr[6]  = [0,0,0]; $smtpdPerHr[7]  = [0,0,0]; $smtpdPerHr[8]  = [0,0,0];
$smtpdPerHr[9]  = [0,0,0]; $smtpdPerHr[10] = [0,0,0]; $smtpdPerHr[11] = [0,0,0];
$smtpdPerHr[12] = [0,0,0]; $smtpdPerHr[13] = [0,0,0]; $smtpdPerHr[14] = [0,0,0];
$smtpdPerHr[15] = [0,0,0]; $smtpdPerHr[16] = [0,0,0]; $smtpdPerHr[17] = [0,0,0];
$smtpdPerHr[18] = [0,0,0]; $smtpdPerHr[19] = [0,0,0]; $smtpdPerHr[20] = [0,0,0];
$smtpdPerHr[21] = [0,0,0]; $smtpdPerHr[22] = [0,0,0]; $smtpdPerHr[23] = [0,0,0];


## Main start
my $mailqCmd = "mailq";
$usageMsg =
    "usage: $progName -[eq]
       [--iso_date_time]
       [-m|--uucp_mung] [-i|--ignore_case]
       [file1 [filen]]

     usage for normal statistics: $progName

     general
      [--show_users]               Display statistics also per user

";

# Some pre-inits for convenience
$isoDateTime = 0;	# Don't use ISO date/time formats

sub print_help() {
	print "$progName $release\n";
	print $copyright . "\n\n";

	print STDERR "  Options (reviewed):\n\n";
	foreach my $line (@mainhelptext) {
		print STDERR $line . "\n\n";
	};
	print STDERR "\n";

	print STDERR "  Options from included modules (new style):\n\n";

	## Hook 'help'
	for my $p_hook (sort keys %{$hooks{'help'}}) {
		my $helpstring = &{$hooks{'help'}->{$p_hook}};
		print STDERR "    Options from module '" . $p_hook . "':";
		print STDERR $helpstring . "\n";
	};

	print STDERR "  Options (still not reviewed):\n\n";
	print STDERR $usageMsg . "\n\n";
};

if (defined $main::opts{'verbose'}) {
	print "Calling options: " . "@ARGV" . "\n";
};

my $ret = GetOptions(%options);

if (! $ret ) {
	print "Use help to see more\n";
	exit 1;
};

# Print help or version
if(defined($opts{'help'})) {
	print_help();
	exit 0;
};
if(defined($opts{'version'})) {
	print "$progName $release\n";
	print $copyright . "\n\n";
	exit 0;
};


#### Check values of given options

##  Check sort option
if (! defined $opts{'sort_type'}) {
	# Default
	$opts{'sort_type'} = "alpha";
};
if ($opts{'sort_type'} ne "alpha" && $opts{'sort_type'} ne "maxmin" && $opts{'sort_type'} ne "minmax") {
	die "ERROR: Unsupported sort type: " . $opts{'sort_type'} . "\n";
};


# internally: 0 == none, undefined == -1 == all
$opts{'h'} = -1 unless(defined($opts{'h'}));
$opts{'u'} = -1 unless(defined($opts{'u'}));



## Hook 'checkoptions'
for my $p_hook (keys %{$hooks{'checkoptions'}}) {
	&{$hooks{'checkoptions'}->{$p_hook}};
};


## Parse and check type
if ( $#opt_types < 0 ) {
	$types{'default'} = 1;
} else {
	foreach my $type ( @opt_types ) {
		print STDERR "DEBUG: Got type: " . $type . "\n" if ($opts{'debug'});
		if (defined $types{$type}) {
			print STDERR "DEBUG: valid type: " . $type . "\n" if ($opts{'debug'});
			$types{$type} = 1;
		} else {
			die "ERROR: unsupported type: " . $type . "\n";
		};
	};
};
	

$dateStr = get_datestr($opts{'range'}) if(defined($opts{'range'}));


print "\nStatistics generated by: $progName $release";

if (defined $main::opts{'verbose'}) {
	print "\n" . $copyright . "\n";
	print " Included modules:\n";

	for my $module ( sort { $moduleinfo{$a}->{'name'} cmp $moduleinfo{$b}->{'name'} } sort { $moduleinfo{$a}->{'type'} cmp $moduleinfo{$b}->{'type'} } keys %moduleinfo) {
		print "  " . $moduleinfo{$module}->{'type'} . "/" . $moduleinfo{$module}->{'name'} . ":" .  $moduleinfo{$module}->{'version'} . "\n";
	};
	print "\n";
} else {
	my $modulesum = 0;
	for my $module ( keys %moduleinfo) {
		$modulesum += $moduleinfo{$module}->{'version'};
	};
	printf " (module version sum: %0.2f)\n", $modulesum;
	print $copyright . "\n";
};


## Any data to read from stdin?
#if (-t) {
#	# Skip log file parsing
#	print "\nINFO: no data given, proceed without parsing\n";
#	goto("LABEL_end_logfileparsing");
#};

## Show timerange on log file parsing
if ( defined($opts{'range'}) ) {
	print "Statistics generated for timerange: $dateStr\n";
};
print "\n\n";

## Hook 'beforemainloopstarts'
for my $p_hook (keys %{$hooks{'beforemainloopstarts'}}) {
	&{$hooks{'beforemainloopstarts'}->{$p_hook}};
};


## Start parsing logfile #################################################
print "DEBUG: start parsing logfile\n" if ($opts{'debug'});

## Hook 'loop_beforestart'
for my $p_hook (keys %{$hooks{'loop_beforestart'}}) {
	&{$hooks{'loop_beforestart'}->{$p_hook}};
};

while(<>) {
	chomp;
	$_ =~ s/
$//g; # Remove trailing CR
	$~ =~ s/^[[:space:][:cntrl:]]+$//g; # Remove spaces and ctrl chars only

	next if (length($_) == 0); # skip empty lines

	if ( $types{'test_verp_mung'} != 0 ) {
		die "Currently not supported at the moment";

		# Debug/developing: test verp_mung code
		my $addr = lc($_);

		print "\n" . $addr . "\n";

		$opts{'verpMung'} = 1;
		#my $verpmung1 = do_verp_mung($addr);
		# print " -> " . $verpmung1 . "\n";

		$opts{'verpMung'} = 2;
		#my $verpmung2 = do_verp_mung($addr);
		# print " -> " . $verpmung2 . "\n";

		next; # No others here for testing
	};

	## Now starting here, only a maillog is valid!

	# Skip not selected date
	next if (defined($dateStr) && ! /^$dateStr/);

	# Extract date & time
	($msgMonStr, $msgDay, $msgTimeStr) = /^(...)\s+([0-9]+)\s(..:..:..)\s.*/;

	if (! defined $msgMonStr || ! defined $msgDay || ! defined $msgTimeStr) {
		warn "WARNING(skipped-line): Date&Time in logline is not valid: $_";
		next;
	};
	
	# snatch out log entry date & time
	($msgHr, $msgMin, $msgSec) = split(/:/, $msgTimeStr);
	$msgMon = $monthNums{$msgMonStr};
	$msgYr = $thisYr; --$msgYr if($msgMon > $thisMon);

	if (! defined $msgHr || ! defined $msgMin || ! defined $msgSec ) {
		warn "WARNING(skipped-line): Date&Time in logline is not valid: $_";
		next;
	};

	# Calculate Unixtime
	$time = timelocal( $msgSec, $msgMin, $msgHr, $msgDay, $msgMon, $msgYr );

	# Catch min/max times for late timerange display
	if (! defined $timemin || ! defined $timemax ) {
		# initial values
		if (! defined $timemin) { $timemin = $time };
		if (! defined $timemax) { $timemax = $time };
	} else {
		# get min/max
		if    ($time < $timemin) { $timemin = $time; }
		elsif ($time > $timemax) { $timemax = $time; };
	};

	# Hook "loglineparser"
	for my $p_hook (keys %{$hooks{'loglineparser'}}) {
		&{$hooks{'loglineparser'}->{$p_hook}} (\$time, \$_);
	};
}

## Hook 'loop_afterfinish'
for my $p_hook (keys %{$hooks{'loop_afterfinish'}}) {
	&{$hooks{'loop_afterfinish'}->{$p_hook}} (\$_);
};

print "DEBUG: end parsing logfile\n" if ($opts{'debug'});


##### end of log file parsing

if ( $types{'test_verp_mung'} != 0 ) {
	goto "LABEL_end";
};

## Hook 'register_intermediate_storage' (register global data)
$timerange{'timemin'} = $timemin;
$timerange{'timemax'} = $timemax;
for my $p_hook (keys %{$main::hooks{'register_intermediate_data'}}) {
	&{$main::hooks{'register_intermediate_data'}->{$p_hook}} ("timerange", \%timerange);
};

LABEL_end_logfileparsing:

## Hook 'intermediate_storage'
for my $p_hook (keys %{$hooks{'intermediate_storage'}}) {
	&{$hooks{'intermediate_storage'}->{$p_hook}} ();
};


## explicit disable of printing statistics
if (defined $format{'none'}) {
	goto "LABEL_end";
};

## Hook 'before_print_result'
for my $p_hook (keys %{$hooks{'before_print_result'}}) {
	&{$hooks{'before_print_result'}->{$p_hook}} ();
};

## Hook 'retrieve_intermediate_data' (retrieve global data)
for my $p_hook (keys %{$main::hooks{'retrieve_intermediate_data'}}) {
	&{$main::hooks{'retrieve_intermediate_data'}->{$p_hook}} ("timerange", \%timerange);
};

$timemin = $timerange{'timemin'};
$timemax = $timerange{'timemax'};

## Hook 'print_result'
for my $p_hook (keys %{$hooks{'print_result'}}) {
	&{$hooks{'print_result'}->{$p_hook}};
};

## Hook 'printstatistics'
if (defined $main::opts{'printstatistics'}) {
	for my $p_hook (keys %{$hooks{'print_additional_statistics'}}) {
		&{$hooks{'print_additional_statistics'}->{$p_hook}};
	};
};



LABEL_end:

## Hook 'final_end'
for my $p_hook (keys %{$hooks{'final_end'}}) {
	&{$hooks{'final_end'}->{$p_hook}};
};



##################### End of main program


# return a date string to match in log
sub get_datestr {
    my $dateOpt = $_[0];
    my ($t_mday, $t_mon);

    my $aDay = 60 * 60 * 24;

    my $time = time();
    if($dateOpt eq "yesterday") {
	$time -= $aDay;
    	($t_mday, $t_mon) = (localtime($time))[3,4];
    } elsif($dateOpt eq "today") {
    	($t_mday, $t_mon) = (localtime($time))[3,4];
    } else {
	my ($m, $d) = split / +/, $dateOpt;
	if (! defined $monthNums{$m} ) {
		die "Illegal month string in: " . $dateOpt . "\n";
	};
	if ($d < 1 || $d >31) {
		die "Illegal day in: " . $dateOpt . "\n";
	};
	$t_mday = $d;
	$t_mon = $monthNums{$m};
    }

    return sprintf("%s %2d", $monthNames[$t_mon], $t_mday);
}