/*
** Modular Logfile Analyzer
** Copyright 2000 Jan Kneschke <jan@kneschke.de>
**
** Homepage: http://www.modlogan.org
**

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version, and provided that the above
    copyright and permission notice is included with all distributed
    copies of this or derived software.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA

**
** $Id: main.c,v 1.115 2004/08/27 20:07:37 ostborn Exp $
*/

/**
 * \mainpage ModLogAn API Documenation
 *
 * \section Introduction
 * Add docs.
 *
 */

#include <stdlib.h>
#include <stdio.h>
#include <time.h>
#include <sys/time.h>
#include <unistd.h>
#include <dirent.h>
#include <errno.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <assert.h>

#include <zlib.h>

#include "config.h"

#include "gettext.h"

#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif

#ifdef HAVE_PWD_H
#include <pwd.h>
#ifdef HAVE_GRP_H
#include <grp.h>
#define RUN_AS_USER
#endif
#endif

#include "mrecord.h"
#include "mhistory.h"
#include "mstate.h"
#include "mlocale.h"
#include "mconfig.h"
#include "mplugins.h"
#include "mdatatypes.h"
#include "misc.h"
#include "datatypes/webhist/datatype.h"
#include "datatypes/mailhist/datatype.h"
#include "datatypes/state/datatype.h"
#include "datatypes/query/datatype.h"
#include "datatypes/string/datatype.h"
#include "datatypes/count/datatype.h"

#include "mresolver.h"

#define MSTATE_WRITE 1

size_t mem_mhash_size = 0;
size_t mem_mhash_count = 0;
size_t mem_mlist_size = 0;
size_t mem_mlist_count = 0;
size_t mem_mdata_size = 0;
size_t mem_mdata_count = 0;

size_t mem_mdata_type_count[M_DATA_TYPE_IPPLWATCH] = { 0, };

void show_version() {
	printf("%s %s\n", PACKAGE, VERSION);
	fflush(stdout);
}

void show_header() {
	printf("%s %s\n", PACKAGE, VERSION);
	fflush(stdout);
}

void show_usage() {
	show_header();

	printf("Options:\n");
	printf(" -c <configfile>    filename of the configfile\n");
	printf(" -r                 force execution as root\n");
	printf(" -o <section>:<key>=<value>\n");
	printf("                    set a config-option in section <section>\n");
#ifdef RUN_AS_USER
	printf(" -u <username>      force execution as username (need -r)\n");
#endif
#ifdef HAVE_GETOPT_LONG
	printf(" -h --help          this help screen\n");
	printf(" -v --version       display version\n");
#else
	printf(" -h                 this help screen\n");
	printf(" -v                 display version\n");
#endif
};

int restore_internal_state (mconfig *conf, mlist *state_list) {
	char *fn;
	char buf[255];
	FILE *split_file;
	int empty = 1;
	
	if (conf->statedir == NULL) {
		fprintf(stderr, "%s.%d: global:statdir is empty\n", __FILE__, __LINE__);
		return -1;
	}

	fn = malloc(strlen(conf->statedir) + strlen("/modlogan.statefiles") + 1);
	strcpy(fn, conf->statedir);
	strcat(fn, "/modlogan.statefiles");

	if ((split_file = fopen(fn, "r")) == NULL) {
		M_DEBUG2(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_ERRORS,
			 "NOTICE: can't open %s: %s, <- ignore this message on the first run\n", fn, strerror(errno));
		
		free(fn);
		return 0;
	}
	
	while (fgets(buf, sizeof(buf), split_file)) {
		mstate *state = NULL;
		mlist *hist = mlist_init();
		
		empty = 0;
		
		/* remove the \n */
		buf[strlen(buf)-1] = '\0';

		M_DEBUG2(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
			"reading statefile from %s/%s/mla.state.xml\n", conf->statedir, buf);

		if (history_read(conf, hist, buf)) {
			fprintf(stderr, "%s.%d: hist failed ?\n", __FILE__, __LINE__);
			mlist_free(hist);
			hist = NULL;
		}

		if (conf->incremental) {
			state = mstate_init();

			if (mstate_read(conf, state, 0, 0, buf)) {
				M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
					 "reading state failed");
				
				mstate_free(state);
				state = NULL;
			}
		}

		if (!hist || (conf->incremental && !state)) {
			if (hist) {
				mlist_free(hist);
			} else {
				M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
					 "reading history failed");
			}

			if (state) {
				mstate_free(state);
			} else if (conf->incremental) {
				M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
					 "reading state failed");
			}

			state = NULL;
			hist = NULL;
		} else {
			mdata *data;
			const char *key = splaytree_insert(conf->strings, buf);
			
			data = mdata_State_create(key, state, hist);
			mlist_insert(state_list, data);

			M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
				"state restored for '%s'\n", buf);
		}
	}
	
	fclose(split_file);
	
	if (empty) {
		fprintf(stderr, "%s.%d: \n"
			"  modlogan.statefiles (%s) was found, but the file is empty\n"
			"  looks like modlogan died in the previous run while writing the output\n", 
			__FILE__, __LINE__, fn);
		free(fn);
		return -1;
	} 
	
	free(fn);
	
	return 0;
}



typedef struct {
	mlogrec *record;
	int rec_state;
} mgetrec;


int get_next_record (mconfig *conf, mlogrec *rec) {
	mplugin *func;
	int i, j;
	int state = M_RECORD_EOF;
	int last_rec_state = M_RECORD_NO_ERROR;
	static mgetrec **recs = NULL;
	static mgetrec **readahead = NULL;
	time_t t = -1;
	int ndx = -1;
	time_t rt = -1;
	int rndx = -1;

	if (recs == NULL && conf->plugin_count) {
		recs = malloc(sizeof(mgetrec *) * conf->plugin_count);
		for (i = 0; i < conf->plugin_count; i++) {
			recs[i] = malloc(sizeof(mgetrec));
			recs[i]->record = mrecord_init();
			recs[i]->rec_state = M_RECORD_EOF;
		}

	}

	if (readahead == NULL) {
		readahead = malloc(sizeof(mgetrec *) * conf->read_ahead_limit);
		for (i = 0; i < conf->read_ahead_limit; i++) {
			readahead[i] = malloc(sizeof(mgetrec));
			readahead[i]->record = mrecord_init();
			readahead[i]->rec_state = M_RECORD_EOF;
		}

	}

	/* fill the read-ahead-buffer */
	for (j = 0; j < conf->read_ahead_limit; j++) {
		if (last_rec_state != M_RECORD_EOF &&
		    readahead[j]->record->timestamp == 0) {
			/* slot is free */

			ndx = -1;
			t = -1;
			/* check the plugins for new records */
			for (i = 0; i < conf->plugin_count; i++) {
				func = ((mplugin **)(conf->plugins))[i];
				conf->plugin_conf = func->config;

				if (func->get_next_record) {
					if (recs[i]->record->timestamp == 0) {
						recs[i]->rec_state = func->get_next_record(conf, recs[i]->record);
						
						if (recs[i]->record->timestamp < 0) {
							fprintf(stderr, "%s.%d: %ld %ld\n", 
								__FILE__, __LINE__, 
								recs[i]->record->timestamp,
								t);
						}

						last_rec_state = recs[i]->rec_state;

						switch(recs[i]->rec_state) {
						case M_RECORD_NO_ERROR:
							/* start the resolver for this record */
							resolver_start(conf, recs[i]->record);
							break;
						case M_RECORD_SKIPPED:
						case M_RECORD_CORRUPT:
						case M_RECORD_IGNORED:
						case M_RECORD_HARD_ERROR:
#if 0
							M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_PARSING, M_DEBUG_LEVEL_WARNINGS,
								 "got %d\n",
								 recs[i]->rec_state);
#endif
							return recs[i]->rec_state;
						case M_RECORD_EOF:
							break;
						}
					}

#if 0					
					fprintf(stderr, "%s.%d: %d %d\n", 
						__FILE__, __LINE__, 
						recs[i]->record->timestamp,
						t);
#endif
					if ((recs[i]->record->timestamp > 0) &&    /* valid timestamp */
					    ((t == -1) ||                          /* first result */
					     (recs[i]->record->timestamp < t))) {  /* oldest timestamp */
						t = recs[i]->record->timestamp; 
						ndx = i;
					} else {
#if 0
						fprintf(stderr, "%s.%d: %ld %ld %d\n", 
							__FILE__, __LINE__, 
							recs[i]->record->timestamp,
							t, i);
#endif
					}
				}
			}

			/* got a new record */
			if (ndx != -1) {
				mrecord_move(readahead[j]->record, recs[ndx]->record);
				readahead[j]->rec_state = recs[ndx]->rec_state;
				
				if (recs[ndx]->record->timestamp < 0) {
					fprintf(stderr, "%s.%d: %ld %ld %d\n", 
						__FILE__, __LINE__, 
						recs[ndx]->record->timestamp,
						t, i);
				}
#if 0
				fprintf(stderr, "%s.%d: readahead: filled slot %d, %d\n",
					__FILE__, __LINE__,
					j, readahead[j]->record->ext_type);
#endif
			} else {
#if 0
				fprintf(stderr, "%s.%d: no record for me\n", __FILE__, __LINE__);
#endif
			}
		}

		/* select the oldest record from our readahead storage */
		if (readahead[j]->record->timestamp != 0) {
			if  (rt == -1 || readahead[j]->record->timestamp < rt) {
				rt = readahead[j]->record->timestamp;
				rndx = j;
			} else {
#if 0
				fprintf(stderr, "%s.%d: %ld - %ld\n", __FILE__, __LINE__, rt, readahead[j]->record->timestamp);
#endif
			}
		} else {
#if 0
			fprintf(stderr, "%s.%d - no record\n", __FILE__, __LINE__);
#endif
		}
	}

	if (rndx != -1) {
		mrecord_move(rec, readahead[rndx]->record);
		state = readahead[rndx]->rec_state;
		if (state == M_RECORD_EOF) {
			M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_PARSING, M_DEBUG_LEVEL_WARNINGS,
				 "got %d\n",
				 state);
		}
	} else {
#if 0
		M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_PARSING, M_DEBUG_LEVEL_WARNINGS,
			 "got - not records\n");
#endif
		
		
		/* got no record, kill myself */
		for (i = 0; i < conf->plugin_count; i++) {
			mrecord_free(recs[i]->record);
			free(recs[i]);
		}

		free(recs);
		recs = NULL;

		for (i = 0; i < conf->read_ahead_limit; i++) {
			mrecord_free(readahead[i]->record);
			free(readahead[i]);
		}

		free(readahead);
		readahead = NULL;
	}

	return state;
}

int generate_monthly_output(mconfig *conf, mstate *state, const char *subpath) {
	int i;
	mplugin *func;

	if (subpath) {
		/* set the splitter */
		const char *key = splaytree_insert(conf->strings, "splitby");
		mdata *data = mdata_String_create(key, subpath);
		
		if (data) {
			mhash_insert_replace(conf->variables, data);
		}
	}

	/* call all output-generators */
	for (i = 0; i < conf->plugin_count; i++) {
		func = ((mplugin **)(conf->plugins))[i];
		conf->plugin_conf = func->config;

		if (func->gen_report) {
			if (func->gen_report(conf, state, subpath))
				return -1;
		}
	}

	return 0;
}
int generate_history_output(mconfig *conf, mlist *history, const char *subpath) {
	int i;
	mplugin *func;

	if (subpath) {
		const char *key = splaytree_insert(conf->strings, "splitby");
		mdata *data = mdata_String_create(key, subpath);
		
		if (data) {
			mhash_insert_replace(conf->variables, data);
		}
	}

	for (i = 0; i < conf->plugin_count; i++) {
		func = ((mplugin **)(conf->plugins))[i];
		conf->plugin_conf = func->config;

		if (func->gen_history && func->gen_history(conf, history, subpath)) return -1;
	}

	return 0;
}

void dump (mlist *state_list) {
	mlist *s;
	int i;

	for (s = state_list; s; s = s->next) {
		mstate_web *w;
		if (!s->data) return;

		w = ((mstate *)(s->data->data.state.state))->ext;
		
		for (i = 0; i < 31; i++) {
			fprintf(stderr, "dump_list (%p)->'%s': %d: %ld\n", s, s->data->key, i, w->days[i].hits);
		}
	}
}

int insert_record(mconfig *conf, mlist *state_list, mlogrec *record) {
	int i = 0;
	mplugin *func;


	resolver_finish(conf, record);

	for (i = 0; i < conf->plugin_count; i++) {
		func = ((mplugin **)(conf->plugins))[i];
		conf->plugin_conf = func->config;

		if (func->insert_record)
			if (func->insert_record(conf, state_list, record))
				return -1;
	}

	return 0;
}

#ifdef RUN_AS_USER
/* Change to user privileges. May be only called by root.
 * It returns 0 on success.
 * -1 on any error.
 */
static int change_to_user(char *user) {
	struct passwd *pwd;

	/* Get user info */
	pwd = getpwnam (user);
	if (!pwd) return -1;

	/* We don't want to change to root user, -r is here for that. */
	if (!pwd->pw_uid) return -1;

#ifdef HAVE_SETGROUPS
#ifdef HAVE_GETGROUPLIST
#ifndef NGROUPS
#define NGROUPS 32
#endif
	{
		gid_t *groups;
		int ngroups = NGROUPS;

		/* Get user group list */
		groups = (gid_t *) malloc (ngroups * sizeof (gid_t));
		if (!groups) return -1;
		if (getgrouplist(user, pwd->pw_gid, groups, &ngroups) == -1) {
			groups = (gid_t *) realloc (groups, ngroups * sizeof (gid_t));
			if (!groups) return -1;
			if (getgrouplist(user, pwd->pw_gid, groups, &ngroups) == -1) {
				free(groups);
				return -1;
			}
		}

		/* Set additionnal groups */
		if (setgroups(ngroups, groups) == -1) {
			free(groups);
			return -1;
		}
		free(groups);

	}
#undef NGROUPS
#else
	/* Remove any additionnal group */
	if (setgroups(0, NULL) == -1) return -1;
#endif /* HAVE_GETGROUPLIST */
#endif /* HAVE_SETGROUPS */

	/* Set main group */
	if (setgid (pwd->pw_gid) == -1) return -1;
	/* Set user */
	if (setuid (pwd->pw_uid) == -1) return -1;

	/* Re-check ... */
	if (getuid() != pwd->pw_uid || geteuid() != pwd->pw_uid) return -1;
	return 0;
}
#endif /* RUN_AS_USER */

int main (int argc, char **argv) {
	mconfig *conf = NULL;
	mlogrec *rec = NULL;
	mlist *state_list = NULL;
	mlist *l;

	int ret, l_month = -1, l_year = -1, l_hour = -1, l_day = -1, i;
	char *conf_filename = NULL;
	long lines = 0, lines_corrupt = 0, lines_skipped = 0, lines_ignored = 0;
	int first_valid_line = 0;
	int root_check = 1;

	struct tm *tm;

	time_t l_stamp = -1;
	mtimer process_timer, post_process_timer, parse_timer, setup_timer;

#ifdef RUN_AS_USER
	char *run_as_user = NULL;
#define CMD_OPTIONS "c:hvro:u:"
#else
#define CMD_OPTIONS "c:hvro:"
#endif

#ifdef HAVE_GETOPT_LONG
	int option_index = 0;
	static struct option long_options[] = {
		{ "help", 0, NULL, 'h' },
		{ "version", 0, NULL, 'v' },
		{ "root", 0, NULL, 'r' },
		{ NULL, 0, NULL, 0 }
	};
#endif

	MTIMER_RESET(process_timer);
	MTIMER_RESET(post_process_timer);
	MTIMER_RESET(parse_timer);
	MTIMER_RESET(setup_timer);
	
	if (!(conf = mconfig_init())) {
		fprintf(stderr, "%s.%d: init config failed\n", __FILE__, __LINE__);
		return -1;
	}
	
	/* 
	 * command line options
	 * 
	 * -h/--help    help
	 * -r/--root    allow root
	 * -u <uid>     set new uid
	 * -v/--version version
	 * -c <file>    configfile
	 * 
	 * -o <section>:<key>=<value>
	 */
	
	opterr = 0;

	while (
#ifdef HAVE_GETOPT_LONG
	       (i = getopt_long(argc, argv, CMD_OPTIONS,
				long_options, &option_index)) != -1
	       
#else
	       (i = getopt(argc,argv,CMD_OPTIONS)) != EOF
#endif
	) {
		switch(i) {
		case 'c':
			conf_filename = optarg;
			break;
		case 'h':
			show_usage();
			exit( EXIT_SUCCESS);
		case 'v':
			show_version();
			exit( EXIT_SUCCESS);
		case 'r':
			root_check = 0;
			break;
#ifdef RUN_AS_USER
		case 'u':
			run_as_user = optarg;
			break;
#endif
		case 'o': {
			char *key = splaytree_insert(conf->strings, optarg);
			mdata *data = mdata_Count_create(key, 1, 0);
			/* add option to the config-file handling */
			
			mlist_insert_sorted(conf->cmdlineoptions, data);
			
			break;
		}
		default:
			show_usage();
			exit( EXIT_FAILURE);
		}
	}

	show_header();

	if (getuid() == 0 || geteuid() == 0) {
		if (root_check) {
			fprintf(stderr, "ModLogAn detected that it is running with root permissions.\n"
				"As root permissions are not directly required to run modlogan, \n"
				"modlogan decided do stop working with root permission.\n\n"
				"The reason for this decision is quite simple:\n"
				"ModLogAn doesn't want to be announced at BugTraq.\n"
				"Although this might be a little bit too defensive it is still better\n"
				"then nothing.\n\n"
				"If you still have to run modlogan with root permissions \n"
				"use the -r/--root switch which will disable this test\n");
			exit(-1);
		}
#ifdef RUN_AS_USER
		if (run_as_user) {
			if (change_to_user(run_as_user)) {
				fprintf(stderr, "ModLogAn failed to change to user '%s'. Exiting.\n", run_as_user);
				exit(-1);
			} else {
				fprintf(stderr, "ModLogAn is running as user '%s'.\n", run_as_user);
			}
		} else
#endif
			fprintf(stderr, "WARNING: ModLogAn is running as user 'root'.\n");

	}

	init_locale(conf);

	MTIMER_START(setup_timer);

	if (mconfig_read(conf, conf_filename)) {
		mconfig_free(conf);
		
		fprintf(stderr, "%s.%d: reading configfile failed - going down hard\n", __FILE__, __LINE__);
		return -1;
	}

	if (conf->show_options) {
		/* we only have to show some options */

		mconfig_free(conf);

		return 0;
	}

	state_list = mlist_init();

	if (restore_internal_state(conf, state_list)) {
		
		mconfig_free(conf);
		mlist_free(state_list);
		
		return -1;
	}
	
	rec = mrecord_init();

	if (conf->incremental) {
		l = state_list;

		while (l) {
			mdata *data = l->data;
			mstate *int_state;
			if (!data) break;

			int_state = data->data.state.state;
			if (int_state->timestamp > l_stamp) {
				l_stamp = int_state->timestamp;
				l_year = int_state->year-1900;
				l_month = int_state->month-1;
			}

			l = l->next;
		}
	}

	if (l_stamp != -1) {
		M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_VERBOSE,
			 "Ignoring all records before %s", ctime(&l_stamp));
	}
	
	M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_INIT, M_DEBUG_LEVEL_NONE,
		"NOTICE: startup - finished\n");

	MTIMER_STOP(setup_timer);
	MTIMER_CALC(setup_timer);

	printf("[");

	/* mainloop */
	MTIMER_START(parse_timer);
	while ((ret = get_next_record(conf, rec)) != M_RECORD_EOF) {
		MTIMER_STOP(parse_timer);
		MTIMER_CALC(parse_timer);

		MTIMER_START(process_timer);

		lines++;
		if (ret == M_RECORD_NO_ERROR) {
			/* HACK */
			if (rec->timestamp == 0) {
				fprintf(stderr, "%s.%d: line %ld returned no timestamp !! something strange will happen. Going down hard\n",
					__FILE__, __LINE__,
					lines);
				return -1;
			}

			/* set the first timestamp */
			if (l_month == -1) {
				tm = localtime(&(rec->timestamp));
				l_month = tm->tm_mon;
				l_year = tm->tm_year;
				l_stamp = rec->timestamp;
				l_hour = tm->tm_hour;
			}

			/* is the current timestamp below the timestamp
			 * of the last successfully parsed record ?
			 */
			if (rec->timestamp < l_stamp) {
				/* skip the record if we are in incremental mode 1 and no
				 * record has been parsed successfully yet
				 */
				if (first_valid_line == 0 && conf->incremental == 1) {
					lines_skipped++;
					MTIMER_STOP(process_timer);
					MTIMER_CALC(process_timer);
					MTIMER_START(parse_timer);
					continue;
				}
				M_DEBUG2(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_VERBOSE,
					"Out of Sequence - ignoring timestamp: %ld > %ld\n", l_stamp, rec->timestamp);

				rec->timestamp = l_stamp;
			}

			tm = localtime(&(rec->timestamp));

			/* set last day of month */
			l_day = tm->tm_mday;

			/* generate report if we have reached a threshold (semi-online mode) */
			if (conf->gen_report_threshold > 0 &&
				(lines - lines_corrupt + 1) % conf->gen_report_threshold == 0) {

				MTIMER_STOP(process_timer);
				MTIMER_CALC(process_timer);

				printf("]\n");

				MTIMER_START(post_process_timer);
				M_DEBUG2(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_WARNINGS,
					"o(t) writing month %02i - %04i\n", l_month+1, l_year+1900);


				for (l = state_list; l; l = l->next) {
					mdata *data = l->data;
					mdata *histdata = NULL;
					mstate *state;

					if (!data) break;

					state = data->data.state.state;

					switch (state->ext_type) {
					case M_STATE_TYPE_WEB:
						histdata = mdata_WebHist_create_by_state(state);
						break;
					case M_STATE_TYPE_MAIL:
						histdata = mdata_Mailhist_create_by_state(state);
						break;
					case M_STATE_TYPE_UNSET:
						continue;
					default:
						M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
							 "Unknown state-type: %d\n", state->ext_type);
						return (-1);
					}

					mlist_insert_replace(data->data.state.history, histdata);

					/* sort the history list */
					data->data.state.history = mlist_sort_full_by_string(data->data.state.history);
					
#if MSTATE_WRITE
					
					if (0 != mstate_write(conf, data->data.state.state,
							      M_STATE_WRITE_BY_MONTH,
							      *(data->key) ? data->key : NULL)) {
						fprintf(stderr, "%s.%d: writing state file failed\n", __FILE__, __LINE__);
						return -1;
					}
					history_write(conf, data->data.state.history, *(data->key) ? data->key : NULL);
#endif
					if (0 != generate_monthly_output(conf, data->data.state.state, *(data->key) ? data->key : NULL)) {
						fprintf(stderr, "%s.%d: output generation failed - no data this month?\n", __FILE__, __LINE__);
						/* return -1; */
					}
					if (0 != generate_history_output(conf, data->data.state.history, *(data->key) ? data->key : NULL)) {
						fprintf(stderr, "%s.%d: history generation failed - going down hard\n", __FILE__, __LINE__);
						return -1;
					}

				}
				MTIMER_STOP(post_process_timer);
				MTIMER_CALC(post_process_timer);


				printf("[");
				MTIMER_START(process_timer);
			}

			/* we have left a month, generate the reports */
			if (tm->tm_mon != l_month || tm->tm_year != l_year) {

				MTIMER_STOP(process_timer);
				MTIMER_CALC(process_timer);

				printf("]\n");

				MTIMER_START(post_process_timer);

				M_DEBUG2(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_WARNINGS,
					"o writing month %02i - %04i\n", l_month+1, l_year+1900);


				for (l = state_list; l; l = l->next) {
					mdata *data = l->data;
					mdata *histdata = NULL;
					mstate *state;

					if (!data) break;

					state  = data->data.state.state;

					switch (state->ext_type) {
					case M_STATE_TYPE_WEB:
						histdata = mdata_WebHist_create_by_state(state);
						break;
					case M_STATE_TYPE_MAIL:
						histdata = mdata_Mailhist_create_by_state(state);
						break;
					case M_STATE_TYPE_UNSET:
						if (state->ext) {
							M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
								 "no ext_type, but ext is set\n");
							return -1;
						}
						continue;
					default:
						M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
							 "Unknown state-type: %d\n", state->ext_type);
						return (-1);
					}

					mlist_insert_replace(data->data.state.history, histdata);

					/* sort the history list */
					data->data.state.history = mlist_sort_full_by_string(data->data.state.history);
					
					if (conf->debug_level > 3)
						fprintf(stderr, "o(t) -- state written: (...)/%s/\n", *(data->key) ? data->key : ".");
#if MSTATE_WRITE
					if (0 != mstate_write(conf, data->data.state.state,
							      M_STATE_WRITE_BY_MONTH,
							      *(data->key) ? data->key : NULL)) {
						fprintf(stderr, "%s.%d: writing state file failed\n", __FILE__, __LINE__);
						return -1;
					}
					history_write(conf, data->data.state.history, *(data->key) ? data->key : NULL);
#endif
					if (0 != generate_monthly_output(conf, data->data.state.state, *(data->key) ? data->key : NULL)) {
						fprintf(stderr, "%s.%d: output generation failed - no data this month?\n", __FILE__, __LINE__);
						/* return -1; */
					}
					if (0 != generate_history_output(conf, data->data.state.history, *(data->key) ? data->key : NULL)) {
						fprintf(stderr, "%s.%d: history generation failed - going down hard\n", __FILE__, __LINE__);
						return -1;
					}

					mstate_free(data->data.state.state);
					data->data.state.state = mstate_init();
				}

				tm = localtime(&(rec->timestamp));

				l_month = tm->tm_mon;
				l_year = tm->tm_year;

				MTIMER_STOP(post_process_timer);
				MTIMER_CALC(post_process_timer);

				for (i = 0; i < ((lines % 50000) / 1000); i++) printf(" ");
				printf("[");

				MTIMER_START(process_timer);
			}

			if (insert_record(conf, state_list, rec) != 0) {
				fprintf(stderr, "%s.%d: inserting record failed - going down hard\n", __FILE__, __LINE__);
				return -1;
			}

			first_valid_line = 1;

			l_stamp = rec->timestamp;
		} else if (ret == M_RECORD_SKIPPED) {
			lines_skipped++;
		} else if (ret == M_RECORD_IGNORED) {
			lines_ignored++;
		} else if (ret == M_RECORD_HARD_ERROR) {
			M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
				 "a hard error occured in line %ld - going down hard\n", lines);

			exit(-1);
		} else {
			lines_corrupt++;

			M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
				 "parser reported an error in line %ld\n", lines);
		}
		mrecord_reset(rec);

		/* cosmetics */
		if (lines % 1000 == 0) {
			printf(".");
			if (lines % (1000 * 50)== 0) {
				printf(" %8ld", lines);
				if (conf->debug_level > 1) {
					printf(" - %10.2f (%4ld, %4ld)",
					       (lines)/((MTIMER_GET_USER_MSEC(parse_timer)+MTIMER_GET_USER_MSEC(process_timer))/1000.0),
					       lines_corrupt,
					       lines_skipped);
				}
				printf("\n ");
			}
			fflush(stdout);
		}

		MTIMER_STOP(process_timer);
		MTIMER_CALC(process_timer);

		MTIMER_START(parse_timer);
	}

	MTIMER_STOP(parse_timer);
	MTIMER_CALC(parse_timer);

	printf("]\n");

	MTIMER_START(post_process_timer);

	if (l_month != -1) {
		FILE *split_file;
		char *fn;
		printf(" ");

		if (conf->debug_level > 0)
			printf("writing month %02i - %04i\n",l_month+1, l_year+1900);

		if (conf->debug_level > 1)
			fprintf(stderr, "%s.%d: writing the last month\n", __FILE__, __LINE__ );
#if MSTATE_WRITE
		/* create filename */
		fn = malloc(strlen(conf->statedir) + strlen("/modlogan.statefiles") + 1);
		strcpy(fn, conf->statedir);
		strcat(fn, "/modlogan.statefiles");
		if (NULL == (split_file = fopen(fn, "w"))) {
			fprintf(stderr, "%s.%d: %s\n",
				__FILE__, __LINE__,
				strerror(errno));
		} else {
			fclose(split_file);
		}
#endif

		for (l = state_list; l; l = l->next) {
			mdata *data = l->data;
			mdata *histdata = NULL;
			int faulty = 0;
			mstate *state;

			if (!data) break;

			state = data->data.state.state;
			
			if (conf->debug_level > 1)
				fprintf(stderr, "%s.%d: subpath = '%s'\n",
					__FILE__, __LINE__,
					data->key );

			switch (state->ext_type) {
			case M_STATE_TYPE_WEB:
				if (NULL == (histdata = mdata_WebHist_create_by_state(state))) {
					M_WP();
					
					return -1;
				}
				break;
			case M_STATE_TYPE_MAIL:
				histdata = mdata_Mailhist_create_by_state(state);
				break;
			case M_STATE_TYPE_TRAFFIC:
				break;
			case M_STATE_TYPE_IPPL:
				//histdata = mdata_Ipplhist_create_by_state(state);
				break;
			case M_STATE_TYPE_UNSET:
				if (state->ext) {
					M_DEBUG0(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
						 "no ext_type, but ext is set\n");
					return -1;
				}
				break;
			default:
				M_DEBUG1(conf->debug_level, M_DEBUG_SECTION_MAINLOOP, M_DEBUG_LEVEL_ERRORS,
					 "Unknown state-type: %d\n", state->ext_type);
				return -1;
			}
			
			if (histdata) mlist_insert_replace(data->data.state.history, histdata);

#if MSTATE_WRITE
			if (!faulty && mstate_write(conf, data->data.state.state, M_STATE_WRITE_DEFAULT, *(data->key) ? data->key : NULL)) {
				faulty = 1;
			}
			if (!faulty && history_write(conf, data->data.state.history, *(data->key) ? data->key : NULL)) {
				faulty = 1;
			}
			if (!faulty) {
				/* write the current directoy to the split file */
				if (NULL != (split_file = fopen(fn, "a+"))) {
					fprintf(split_file, "%s\n", data->key);
					fclose(split_file);
				}
			}
#endif

			if (state->ext_type != M_STATE_TYPE_UNSET) {
				/* generate reports */
				if (!faulty && 0 != generate_monthly_output(conf, data->data.state.state, *(data->key) ? data->key : NULL)) {
					fprintf(stderr, "%s.%d: output generation failed for %s: no data this month?\n", 
						__FILE__, __LINE__, data->key);
					faulty = 1;
				}
				
				/* generate history */
				if (!faulty && 0 != generate_history_output(conf, data->data.state.history, *(data->key) ? data->key : NULL)) {
					fprintf(stderr, "%s.%d: history generation failed for %s\n", 
						__FILE__, __LINE__, data->key);
					faulty = 1;
				}
			} else {
				fprintf(stderr, "%s.%d: skipped monthly / history output generation for %s, no data\n", 
						__FILE__, __LINE__, data->key);
			}
		}

		free(fn);
	}

	MTIMER_STOP(post_process_timer);
	MTIMER_CALC(post_process_timer);

	mlist_free(state_list);
	mrecord_free(rec);

	if (conf->debug_level > 0) {
		printf(" --> Setup       : Wall %10.2fs, User %10.2fs, System %10.2fs <--\n",
			MTIMER_GET_WALL_MSEC(setup_timer)/1000.0,
			MTIMER_GET_USER_MSEC(setup_timer)/1000.0,
			MTIMER_GET_SYSTEM_MSEC(setup_timer)/1000.0);
		printf(" --> Parse       : Wall %10.2fs, User %10.2fs, System %10.2fs <--\n",
			MTIMER_GET_WALL_MSEC(parse_timer)/1000.0,
			MTIMER_GET_USER_MSEC(parse_timer)/1000.0,
			MTIMER_GET_SYSTEM_MSEC(parse_timer)/1000.0);
		printf(" --> Process     : Wall %10.2fs, User %10.2fs, System %10.2fs <--\n",
			MTIMER_GET_WALL_MSEC(process_timer)/1000.0,
			MTIMER_GET_USER_MSEC(process_timer)/1000.0,
			MTIMER_GET_SYSTEM_MSEC(process_timer)/1000.0);
		printf(" --> Post-Process: Wall %10.2fs, User %10.2fs, System %10.2fs <--\n",
			MTIMER_GET_WALL_MSEC(post_process_timer)/1000.0,
			MTIMER_GET_USER_MSEC(post_process_timer)/1000.0,
			MTIMER_GET_SYSTEM_MSEC(post_process_timer)/1000.0);

		printf("%s: %.2f %s (%ld %s, %ld %s, %ld %s, %ld %s)\n",
		       "Throughput", (lines)/((MTIMER_GET_USER_MSEC(parse_timer)+MTIMER_GET_USER_MSEC(process_timer))/1000.0),
		       "rec/s",
		       lines, "records",
		       lines_corrupt, "corrupt records",
		       lines_skipped, "skipped records",
		       lines_ignored, "ignored records"
		       );
	}
#if 0	
	fprintf(stdout, "mhash: c: %d, s: %d\n", mem_mhash_count, mem_mhash_size);
	fprintf(stdout, "mlist: c: %d, s: %d\n", mem_mlist_count, mem_mlist_size);
	fprintf(stdout, "mdata: c: %d, s: %d\n", mem_mdata_count, mem_mdata_size);
	
	for (i = 0; i < M_DATA_TYPE_IPPLWATCH; i++) {
		fprintf(stdout, "mdata %d: c: %d\n", i, mem_mdata_type_count[i]);
	}
#endif
	mconfig_free(conf);
	return 0;
}