/* 1515, Tue 5 Jan 99

   FD_EXTRACT.C:  Extracts data from NeTraMet flow-data files
                     for plotting by GnuPlot

   Copyright (C) 1994-2002 by Nevil Brownlee
   CAIDA | University of Auckland */

/*
 * $Log: fd_extract.c,v $
 * Revision 1.1.1.2.2.9  2002/02/23 01:57:15  nevil
 * Moving srl examples to examples/ directory.  Modified examples/Makefile.in
 *
 * Revision 1.1.1.2.2.5  2000/08/08 19:39:05  nevil
 * Fix attr_ix bug when printing column info
 *
 * Revision 1.1.1.2.2.3  2000/06/06 03:38:07  nevil
 * Combine NEW_ATR with TCP_ATR, various bug fixes
 *
 * Revision 1.1.1.2  1999/10/03 21:06:14  nevil
 * *** empty log message ***
 *
 * Revision 1.1.1.1.2.3  1999/09/22 05:38:33  nevil
 * Improve code to work properly on 64-bit machines
 * - Add OS=ALPHA handling to configure.in
 * - Clean up the Alpha compiler warnings
 * - Change all the snmp-related code to use Bit32 instead of unsigned long
 *
 * Revision 1.1.1.1.2.2  1999/01/08 01:36:57  nevil
 * Implement TCPdata and DScodepoint attribute
 *
 * Revision 1.1.1.1.2.1  1998/11/26 04:13:49  nevil
 * Fix problems in fd_filter and fd_extract when dealing with 'new'
 * attributes, especially those used with NETFLOW. In particular:
 * - Make sure attribs[] is referred to via attr_ix[]
 * - Get the separators correctly from a #Format record (no quotes)
 * - fd_extract wasn't writing out its data for the last sample!
 *
 * Revision 1.1.1.1  1998/11/16 03:57:26  nevil
 * Import of NeTraMet 4.3b3
 *
 * Revision 1.1.1.1  1998/11/16 03:22:00  nevil
 * Import of release 4.3b3
 *
 * Revision 1.1.1.1.2.1  1998/11/11 23:14:35  nevil
 * Only include malloc.h if we HAVE_MALLOC_H
 *
 * Revision 1.1.1.1  1998/10/28 20:31:23  nevil
 * Import of NeTraMet 4.3b1
 *
 * Revision 1.1.3.2.2.1  1998/10/27 04:39:13  nevil
 * 4.3b1 release
 *
 * Revision 1.1.3.2  1998/10/18 23:44:03  nevil
 * Added Nicolai's patches, some 'tidying up' of the source
 *
 * Revision 1.1.3.1  1998/10/13 02:48:16  nevil
 * Import of Nicolai's 4.2.2
 *
 * Revision 1.1.1.1  1998/08/24 12:09:28  nguba
 * NetraMet 4.2 Original Distribution
 *
 * Revision 1.3  1998/05/07 04:28:53  rtfm
 * Implement NetFlowMet, the Cisco NetFlow RTFM meter
 */

#if HAVE_CONFIG_H
#include <ntm_conf.h>
#endif

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#if HAVE_MALLOC_H
# include <malloc.h>
#endif

#include "ausnmp.h"
#include "asn1.h"

#define EXTERN
#include "fd_data.h"
#include "nmc_c64.h"

struct col_info {
   int col_nbr;
   float scale;
   int stat_index;
   unsigned char att_nbr[1+N_ATTRIBS];
   unsigned int tag_reqd[1+MXTAGS];
   double value;
   };

EXTERN struct col_info *cols[MXCOLS+1];
EXTERN int ncols INIT(1);  /* Time is col 1 */
int stats_reqd = 0;

int tod_clock INIT(1),  /* Wall-clock hours by default */
   time_units INIT(EX_HOURS);
char *time_u_str INIT("hours");
float time_scale INIT(3600.0);

void scan_time(void)
{
   int n;
   if (!nexttoken()) return;
   n = getnbr();
   if (n == EX_ELAPSED || n == EX_CLOCK) {
      tod_clock = n == EX_CLOCK;
      if (!nexttoken()) return;
      n = getnbr();
      }
   switch (time_units = n) {
   case EX_SECS:
      time_scale = 1.0;  time_u_str = "seconds";
      break;
   case EX_MINS:
      time_scale = 60.0;  time_u_str = "minutes";
      break;
   case EX_HOURS:
      time_scale = 3600.0;  time_u_str = "hours";
      break;
   case EX_DAYS:
      time_scale = 24.0*3600.0;  time_u_str = "days";
      break;
   default:
      p_error("Time units not seconds, minutes, hours, days");
      }
   }

#define NSTATS  15
float stat_val[NSTATS];
char *stat_name[NSTATS] = {
   "aps", "apb", "mps", "mpb", "lsp",  /*  0 */
   "avi", "mni", "fiu", "frc", "gci",  /*  5 */
   "rpp", "tpp", "cpt", "tts", "tsu"   /* 10 */
   };

struct col_info *scan_col(void)
{
   struct col_info *c;
   int cn, a, n;
   char scbuf[4];

   if ((c = (struct col_info *)calloc(1, sizeof(struct col_info))) == NULL) {
      fprintf(stderr,"Failed to allocate memory for col info, exiting\n");
      exit(10);
      }
   c->stat_index = -1;  /* Attributes (not statistic) */
   cn = getnbr();  /* Get col nbr */
   c->col_nbr = cn;
   if (cn < 0 || cn > MXCOLS) {
      sprintf(ebuf, "Column number must be < %d", MXCOLS);
      p_error(ebuf);
      }
   n = getnbr();
   c->scale = 1.0;
   if (n == EX_SCALE) {  /* Scale (dividing) factor given */
      c->scale = (float)getnbr();
      n = getnbr();
      }
   else if (n == EX_KILO || n == EX_MEGA) {
      c->scale = n == EX_KILO ? 1.0E3 : 1.0E6;
      n = getnbr();
      }
   if (n == RF_STATS) {
      if (!nexttoken()) return;
      for (a = 0; a != 3; ++a) {
         scbuf[a] = ic;  nextchar();
         }
      for (a = 0; a != NSTATS; ++a) {
         if (strncmp(scbuf,stat_name[a],3) == 0) break;
         }
      if (a == NSTATS) p_error("Unrecognised statistic");
      else c->stat_index = a;
      stats_reqd = 1;
      return c;
      }
   else {
     for (a = 0; ; ++a) {
         if (n < 0 || n > LASTATTRIB || attr_ix[n] == 0) {
            sprintf(ebuf, "Attribute %d not allowed in column %d",n,cn);
            p_error(ebuf);
            }
         else c->att_nbr[a] = n;
         n = getnbr();  /* Get attribute */
         if (n == RF_TAG) break;
	 }
     for (;;) {
         n = getnbr();  /* Get tag nbr */
         if (n < 0 || n > MXTAGS) {
            sprintf(ebuf, "Tag number must be > 0 and < %d", MXTAGS);
            p_error(ebuf);
            }
         else c->tag_reqd[n] = 1;
         for (;;) {  /* Any more tags? */
            if (ic == ';') return c;
            if (isalnum(ic)) break;
            nextchar();
            }
	 }
      }
   }
 
struct flow_data {
   unsigned short ruleset;
   unsigned int flownum, starttime,
      upbc, dnbc, uppc, dnpc;  /* Counts from last sample */
   };
struct flow_data *flow;

int next_flow = 1, max_flows;

int main(int argc, char **argv)
{
   char *pb, *p;
   int n, a, b, i, x;
   FILE *input, *output;
   int fi, ruleset=0, sampleno=-1, recno=0;
   struct flow_stats in_set;
   struct col_info *cip;
   int hh,mm,ss, lasthh;
   float sample_time, start_time, time_offset, t;

   if (argc < 2) {
      fprintf(stderr,
         "%s columns_file [ input_file [ > output_file ]]\n", argv[0]);
      exit(-1);
      }

   memset(attr_ix, 0, sizeof(attr_ix));  /* Make attribute index */
   for (x = 0; x != SZ_ATTRIBS; ++x)
      attr_ix[attribs[x].index] = x; 

   if (!parse_open(argv[1])) {
      printf("   Couldn't open format file %s !!!\n", argv[1]);
      exit(-2);
      }

   listrules = 1;  listformat = 0;
   for (;;) {  /* Read the format file */
      do {  /* First char of a line */
	 nextchar();
	 if (ic == EOF) break;
	 } while (lic != '\n');
      if (ic == EOF) break;
      n = getnbr();  /* What kind of line is it? */
      if (n == RF_SET) {
         ruleset = getnbr();
         continue;
         }
      if (n == RF_COLUMN) {
         cip = scan_col();
         n = cip->col_nbr;
         if (cols[n] != NULL) {
            sprintf(ebuf,"Column %d already specified", n);
            p_error(ebuf);
	    }
         else cols[n] = cip;
         continue;
	 }
      if (n == RF_TIME) {
         scan_time();
         continue;
         }
      if (ic == EOF) break;
      p_error("Unexpected line in format file");
      }
   fclose(rfp);
   for (ncols = MXCOLS; ncols != 1;  --ncols) {
      if (cols[ncols] != NULL) break;
      }
   if (ncols == 1) p_error("No columns specified");
   else for (n = 2; n != ncols; ++n) {
      if (cols[2] == NULL) {
         sprintf(ebuf,"Column %d not specified", n);
         p_error(ebuf);
         }
      }
   if (rferrors != 0) {
      fprintf(stderr,"%d errors in format file!\n", rferrors);
      exit(10);
      }

   if (listrules) {
      fprintf(stderr,"Time:  %s %s\n\n",
         tod_clock ? "clock" : "elapsed", time_u_str);
      for (n = 2; n != ncols+1; ++n) {  /* List the columns */
         cip = cols[n];
         fprintf(stderr,"Column %d:  Scale=%g\n   ",
            cip->col_nbr,cip->scale);
         if (cip->stat_index != -1)
            fprintf(stderr,"Statistics %s",stat_name[cip->stat_index]);
         else {
            fprintf(stderr,"Attributes ");
            for (i = 0; ; ++i) {
               if ((a = cip->att_nbr[i]) == 0) break;
               fprintf(stderr, " %s", attribs[attr_ix[a]].name);
               }
            fprintf(stderr,"\n   Tags ");
            for (i = 1; i != MXTAGS+1; ++i) {
               if (cip->tag_reqd[i] == 0) continue;
               fprintf(stderr, " %d", i);
               }
            }
         fprintf(stderr,"\n\n");
         }
      }

   if (argc > 2) {
      if ((input = fopen(argv[2], "r")) == NULL) {
         fprintf(stderr, "Cant open %s:",argv[2]);
         perror("");
         exit(1);
         }
      }
   else input = stdin;

/* Read the header line and check it */

   if (fgets(inbuf, sizeof(inbuf), input) == NULL) {
      fprintf(stderr, "Initial read failed on input file\n");
      exit(10);
      }

   recno++;
   if (strncmp(inbuf, "##NeTraMet ", 10) != 0) {
      fprintf(stderr,
	      "Hmm ... this does not appear to be a NeTraMet file, exiting\n");
      exit(10);
      }
   in_set.MIBver = 4;  /* NeTraMet v4 or later */
   if (strncmp(inbuf+11, ver_str ":", 5) != 0) {
      fprintf(stderr, 
         "Caution: this program was written for version " ver_str " files,"
	 "continuing\n");
      if (strncmp(inbuf+11, "v3.", 3) == 0)
         in_set.MIBver = 3;
      }
   if ((p = strstr(inbuf+12, "flows")) == NULL) {
      fprintf(stderr, "Can't find maximum flow number! exiting\n");
      exit(10);
      }
   for (--p; isspace(*p); ) --p;
   for ( ; isdigit(*p); ) --p;
   sscanf(p,"%d",&max_flows);

   /* Allocate two extra flows since they start from 2 not 0 */

   if ((flow=(struct flow_data *)calloc(
         max_flows+2, sizeof(struct flow_data))) == NULL) {
      fprintf(stderr, "Failed to allocate memory for flow table, exiting\n");
      exit(10);
      }

   output = stdout;

   if (fgets(inbuf, sizeof(inbuf), input) == NULL) {
      fprintf(stderr,"2nd read failed on flows file\n");
      exit(10);
      }
   recno++;

   if (inbuf[1] != 'F') {  /* Format record */
      fprintf(stderr,"failed to find format record in flows file\n");
      exit(10);
      }
   else {
      ic = ' ';  rferrors = 0;  ibp = inbuf+8;  /* Scan format line */
      scan_format(&in_set,1,listformat);  /* Single_record */
      if (rferrors != 0) exit(1);
      }

   for (a = 1; a <= LASTATTRIB; ++a) {  /* Make ptrs to curr_flow attribs */
      x = attr_ix[a];
      if (x != 0) attribs[x].value = attrib_ptr(&curr_flow, a);
      }

   while (fgets(inbuf, sizeof(inbuf), input)) {
      ++recno;
      if (inbuf[0] == '#') {  /* Info record */
         if (stats_reqd && strncmp(&inbuf[1],"Stats: ",7) == 0) {
            sscanf(&inbuf[8],"aps=%g apb=%g mps=%g mpb=%g lsp=%g"
               " avi=%g mni=%g fiu=%g frc=%g gci=%g rpp=%g"
               " tpp=%g cpt=%g tts=%g tsu=%g",
               &stat_val[0],&stat_val[1],&stat_val[2],&stat_val[3],
               &stat_val[4],&stat_val[5],&stat_val[6],&stat_val[7],
               &stat_val[8],&stat_val[9],&stat_val[10],&stat_val[11],
               &stat_val[12],&stat_val[13],&stat_val[14]);
            }
         if (strncmp(&inbuf[1],"Time: ",6) == 0) {
            ++sampleno;
            /* sampleno = 0:  first Time, start time for file
                        = 1:  second Time, start of second sample
                        > 1:  output time (and stats) at beginning of sample,
                              together with counts accumulated since then */
            if (sampleno > 1) {
               fprintf(output,"%g ", t/time_scale);
               for (n = 2; n != ncols+1; ++n) {
                  cip = cols[n];
                  if (cip->stat_index != -1) fprintf(output," %g", 
                        stat_val[cip->stat_index]/cip->scale);
                  else fprintf(output," %g", cip->value/cip->scale);
		  }
	       fprintf(output,"\n");
	       }
            sscanf(&inbuf[7],"%d:%d:%d",&hh,&mm,&ss);
            sample_time = hh*3600.0 + mm*60.0 + ss;
            if (sampleno == 0) {  /* First time record */
               start_time = sample_time;
               time_offset = 0.0;  lasthh = hh;
	       }
            if (tod_clock) t = sample_time;
            else {
               if (hh < lasthh) {
                  time_offset += 24.0*3600.0;
                  }
               t = sample_time-start_time + time_offset;
	       }
            lasthh = hh;
            for (n = 2; n != ncols+1; ++n)
               cols[n]->value = 0.0;
	    }
         }
      else {  /* Flow record */
         pb = inbuf; 
         memset(&curr_flow, 0, sizeof(curr_flow)); /* Zero current values */

         /* Loop to get the values ... */
      
         for (i = 0; *pb && i != sizeof(in_set.format) && in_set.format[i];
               i++) {
            a = in_set.format[i];
            pb = get_value(attribs[attr_ix[a]].value, a, pb);
	    if (in_set.separator[i]) pb += strlen(in_set.separator[i]);
            }
         if (in_set.MIBver == 3) {
            curr_flow.LowPeerType = new_addr_type[curr_flow.LowPeerType];
            curr_flow.HighPeerType = new_addr_type[curr_flow.HighPeerType];
            }
         if ((fi=curr_flow.FlowIndex) > max_flows+1) {
	    fprintf(stderr,
	       "flow index (%d) greater than number of flows from "
	       "header (%d) at record number %d\n",
	       fi, max_flows, recno);
            exit(10);
            }
         if (ruleset == curr_flow.FlowRuleSet || ruleset == 0) {
            for (n = 2; n != ncols+1; ++n) {
               cip = cols[n];
               if (cip->tag_reqd[curr_flow.TagNbr]) {
                  for (b = 0; ; ++b) {
                     if ((a = cip->att_nbr[b]) == 0) break;
                     x = attr_ix[a];  /* Index into attribs[] */
                     if (attribs[x].len == 8)  /* Counter64 */
                        cip->value += double64p(attribs[x].value.c64val);
                     else  /* Integer (1, 2 or 4 bytes) */
                        cip->value += (double)*attribs[x].value.intval;
                     }
                  }
	       }
            }
         }
      }

   if (sampleno > 1) {  /* Dump data for last sample */
      fprintf(output,"%g ", t/time_scale);
      for (n = 2; n != ncols+1; ++n) {
         cip = cols[n];
         if (cip->stat_index != -1) fprintf(output," %g", 
               stat_val[cip->stat_index]/cip->scale);
         else fprintf(output," %g", cip->value/cip->scale);
         }
      fprintf(output,"\n");
      }
   }