/* ** Copyright (C) 2001-2007 by Carnegie Mellon University. ** ** @OPENSOURCE_HEADER_START@ ** ** Use of the SILK system and related source code is subject to the terms ** of the following licenses: ** ** GNU Public License (GPL) Rights pursuant to Version 2, June 1991 ** Government Purpose License Rights (GPLR) pursuant to DFARS 252.225-7013 ** ** NO WARRANTY ** ** ANY INFORMATION, MATERIALS, SERVICES, INTELLECTUAL PROPERTY OR OTHER ** PROPERTY OR RIGHTS GRANTED OR PROVIDED BY CARNEGIE MELLON UNIVERSITY ** PURSUANT TO THIS LICENSE (HEREINAFTER THE "DELIVERABLES") ARE ON AN ** "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY ** KIND, EITHER EXPRESS OR IMPLIED AS TO ANY MATTER INCLUDING, BUT NOT ** LIMITED TO, WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE, ** MERCHANTABILITY, INFORMATIONAL CONTENT, NONINFRINGEMENT, OR ERROR-FREE ** OPERATION. CARNEGIE MELLON UNIVERSITY SHALL NOT BE LIABLE FOR INDIRECT, ** SPECIAL OR CONSEQUENTIAL DAMAGES, SUCH AS LOSS OF PROFITS OR INABILITY ** TO USE SAID INTELLECTUAL PROPERTY, UNDER THIS LICENSE, REGARDLESS OF ** WHETHER SUCH PARTY WAS AWARE OF THE POSSIBILITY OF SUCH DAMAGES. ** LICENSEE AGREES THAT IT WILL NOT MAKE ANY WARRANTY ON BEHALF OF ** CARNEGIE MELLON UNIVERSITY, EXPRESS OR IMPLIED, TO ANY PERSON ** CONCERNING THE APPLICATION OF OR THE RESULTS TO BE OBTAINED WITH THE ** DELIVERABLES UNDER THIS LICENSE. ** ** Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie ** Mellon University, its trustees, officers, employees, and agents from ** all claims or demands made against them (and any related losses, ** expenses, or attorney's fees) arising out of, or relating to Licensee's ** and/or its sub licensees' negligent use or willful misuse of or ** negligent conduct or willful misconduct regarding the Software, ** facilities, or other rights or assistance granted by Carnegie Mellon ** University under this License, including, but not limited to, any ** claims of product liability, personal injury, death, damage to ** property, or violation of any laws or regulations. ** ** Carnegie Mellon University Software Engineering Institute authored ** documents are sponsored by the U.S. Department of Defense under ** Contract F19628-00-C-0003. Carnegie Mellon University retains ** copyrights in all material produced under this contract. The U.S. ** Government retains a non-exclusive, royalty-free license to publish or ** reproduce these documents, or allow others to do so, for U.S. ** Government purposes only pursuant to the copyright license under the ** contract clause at 252.227.7013. ** ** @OPENSOURCE_HEADER_END@ */ /* * Rwset.c * * Michael Collins * May 6th * * rwset is an application which takes filter data and generates a * tree (rwset) of ip addresses which come out of a filter file. This * tree can then be used to generate aggregate properties; rwsets will * be used for filtering large sets of ip addresses, aggregate properties * per element can be counted, &c. */ #include "silk.h" RCSIDENT("$SiLK: rwset.c 6689 2007-03-16 20:08:54Z mthomas $"); #include "iptree.h" #include "rwpack.h" #include "utils.h" #include "iochecks.h" #include "skstream.h" #include "sksite.h" /* LOCAL DEFINES AND TYPEDEFS */ /* where to write output from --help */ #define USAGE_FH stdout /* Where to write filenames if --print-file specified */ #define PRINT_FILENAMES_FH stderr /* number of set files that can be created (sip, dip, nhip) */ #define SET_FILE_TYPES 3 /* structure to hold a set and the stream to write it to */ typedef struct _setfile { skIPTree_t *iptree; skstream_t *stream; } setfile_t; /* LOCAL FUNCTIONS */ static void appUsageLong(void); /* never returns */ static void appTeardown(void); static void appSetup(int argc, char **argv); /* never returns */ static int appOptionsHandler(clientData cData, int opt_index, char *opt_arg); /* LOCAL VARIABLES */ /* how to print the ips */ static int ipFormat = SKIP_IPF_DOT; /* stream to use for printing IPs. Null if --print not specified */ static skstream_t *print_stream = NULL; /* streams to store the IPsets in */ static setfile_t sets[SET_FILE_TYPES]; /* the compression method to use when writing the files. * sksiteCompmethodOptionsRegister() will set this to the default or * to the value the user specifies. */ static sk_compmethod_t comp_method; /* whether to print the names of files as they are opened */ static int print_filenames = 0; /* for input processing */ static iochecksInfoStruct_t *ioISP; /* for deprecated --daddress switch */ static uint8_t daddress_flag = 0; /* for deprecated --set-file switch */ static char *set_file_name = NULL; /* OPTIONS SETUP */ typedef enum { OPT_SIP_FILE, OPT_DIP_FILE, OPT_NHIP_FILE, OPT_PRINT_FILENAMES, OPT_COPY_INPUT, OPT_SADDRESS, OPT_DADDRESS, OPT_PRINT_IPS, OPT_INTEGER_IPS, OPT_SET_FILE } appOptionsEnum; static struct option appOptions[] = { {"sip-file", REQUIRED_ARG, 0, OPT_SIP_FILE}, {"dip-file", REQUIRED_ARG, 0, OPT_DIP_FILE}, {"nhip-file", REQUIRED_ARG, 0, OPT_NHIP_FILE}, {"print-filenames", NO_ARG, 0, OPT_PRINT_FILENAMES}, {"copy-input", REQUIRED_ARG, 0, OPT_COPY_INPUT}, /* Deprecated switches follow: */ {"saddress", NO_ARG, 0, OPT_SADDRESS}, {"daddress", NO_ARG, 0, OPT_DADDRESS}, {"print-ips", NO_ARG, 0, OPT_PRINT_IPS}, {"integer-ips", NO_ARG, 0, OPT_INTEGER_IPS}, {"set-file", REQUIRED_ARG, 0, OPT_SET_FILE}, {0,0,0,0} /* sentinel entry */ }; static const int first_deprecated_opt = (int)OPT_SADDRESS; static const char *appHelp[] = { ("Create an IP set from source addresses\n" "\tand write it to the named file (file must not exist)"), ("Create an IP set from destination addresses\n" "\tand write it to the named file (file must not exist)"), ("Create an IP set from next-hop addresses\n" "\tand write it to the named file (file must not exist)"), "Print names of input files as they are opened. Def. No", "Copy all input SiLK Flows to given pipe or file. Def. No", ("Use source addresses as key (default behavior when no\n" "\tother options given). Use --sip-set option instead."), ("Use destination addresses as key (use --dip-set instead)\n" "\n" "The following switches apply to --saddress and --daddress only:"), "Print IP addresses to stdout as dotted decimal. Def. No", "Print IP addresses to stdout as integers. Def. No", "Write binary IPset file to this location. Def. No", (char *)NULL }; /* FUNCTION DEFINITIONS */ /* * appUsageLong(); * * Print complete usage information to USAGE_FH. Pass this * function to skOptionsSetUsageCallback(); optionsParse() will * call this funciton and then exit the program when the --help * option is given. */ static void appUsageLong(void) { #define USAGE_MSG \ ("[SWITCHES] [FILES]\n" \ "\tRead SiLK Flow records and generate binary IPset file(s). When no\n" \ "\tfiles are given on command line, flows are read from STDIN.\n") FILE *fh = USAGE_FH; int i; fprintf(fh, "%s %s", skAppName(), USAGE_MSG); fprintf(fh, "\nSWITCHES:\n"); skOptionsDefaultUsage(fh); sksiteOptionsUsage(fh); for (i = 0; appOptions[i].name; ++i) { if (i == first_deprecated_opt) { sksiteCompmethodOptionsUsage(fh); fprintf(fh, ("\n" "The following options are deprecated" " and cannot be mixed with the above:\n")); } fprintf(fh, "--%s %s. %s\n", appOptions[i].name, SK_OPTION_HAS_ARG(appOptions[i]), appHelp[i]); } } /* * appTeardown() * * Teardown all modules, close all files, and tidy up all * application state. * * This function is idempotent. */ static void appTeardown(void) { static int teardownFlag = 0; int i; if (teardownFlag) { return; } teardownFlag = 1; for (i = 0; i < SET_FILE_TYPES; ++i) { if (sets[i].iptree) { skIPTreeDelete(&sets[i].iptree); sets[i].iptree = NULL; } if (sets[i].stream) { skStreamDestroy(&sets[i].stream); } } skStreamDestroy(&print_stream); iochecksTeardown(ioISP); skAppUnregister(); } /* * appSetup(argc, argv); * * Perform all the setup for this application include setting up * required modules, parsing options, etc. This function should be * passed the same arguments that were passed into main(). * * Returns to the caller if all setup succeeds. If anything fails, * this function will cause the application to exit with a FAILURE * exit status. */ static void appSetup(int argc, char **argv) { int i; int rv; /* verify same number of options and help strings */ assert((sizeof(appHelp)/sizeof(char *)) == (sizeof(appOptions)/sizeof(struct option))); /* register the application */ skAppRegister(argv[0]); skOptionsSetUsageCallback(&appUsageLong); /* initialize globals */ memset(sets, 0, sizeof(sets)); /* initialize input checker */ ioISP = iochecksSetup(0, 0, argc, argv); /* register the options */ if (optionsRegister(appOptions, (optHandler)appOptionsHandler, NULL) || sksiteCompmethodOptionsRegister(&comp_method) || sksiteOptionsRegister(SK_SITE_FLAG_CONFIG_FILE)) { skAppPrintErr("unable to register options"); exit(EXIT_FAILURE); } /* Parse options */ ioISP->firstFile = optionsParse(argc, argv); if (ioISP->firstFile < 0) { skAppUsage(); /* never returns */ } /* try to load site config file; if it fails, we will not be able * to resolve flowtype and sensor from input file names */ sksiteConfigure(0); /* Make certain the user has requested some output. Do not mix * old and new style arguments. */ if (sets[OPT_SIP_FILE].stream || sets[OPT_DIP_FILE].stream || sets[OPT_NHIP_FILE].stream) { if (set_file_name || print_stream) { skAppPrintErr("Cannot mix --set-file or --print-ips with the\n" " --sip, --dip, --nhip switches"); skAppUsage(); } } else if (!set_file_name && !print_stream) { skAppPrintErr("No output specified. Use --sip-file, " "--dip-file, and/or --nhip-file."); skAppUsage(); } /* Convert legacy switches to new form */ if (set_file_name) { /* convert set_file_name to sip or dip */ int idx; if (daddress_flag) { idx = OPT_DIP_FILE; } else { idx = OPT_SIP_FILE; } if ((rv = skStreamCreate(&sets[idx].stream, SK_IO_WRITE, SK_CONTENT_SILK)) || (rv = skStreamBind(sets[idx].stream, set_file_name))) { skStreamPrintLastErr(sets[idx].stream, rv, &skAppPrintErr); exit(EXIT_FAILURE); } } else if (print_stream) { int idx; if (daddress_flag) { idx = OPT_DIP_FILE; } else { idx = OPT_SIP_FILE; } if (skIPTreeCreate(&(sets[idx].iptree))) { skAppPrintErr("Error allocating memory for IPset"); exit(EXIT_FAILURE); } } /* Open the output streams */ for (i = 0; i < SET_FILE_TYPES; ++i) { if (sets[i].stream) { if ((rv = skStreamSetCompressionMethod(sets[i].stream, comp_method)) || (rv = skStreamOpen(sets[i].stream))) { skStreamPrintLastErr(sets[i].stream, rv, &skAppPrintErr); exit(EXIT_FAILURE); } } } /* Create IPsets */ for (i = 0; i < SET_FILE_TYPES; ++i) { if (sets[i].stream) { if (skIPTreeCreate(&(sets[i].iptree))) { skAppPrintErr("Error allocating memory for IPset"); exit(EXIT_FAILURE); } } } /* Use STDIN as an input stream if it is not a TTY; make certain * we have some input and we are either reading from STDIN or * using files listed the command line, but not both. */ if (iochecksAcceptFromStdin(ioISP) || iochecksInputs(ioISP, 0)) { skAppUsage(); } /* looks good, open the --copy-input destination */ if (iochecksOpenCopyDest(ioISP)) { exit(EXIT_FAILURE); } if (atexit(appTeardown) < 0) { skAppPrintErr("unable to register appTeardown() with atexit()"); appTeardown(); exit(EXIT_FAILURE); } return; /* OK */ } /* * status = appOptionsHandler(cData, opt_index, opt_arg); * * This function is passed to optionsRegister(); it will be called * by optionsParse() for each user-specified switch that the * application has registered; it should handle the switch as * required---typically by setting global variables---and return 1 * if the switch processing failed or 0 if it succeeded. Returning * a non-zero from from the handler causes optionsParse() to return * a negative value. * * The clientData in 'cData' is typically ignored; 'opt_index' is * the index number that was specified as the last value for each * struct option in appOptions[]; 'opt_arg' is the user's argument * to the switch for options that have a REQUIRED_ARG or an * OPTIONAL_ARG. */ static int appOptionsHandler( clientData UNUSED(cData), int opt_index, char *opt_arg) { static int stdout_used = 0; int rv; switch ((appOptionsEnum)opt_index) { case OPT_SIP_FILE: case OPT_DIP_FILE: case OPT_NHIP_FILE: if (sets[opt_index].stream) { skAppPrintErr("The --%s switch was given multiple times", appOptions[opt_index].name); return 1; } if (0 == strcmp(opt_arg, "stdout")) { if (stdout_used) { skAppPrintErr("Multiple outputs are trying to use stdout"); return 1; } stdout_used = 1; } if ((rv = skStreamCreate(&(sets[opt_index].stream), SK_IO_WRITE, SK_CONTENT_SILK)) || (rv = skStreamBind(sets[opt_index].stream, opt_arg))) { skStreamPrintLastErr(sets[opt_index].stream, rv, &skAppPrintErr); skStreamDestroy(&(sets[opt_index].stream)); return 1; } break; case OPT_PRINT_FILENAMES: print_filenames = 1; break; case OPT_COPY_INPUT: if (0 == strcmp(opt_arg, "stdout")) { if (stdout_used) { skAppPrintErr("Multiple outputs are trying to use stdout"); return 1; } stdout_used = 1; } if (iochecksAllDestinations(ioISP, opt_arg)) { return 1; } break; case OPT_SADDRESS: daddress_flag = 0; break; case OPT_DADDRESS: daddress_flag = 1; break; case OPT_INTEGER_IPS: ipFormat = SKIP_IPF_DEC; /* FALL THROUGH to set print flag as well */ case OPT_PRINT_IPS: if (!print_stream) { int rv; if (stdout_used) { skAppPrintErr("Multiple outputs are trying to use stdout"); return 1; } stdout_used = 1; if ((rv = skStreamCreate(&print_stream, SK_IO_WRITE, SK_CONTENT_TEXT)) || (rv = skStreamBind(print_stream, "stdout")) || (rv = skStreamOpen(print_stream))) { skStreamPrintLastErr(print_stream, rv, &skAppPrintErr); skStreamDestroy(&print_stream); return 1; } } break; case OPT_SET_FILE: if (0 == strcmp(opt_arg, "stdout")) { if (stdout_used) { skAppPrintErr("Multiple outputs are trying to use stdout"); return 1; } stdout_used = 1; } set_file_name = opt_arg; return 0; } return 0; /* OK */ } /* * status = rwsetProcessFile(filename); * * Read the records from 'filename' and add them to the appropriate * IPSet files. */ static int rwsetProcessFile(const char *filename) { static rwRec rwrec; rwIOStruct_t *rwIOS; uint32_t *ip_p[SET_FILE_TYPES]; int i; /* point ip_p[] at the address of the desired IP in the rwrec. */ ip_p[OPT_SIP_FILE] = &rwrec.sIP.ipnum; ip_p[OPT_DIP_FILE] = &rwrec.dIP.ipnum; ip_p[OPT_NHIP_FILE] = &rwrec.nhIP.ipnum; /* Open file */ rwIOS = rwOpenFile(filename, ioISP->inputCopyFD); if (NULL == rwIOS) { return 1; } if (print_filenames) { fprintf(PRINT_FILENAMES_FH, "%s\n", rwGetFileName(rwIOS)); } /* Read in records */ while (rwRead(rwIOS, &rwrec)) { for (i = 0; i < SET_FILE_TYPES; ++i) { if (sets[i].iptree) { skIPTreeAddAddress(*(ip_p[i]), sets[i].iptree); } } } rwCloseFile(rwIOS); return 0; } int main(int argc, char **argv) { int rv = 0; int i; int counter; appSetup(argc, argv); /* never returns */ /* Process files */ for (counter = ioISP->firstFile; counter < ioISP->fileCount ; counter++) { if (rwsetProcessFile(ioISP->fnArray[counter])) { exit(EXIT_FAILURE); } } /* Generate output */ for (i = 0; i < SET_FILE_TYPES; ++i) { if (sets[i].iptree) { if (print_stream) { skIPTreePrint(print_stream, sets[i].iptree, ipFormat); } if (sets[i].stream) { rv += skIPTreeWrite(sets[i].stream, sets[i].iptree); skStreamClose(sets[i].stream); } skIPTreeDelete(&sets[i].iptree); sets[i].iptree = NULL; } } /* done */ appTeardown(); return rv; } /* ** Local variables: ** mode:c ** indent-tabs-mode:nil ** c-basic-offset:4 ** End: */