/* ** Copyright (C) 2004-2007 by Carnegie Mellon University. ** ** @OPENSOURCE_HEADER_START@ ** ** Use of the SILK system and related source code is subject to the terms ** of the following licenses: ** ** GNU Public License (GPL) Rights pursuant to Version 2, June 1991 ** Government Purpose License Rights (GPLR) pursuant to DFARS 252.225-7013 ** ** NO WARRANTY ** ** ANY INFORMATION, MATERIALS, SERVICES, INTELLECTUAL PROPERTY OR OTHER ** PROPERTY OR RIGHTS GRANTED OR PROVIDED BY CARNEGIE MELLON UNIVERSITY ** PURSUANT TO THIS LICENSE (HEREINAFTER THE "DELIVERABLES") ARE ON AN ** "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY ** KIND, EITHER EXPRESS OR IMPLIED AS TO ANY MATTER INCLUDING, BUT NOT ** LIMITED TO, WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE, ** MERCHANTABILITY, INFORMATIONAL CONTENT, NONINFRINGEMENT, OR ERROR-FREE ** OPERATION. CARNEGIE MELLON UNIVERSITY SHALL NOT BE LIABLE FOR INDIRECT, ** SPECIAL OR CONSEQUENTIAL DAMAGES, SUCH AS LOSS OF PROFITS OR INABILITY ** TO USE SAID INTELLECTUAL PROPERTY, UNDER THIS LICENSE, REGARDLESS OF ** WHETHER SUCH PARTY WAS AWARE OF THE POSSIBILITY OF SUCH DAMAGES. ** LICENSEE AGREES THAT IT WILL NOT MAKE ANY WARRANTY ON BEHALF OF ** CARNEGIE MELLON UNIVERSITY, EXPRESS OR IMPLIED, TO ANY PERSON ** CONCERNING THE APPLICATION OF OR THE RESULTS TO BE OBTAINED WITH THE ** DELIVERABLES UNDER THIS LICENSE. ** ** Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie ** Mellon University, its trustees, officers, employees, and agents from ** all claims or demands made against them (and any related losses, ** expenses, or attorney's fees) arising out of, or relating to Licensee's ** and/or its sub licensees' negligent use or willful misuse of or ** negligent conduct or willful misconduct regarding the Software, ** facilities, or other rights or assistance granted by Carnegie Mellon ** University under this License, including, but not limited to, any ** claims of product liability, personal injury, death, damage to ** property, or violation of any laws or regulations. ** ** Carnegie Mellon University Software Engineering Institute authored ** documents are sponsored by the U.S. Department of Defense under ** Contract F19628-00-C-0003. Carnegie Mellon University retains ** copyrights in all material produced under this contract. The U.S. ** Government retains a non-exclusive, royalty-free license to publish or ** reproduce these documents, or allow others to do so, for U.S. ** Government purposes only pursuant to the copyright license under the ** contract clause at 252.227.7013. ** ** @OPENSOURCE_HEADER_END@ */ /* ** rwbagtool performs various operations on bags. It can add bags ** together, subtract a subset of data from a bag, perform key ** intersection with an IP set, extract the key list of a bag as an ** IP set, or filter bag records based on their counter value. ** */ #include "silk.h" RCSIDENT("$SiLK: rwbagtool.c 7514 2007-06-14 12:20:54Z mthomas $"); #include "utils.h" #include "bagtree.h" #include "iptree.h" #include "sksite.h" /* LOCAL DEFINES AND TYPEDEFS */ /* where to write --help output */ #define USAGE_FH stdout typedef enum _appOptionsEnum { OPT_ADD, OPT_INTERSECT, OPT_COMPLEMENT, OPT_DIVIDE, OPT_SUBTRACT, OPT_COVER_SET, OPT_INVERT, OPT_MINCOUNTER, OPT_MAXCOUNTER, OPT_MINKEY, OPT_MAXKEY, OPT_OUTPUT } appOptionsEnum; /* A struct to hold a bag and all information about its relation to * the file system. */ typedef struct { skBag_header_t *bag; skstream_t *stream; } bagfile_t; /* EXPORTED FUNCTIONS */ /* LOCAL FUNCTIONS */ static void appUsageLong(void); /* never returns */ static void appTeardown(void); static void appSetup(int argc, char **argv); /* never returns when error */ static int appOptionsHandler(clientData, int, char *); static int processBagAdd(const char *pathname); static int processBagDivide(const char *pathname); static int processBagSubtract(const char *pathname); static int readBagFromFile( skBag_header_t **bag, const char *pathname); static int openOutput(const char *file); static int writeOutput(void); /* LOCAL VARIABLES */ /* What action the user wants to take (add, intersect, etc). The * variable 'user_action' should be a value between OPT_ADD and * OPT_COVER_SET. Set 'user_action_none' to a value outside that * range; this denotes that the user did not choose a value. */ static const appOptionsEnum user_action_none = OPT_OUTPUT; static appOptionsEnum user_action; /* When outputting the new bag, only output entries whose keys and/or * whose values lie between these limits. Initialize these to their * opposites to know when they have been set by the user. */ static skBag_counter_t mincounter = SKBAG_COUNTER_MAX; static skBag_counter_t maxcounter = SKBAG_COUNTER_MIN; static skBag_key_t minkey = SKBAG_KEY_MAX; static skBag_key_t maxkey = SKBAG_KEY_MIN; /* Index of current file argument in argv */ static int arg_index = 0; /* When data appears on stdin, this var is 1; else it's 0 */ static int data_on_stdin = 0; /* The bag we will output */ static bagfile_t output_bagfile; /* the compression method to use when writing the file. * sksiteCompmethodOptionsRegister() will set this to the default or * to the value the user specifies. */ static sk_compmethod_t comp_method; /* Set to use for intersect and complement intersect */ static skIPTree_t *mask_set = NULL; /* The file from which to read that set */ static char *mask_set_path = NULL; /* OPTIONS SETUP */ static struct option appOptions[] = { {"add", NO_ARG, 0, OPT_ADD}, {"intersect", REQUIRED_ARG, 0, OPT_INTERSECT}, {"complement-intersect", REQUIRED_ARG, 0, OPT_COMPLEMENT}, {"divide", NO_ARG, 0, OPT_DIVIDE}, {"subtract", NO_ARG, 0, OPT_SUBTRACT}, {"coverset", NO_ARG, 0, OPT_COVER_SET}, {"invert", NO_ARG, 0, OPT_INVERT}, {"mincounter", REQUIRED_ARG, 0, OPT_MINCOUNTER}, {"maxcounter", REQUIRED_ARG, 0, OPT_MAXCOUNTER}, {"minkey", REQUIRED_ARG, 0, OPT_MINKEY}, {"maxkey", REQUIRED_ARG, 0, OPT_MAXKEY}, {"output", REQUIRED_ARG, 0, OPT_OUTPUT}, {0,0,0,0} /* sentinel entry */ }; static const char *appHelp[] = { "Add all bag files read from stdin and the command line", "Masks keys in bag file using IPs in given SET file", ("Masks keys in bag file using IPs NOT\n" "\tin given SET file"), "Divide the first bag by all subsequent bags", "Subtract from first bag file all subsequent bag files", "Extract the IPs from the bag file into a set file", "Count keys for each unique counter value", ("Output records whose counter is at least VALUE, an\n" "\tinteger. Def. 1"), ("Output records whose counter is not more than VALUE, an\n" "\tinteger. Def. 18446744073709551615"), ("Output records whose key is at least VALUE, an integer or an\n" "\tIP address. Def. 0 or 0.0.0.0"), ("Output records whose key is not more than VALUE, an integer\n" "\tor an IP address. Def. 4294967295 or 255.255.255.255"), "Redirect output to specified file.", (char *)NULL }; /* FUNCTION DEFINITIONS */ /* * appUsageLong(); * * Print complete usage information to USAGE_FH. Pass this * function to skOptionsSetUsageCallback(); optionsParse() will * call this funciton and then exit the program when the --help * option is given. */ static void appUsageLong(void) { #define USAGE_MSG \ ("[SWITCHES] BAG_FILE [BAG_FILES...]\n" \ "\tPerform operations on bag files, creating a new bag file.\n" \ "\tRequires at least one bag file to be given on the command line\n" \ "\tor to be read from the standard input. The resulting bag will\n" \ "\twill be written to the specified output file or to the standard\n" \ "\toutput.\n") FILE *fh = USAGE_FH; skAppStandardUsage(fh, USAGE_MSG, appOptions, appHelp); sksiteCompmethodOptionsUsage(fh); } /* * appTeardown() * * Teardown all modules, close all files, and tidy up all * application state. * * This function is idempotent. */ static void appTeardown(void) { static int teardownFlag = 0; if (teardownFlag) { return; } teardownFlag = 1; /* free the output bag */ if (output_bagfile.bag) { skBag_free(output_bagfile.bag); } if (output_bagfile.stream) { skStreamDestroy(&output_bagfile.stream); } if (mask_set) { skIPTreeDelete(&mask_set); } skAppUnregister(); } /* * appSetup(argc, argv); * * Perform all the setup for this application include setting up * required modules, parsing options, etc. This function should be * passed the same arguments that were passed into main(). * * Returns to the caller if all setup succeeds. If anything fails, * this function will cause the application to exit with a FAILURE * exit status. */ static void appSetup(int argc, char **argv) { /* verify same number of options and help strings */ assert((sizeof(appHelp)/sizeof(char *)) == (sizeof(appOptions)/sizeof(struct option))); /* register the application */ skAppRegister(argv[0]); skOptionsSetUsageCallback(&appUsageLong); /* initialize globals */ user_action = user_action_none; memset(&output_bagfile, 0, sizeof(bagfile_t)); /* register the options */ if (optionsRegister(appOptions, (optHandler)appOptionsHandler, NULL) || sksiteCompmethodOptionsRegister(&comp_method)) { skAppPrintErr("unable to register options"); exit(EXIT_FAILURE); } /* parse the options; returns the index into argv[] of the first * non-option or < 0 on error May re-arrange argv[]. */ arg_index = optionsParse(argc, argv); assert(arg_index <= argc); if (arg_index < 0) { /* options parsing should print error */ skAppUsage(); /* never returns */ } /* Make certain user selected some action to perform */ if (user_action == user_action_none) { skAppPrintErr("One of --add, --subtract, --coverset, --intersect, " "--divide, \n --invert or --complement-intersect " "must be specified."); skAppUsage(); } /* set the minima and maxima */ if (mincounter == SKBAG_COUNTER_MAX) { mincounter = SKBAG_COUNTER_MIN; } if (maxcounter == SKBAG_COUNTER_MIN) { maxcounter = SKBAG_COUNTER_MAX; } if (minkey == SKBAG_KEY_MAX) { minkey = SKBAG_KEY_MIN; } if (maxkey == SKBAG_KEY_MIN) { maxkey = SKBAG_KEY_MAX; } /* error if a minimum is greater than a maximum */ if (mincounter > maxcounter) { skAppPrintErr(("Minimum counter greater than maximum: " "%" PRIu64 " > %" PRIu64), mincounter, maxcounter); exit(EXIT_FAILURE); } if (minkey > maxkey) { skAppPrintErr(("Minimum key greater than maximum: " "%" PRIu32 " > %" PRIu32), minkey, maxkey); exit(EXIT_FAILURE); } /* arg_index is looking at first file name to process. We * require at least one bag file, either from stdin or from the * command line. */ if (arg_index == argc) { if (!FILEIsATty(stdin)) { data_on_stdin = 1; } else { skAppPrintErr("No input files on command line and" " stdin is connected to a terminal"); skAppUsage(); } } if (atexit(appTeardown) < 0) { skAppPrintErr("unable to register appTeardown() with atexit()"); appTeardown(); exit(EXIT_FAILURE); } return; /* OK */ } /* * status = appOptionsHandler(cData, opt_index, opt_arg); * * This function is passed to optionsRegister(); it will be called * by optionsParse() for each user-specified switch that the * application has registered; it should handle the switch as * required---typically by setting global variables---and return 1 * if the switch processing failed or 0 if it succeeded. Returning * a non-zero from from the handler causes optionsParse() to return * a negative value. * * The clientData in 'cData' is typically ignored; 'opt_index' is * the index number that was specified as the last value for each * struct option in appOptions[]; 'opt_arg' is the user's argument * to the switch for options that have a REQUIRED_ARG or an * OPTIONAL_ARG. */ static int appOptionsHandler( clientData UNUSED(cData), int opt_index, char *opt_arg) { uint32_t val32; uint64_t val64; int rv; switch ((appOptionsEnum)opt_index) { case OPT_ADD: if (user_action != user_action_none) { skAppPrintErr("Switches --%s and --%s are incompatible", appOptions[opt_index].name, appOptions[user_action].name); return 1; } user_action = (appOptionsEnum)opt_index; break; case OPT_INTERSECT: if (user_action != user_action_none) { skAppPrintErr("Switches --%s and --%s are incompatible", appOptions[opt_index].name, appOptions[user_action].name); return 1; } user_action = (appOptionsEnum)opt_index; mask_set_path = opt_arg; break; case OPT_COMPLEMENT: if (user_action != user_action_none) { skAppPrintErr("Switches --%s and --%s are incompatible", appOptions[opt_index].name, appOptions[user_action].name); return 1; } user_action = (appOptionsEnum)opt_index; mask_set_path = opt_arg; break; case OPT_DIVIDE: if (user_action != user_action_none) { skAppPrintErr("Switches --%s and --%s are incompatible", appOptions[opt_index].name, appOptions[user_action].name); return 1; } user_action = (appOptionsEnum)opt_index; break; case OPT_SUBTRACT: if (user_action != user_action_none) { skAppPrintErr("Switches --%s and --%s are incompatible", appOptions[opt_index].name, appOptions[user_action].name); return 1; } user_action = (appOptionsEnum)opt_index; break; case OPT_COVER_SET: if (user_action != user_action_none) { skAppPrintErr("Switches --%s and --%s are incompatible", appOptions[opt_index].name, appOptions[user_action].name); return 1; } user_action = (appOptionsEnum)opt_index; break; case OPT_INVERT: if (user_action != user_action_none) { skAppPrintErr("Switches --%s and --%s are incompatible", appOptions[opt_index].name, appOptions[user_action].name); return 1; } user_action = (appOptionsEnum)opt_index; break; case OPT_MINCOUNTER: rv = skStringParseUint64(&val64, opt_arg, 1, 0); if (rv == -11) { skAppPrintErr("Smallest allowable --%s value is 1", appOptions[opt_index].name); return 1; } if (rv) { skAppPrintErr("Could not parse %s value '%s' as a number", appOptions[opt_index].name, opt_arg); return 1; } mincounter = (skBag_counter_t)val64; break; case OPT_MAXCOUNTER: rv = skStringParseUint64(&val64, opt_arg, 1, 0); if (rv == -11) { skAppPrintErr("Smallest allowable --%s value is 1", appOptions[opt_index].name); return 1; } if (rv) { skAppPrintErr("Could not parse %s value '%s' as a number", appOptions[opt_index].name, opt_arg); return 1; } maxcounter = (skBag_counter_t)val64; break; case OPT_MINKEY: if (skStringParseIP(&val32, opt_arg)) { skAppPrintErr("Could not parse %s value '%s' as a number nor IP", appOptions[opt_index].name, opt_arg); return 1; } minkey = (skBag_key_t)val32; break; case OPT_MAXKEY: if (skStringParseIP(&val32, opt_arg)) { skAppPrintErr("Could not parse %s value '%s' as a number nor IP", appOptions[opt_index].name, opt_arg); return 1; } maxkey = (skBag_key_t)val32; break; case OPT_OUTPUT: if (output_bagfile.stream) { skAppPrintErr("The --%s switch was given multiple times", appOptions[opt_index].name); return 1; } if (openOutput(opt_arg)) { return 1; } break; } return 0; /* OK */ } /* * ok = processBagAdd(pathname); * * Add the values found in the bag stored in 'pathname' to the * values in the global 'output_bagfile'. Return 0 on success; * non-zero otherwise. */ static int processBagAdd(const char *pathname) { skBag_err_t rv_bag = SKBAG_OK; skstream_t *s = NULL; int rv; if ((rv = skStreamCreate(&s, SK_IO_READ, SK_CONTENT_SILK)) || (rv = skStreamBind(s, pathname)) || (rv = skStreamOpen(s))) { skStreamPrintLastErr(s, rv, &skAppPrintErr); skStreamDestroy(&s); return -1; } rv_bag = skBag_addBinary(output_bagfile.bag, s); if (rv_bag != SKBAG_OK) { if (rv_bag == SKBAG_ERR_OP_BOUNDS) { skAppPrintErr("Overflow when adding bags"); } else { skAppPrintErr("Error when adding bags: %s", skBag_strerror(rv_bag)); } } skStreamDestroy(&s); return ((rv_bag == SKBAG_OK) ? 0 : -1); } /* * ok = processBagDivide(pathname); * * Divide the values in the global 'output_bagfile' by the values * found in the bag stored in 'pathname'. The keys of the two bags * must match exactly. Return 0 on success; non-zero otherwise. */ static int processBagDivide(const char *pathname) { skBag_header_t *input_bag; skBag_iterator_t *divisor_iter = NULL; skBag_iterator_t *dividend_iter = NULL; skBag_err_t rv_dividend_bag; skBag_err_t rv_divisor_bag; skBag_key_t dividend_key; skBag_key_t divisor_key; skBag_counter_t dividend_counter; skBag_counter_t divisor_counter; skBag_counter_t result_counter; skBag_err_t rv_bag; /* read the bag containing the divisors */ if (readBagFromFile(&input_bag, pathname)) { return SKBAG_ERR_INPUT; } /* Create an iterator to loop over the divisor bag */ rv_bag = skBag_allocIterator(input_bag, &divisor_iter); if (rv_bag != SKBAG_OK) { divisor_iter = NULL; goto END; } /* Create an iterator to loop over the dividend bag */ rv_bag = skBag_allocIterator(output_bagfile.bag, ÷nd_iter); if (rv_bag != SKBAG_OK) { dividend_iter = NULL; goto END; } do { rv_dividend_bag = skBag_getNext(dividend_iter, ÷nd_key, ÷nd_counter); rv_divisor_bag = skBag_getNext(divisor_iter, &divisor_key, &divisor_counter); if (rv_dividend_bag != SKBAG_OK || rv_divisor_bag != SKBAG_OK) { goto END; } if (dividend_key != divisor_key) { skAppPrintErr("Error dividing bags. The keys in the two " "bags do not match."); goto END; } if (dividend_counter < divisor_counter) { skAppPrintErr("Error dividing bags. A divisor counter " "is larger than a\n dividend counter."); goto END; } result_counter = dividend_counter / divisor_counter; rv_bag = skBag_setCounter(output_bagfile.bag, ÷nd_key, &result_counter); if (rv_bag != SKBAG_OK) { skAppPrintErr("Unexpected error while setting counter after " "division: %s", skBag_strerror(rv_bag)); goto END; } } while (rv_dividend_bag == SKBAG_OK && rv_divisor_bag == SKBAG_OK); /* We're either at the end of the bag or the getNext call failed */ if (rv_dividend_bag == SKBAG_ERR_KEY_NOT_FOUND && rv_divisor_bag == SKBAG_ERR_KEY_NOT_FOUND) { /* expected behavior: reached end of both bags at same time. * set rv_bag to SKBAG_OK */ rv_bag = SKBAG_OK; } else if ((rv_dividend_bag == SKBAG_ERR_KEY_NOT_FOUND && rv_divisor_bag == SKBAG_OK) || (rv_dividend_bag == SKBAG_OK && rv_divisor_bag == SKBAG_ERR_KEY_NOT_FOUND)) { /* one bag terminated before the other */ skAppPrintErr("Error dividing bags. The keys in the two " "bags do not match."); rv_bag = SKBAG_ERR_OP_BOUNDS; } else { if (rv_dividend_bag != SKBAG_OK) { skAppPrintErr("Error iterating over dividend bag: %s", skBag_strerror(rv_dividend_bag)); rv_bag = rv_dividend_bag; } if (rv_divisor_bag != SKBAG_OK) { skAppPrintErr("Error iterating over divisor bag: %s", skBag_strerror(rv_divisor_bag)); rv_bag = rv_divisor_bag; } } END: if (dividend_iter) { skBag_freeIterator(dividend_iter); } if (divisor_iter) { skBag_freeIterator(divisor_iter); } /* Blow the bag away. We're done with it */ skBag_free(input_bag); return ((rv_bag == SKBAG_OK) ? 0 : -1); } /* * ok = processBagSubtract(pathname); * * Subtract the values found in the bag stored in 'pathname' from * the values in the global 'output_bagfile'. The keys of the two * bags must match exactly. Return 0 on success; non-zero * otherwise. */ static int processBagSubtract(const char *pathname) { skBag_header_t *input_bag; skBag_iterator_t *iter = NULL; skBag_key_t key; skBag_counter_t counter; skBag_err_t rv_bag = SKBAG_OK; if (readBagFromFile(&input_bag, pathname)) { return -1; } /* Create an iterator to loop over the bag */ rv_bag = skBag_allocIterator(input_bag, &iter); if (rv_bag != SKBAG_OK) { goto END; } while (SKBAG_OK == (rv_bag = skBag_getNext(iter, &key, &counter))) { rv_bag = skBag_subtractFromCounter(output_bagfile.bag, &key, &counter); if (rv_bag != SKBAG_OK) { if (rv_bag == SKBAG_ERR_OP_BOUNDS) { skAppPrintErr("Negative values in bag are not allowed;" " key=%u", key); } else { skAppPrintErr("Error when subtracting from bag for key %u: %s", key, skBag_strerror(rv_bag)); } goto END; } } /* We're either at the end of the bag or the getNext call failed */ if (rv_bag == SKBAG_ERR_KEY_NOT_FOUND) { /* expected behavior: reached end of bag. reset rv_bag */ rv_bag = SKBAG_OK; } else { /* unexpected error: print message and set rv to that error */ skAppPrintErr("Error while iterating over bag: %s", skBag_strerror(rv_bag)); } END: if (iter) { skBag_freeIterator(iter); } /* Blow the bag away. We're done with it */ skBag_free(input_bag); return ((rv_bag == SKBAG_OK) ? 0 : -1); } /* * status = processBagInvert(bag); * * Invert the Bag 'bag' in place. Return 0 on success, or -1 on * error. */ static int processBagInvert(skBag_header_t *bag) { skBag_header_t *inv_bag = NULL; skBag_iterator_t *iter; skBag_key_t key; skBag_counter_t counter; const skBag_counter_t zero_counter = 0; skBag_err_t rv_bag; int rv = -1; /* We can't actually invert the bag in place, so create a new * inverted bag. Once we make a pass through the existing bag, we * make a pass over the inverted bag and set the values in the * original bag. */ if (skBag_create(&inv_bag) != SKBAG_OK) { goto END; } /* Run through the original bag */ rv_bag = skBag_allocIterator(bag, &iter); if (rv_bag != SKBAG_OK) { goto END; } while ((rv_bag = skBag_getNext(iter, &key, &counter)) == SKBAG_OK) { /* remove this key from the bag--set its count to 0 */ rv_bag = skBag_setCounter(bag, &key, &zero_counter); if (rv_bag != SKBAG_OK) { skAppPrintErr("Error when setting key %u in bag: %s", key, skBag_strerror(rv_bag)); goto END; } /* determine key to use in inverted bag, handing overflow */ key = (skBag_key_t)((counter < UINT32_MAX) ? counter : UINT32_MAX); /* increment value in the inverted bag */ rv_bag = skBag_incrCounter(inv_bag, &key); if (rv_bag != SKBAG_OK) { if (rv_bag == SKBAG_ERR_OP_BOUNDS) { skAppPrintErr("Overflow when inverting bag"); goto END; } skAppPrintErr("Error when inverting bag: %s", skBag_strerror(rv_bag)); goto END; } skBag_getCounter(inv_bag, &key, &counter); } skBag_freeIterator(iter); /* Run through the inverted bag */ rv_bag = skBag_allocIterator(inv_bag, &iter); if (rv_bag != SKBAG_OK) { goto END; } while ((rv_bag = skBag_getNext(iter, &key, &counter)) == SKBAG_OK) { rv_bag = skBag_setCounter(bag, &key, &counter); if (rv_bag != SKBAG_OK) { if (rv_bag == SKBAG_ERR_OP_BOUNDS) { skAppPrintErr("Overflow when inverting bag"); goto END; } skAppPrintErr("Error when setting key %u in bag: %s", key, skBag_strerror(rv_bag)); goto END; } } skBag_freeIterator(iter); /* done */ rv = 0; END: if (inv_bag) { skBag_free(inv_bag); } return rv; } /* * status = processBagCoverSet(bag, &set); * * Create an IPset at the location specified by '*set' and fill it * with the keys in the Bag 'bag'. Return 0 on success, or -1 on * failure. */ static int processBagCoverSet( skBag_header_t *bag, skIPTree_t **set) { skBag_iterator_t *iter; skBag_key_t key; skBag_counter_t counter; skBag_err_t rv_bag; int rv = -1; assert(set); *set = NULL; /* Create the cover set */ if (skIPTreeCreate(set)) { skAppPrintErr("Error creating cover set"); goto END; } /* Run through the bag, adding items to the set */ if (skBag_allocIterator(bag, &iter)) { goto END; } while ((rv_bag = skBag_getNext(iter, &key, &counter)) == SKBAG_OK) { skIPTreeAddAddress(key, *set); } if (rv_bag != SKBAG_ERR_KEY_NOT_FOUND) { /* unexpected error */ skAppPrintErr("Error while interating over bag: %s", skBag_strerror(rv_bag)); } skBag_freeIterator(iter); /* done */ rv = 0; END: if (rv != 0) { if (*set != NULL) { skIPTreeDelete(set); } } return rv; } /* * ok = applyCutoffs(bag); * * Run through the bag and zero out any entries not within range or * which aren't in the masking set. Return 0 on success, non-zero * otherwise. * * FIXME: We could do some of this during the insertion stage * instead output to save ourselves some storage. This is not the * most efficient implementation. */ static int applyCutoffs(skBag_header_t *bag) { skBag_iterator_t *iter = NULL; skBag_key_t key; skBag_counter_t counter; skBag_err_t rv_bag; /* Create an iterator to loop over the bag */ rv_bag = skBag_allocIterator(bag, &iter); if (rv_bag != SKBAG_OK) { iter = NULL; goto END; } while ((rv_bag = skBag_getNext(iter, &key, &counter)) == SKBAG_OK) { if ((key < minkey) || (key > maxkey) || (counter < mincounter) || (counter > maxcounter) || ((user_action == OPT_INTERSECT) && !skIPTreeCheckAddress(key, mask_set)) || ((user_action == OPT_COMPLEMENT) && skIPTreeCheckAddress(key, mask_set))) { counter = 0; rv_bag = skBag_setCounter(bag, &key, &counter); if (rv_bag != SKBAG_OK) { skAppPrintErr("Error setting key %u to zero in bag: %s", key, skBag_strerror(rv_bag)); goto END; } } } /* We're either at the end of the bag or the getNext call failed */ if (rv_bag == SKBAG_ERR_KEY_NOT_FOUND) { /* expected behavior: reached end of bag. reset rv_bag */ rv_bag = SKBAG_OK; } else { /* unexpected error: print message and set rv to that error */ skAppPrintErr("Error while interating over bag: %s", skBag_strerror(rv_bag)); } END: if (iter) { skBag_freeIterator(iter); } return ((rv_bag == SKBAG_OK) ? 0 : -1); } /* * stream = openOutput(file); * * Crete, bind, and open the global output stream * 'output_bagfile.stream'. The parameter 'file' is either a * pathname or "stdout". Return 0 on success. On failure, the * stream is not created, an error message is printed, and -1 is * returned. */ static int openOutput(const char *file) { int rv; if ((rv = skStreamCreate(&output_bagfile.stream, SK_IO_WRITE, SK_CONTENT_SILK)) || (rv = skStreamBind(output_bagfile.stream, file)) || (rv = skStreamSetCompressionMethod(output_bagfile.stream, comp_method)) || (rv = skStreamOpen(output_bagfile.stream))) { skStreamPrintLastErr(output_bagfile.stream, rv, &skAppPrintErr); skStreamDestroy(&output_bagfile.stream); return -1; } return 0; } /* * ok = writeOutput(); * * Generate the output as requested by the global 'user_action' * variable. */ static int writeOutput(void) { /* Remove anything that's not in range or not in the intersecting * set (or complement) as appropriate */ applyCutoffs(output_bagfile.bag); if (user_action == OPT_COVER_SET) { skIPTree_t *cover_set; if (processBagCoverSet(output_bagfile.bag, &cover_set)) { return -1; } /* Write the set */ if (skIPTreeWrite(output_bagfile.stream, cover_set)) { skAppPrintErr("Error writing set file."); skIPTreeDelete(&cover_set); return -1; } skIPTreeDelete(&cover_set); } else { if (user_action == OPT_INVERT) { processBagInvert(output_bagfile.bag); } if (SKBAG_OK != skBag_writeBinary(output_bagfile.bag, output_bagfile.stream)) { skAppPrintErr("Error writing bag to output file '%s'", skStreamGetPathname(output_bagfile.stream)); return -1; } } return 0; } /* * ok = readBagFromFile(&bag, path); * * Fill in the bag object pointed to by 'bag' with the contents of * 'path'. Return 0 on success, non-zero otherwise. */ static int readBagFromFile(skBag_header_t **bag, const char *pathname) { skBag_err_t rv_bag = SKBAG_OK; skstream_t *s = NULL; int rv; if ((rv = skStreamCreate(&s, SK_IO_READ, SK_CONTENT_SILK)) || (rv = skStreamBind(s, pathname)) || (rv = skStreamOpen(s))) { skStreamPrintLastErr(s, rv, &skAppPrintErr); skStreamDestroy(&s); return -1; } rv_bag = skBag_readBinary(bag, s); if (rv_bag != SKBAG_OK) { skAppPrintErr("Could not read Bag from '%s'", pathname); } skStreamDestroy(&s); return ((rv_bag == SKBAG_OK) ? 0 : -1); } int main(int argc, char **argv) { appSetup(argc, argv); /* never returns on error */ /* If action is intersect or complement intersect read the set * file */ if (user_action == OPT_INTERSECT || user_action == OPT_COMPLEMENT) { if (skIPTreeLoad(mask_set_path, &mask_set)) { skAppPrintErr("Could not read %s IPset from file %s", appOptions[user_action].name, mask_set_path); return 1; } } /* Open up main output file if necessary */ if (output_bagfile.stream == NULL) { if (openOutput("stdout")) { return 1; } } /* Read the first bag; this will be the basis of the output bag. */ if (data_on_stdin) { /* use stdin as first bag */ if (readBagFromFile(&output_bagfile.bag, "stdin")) { return 1; } } else { if (readBagFromFile(&output_bagfile.bag, argv[arg_index])) { return 1; } arg_index++; } /* Open up each bag and process it appropriately */ for ( ; arg_index < argc; ++arg_index) { switch (user_action) { case OPT_COMPLEMENT: case OPT_INTERSECT: case OPT_ADD: case OPT_COVER_SET: case OPT_INVERT: /* All of these require us to add all the bags */ if (processBagAdd(argv[arg_index])) { return 1; } break; case OPT_DIVIDE: if (processBagDivide(argv[arg_index])) { return 1; } break; case OPT_SUBTRACT: if (processBagSubtract(argv[arg_index])) { return 1; } break; default: /* Processing options not handled in this switch require a * single bag file */ skAppPrintErr("Option %d not handled in switch() at %s:%d", user_action, __FILE__, __LINE__); assert(0); abort(); } } /* Write the output */ if (writeOutput()) { return 1; } /* done */ appTeardown(); return 0; } /* ** Local Variables: ** mode:c ** indent-tabs-mode:nil ** c-basic-offset:4 ** End: */