/* ** Copyright (C) 2004-2007 by Carnegie Mellon University. ** ** @OPENSOURCE_HEADER_START@ ** ** Use of the SILK system and related source code is subject to the terms ** of the following licenses: ** ** GNU Public License (GPL) Rights pursuant to Version 2, June 1991 ** Government Purpose License Rights (GPLR) pursuant to DFARS 252.225-7013 ** ** NO WARRANTY ** ** ANY INFORMATION, MATERIALS, SERVICES, INTELLECTUAL PROPERTY OR OTHER ** PROPERTY OR RIGHTS GRANTED OR PROVIDED BY CARNEGIE MELLON UNIVERSITY ** PURSUANT TO THIS LICENSE (HEREINAFTER THE "DELIVERABLES") ARE ON AN ** "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY ** KIND, EITHER EXPRESS OR IMPLIED AS TO ANY MATTER INCLUDING, BUT NOT ** LIMITED TO, WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE, ** MERCHANTABILITY, INFORMATIONAL CONTENT, NONINFRINGEMENT, OR ERROR-FREE ** OPERATION. CARNEGIE MELLON UNIVERSITY SHALL NOT BE LIABLE FOR INDIRECT, ** SPECIAL OR CONSEQUENTIAL DAMAGES, SUCH AS LOSS OF PROFITS OR INABILITY ** TO USE SAID INTELLECTUAL PROPERTY, UNDER THIS LICENSE, REGARDLESS OF ** WHETHER SUCH PARTY WAS AWARE OF THE POSSIBILITY OF SUCH DAMAGES. ** LICENSEE AGREES THAT IT WILL NOT MAKE ANY WARRANTY ON BEHALF OF ** CARNEGIE MELLON UNIVERSITY, EXPRESS OR IMPLIED, TO ANY PERSON ** CONCERNING THE APPLICATION OF OR THE RESULTS TO BE OBTAINED WITH THE ** DELIVERABLES UNDER THIS LICENSE. ** ** Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie ** Mellon University, its trustees, officers, employees, and agents from ** all claims or demands made against them (and any related losses, ** expenses, or attorney's fees) arising out of, or relating to Licensee's ** and/or its sub licensees' negligent use or willful misuse of or ** negligent conduct or willful misconduct regarding the Software, ** facilities, or other rights or assistance granted by Carnegie Mellon ** University under this License, including, but not limited to, any ** claims of product liability, personal injury, death, damage to ** property, or violation of any laws or regulations. ** ** Carnegie Mellon University Software Engineering Institute authored ** documents are sponsored by the U.S. Department of Defense under ** Contract F19628-00-C-0003. Carnegie Mellon University retains ** copyrights in all material produced under this contract. The U.S. ** Government retains a non-exclusive, royalty-free license to publish or ** reproduce these documents, or allow others to do so, for U.S. ** Government purposes only pursuant to the copyright license under the ** contract clause at 252.227.7013. ** ** @OPENSOURCE_HEADER_END@ */ #ifndef _LIBRW_PRIV_H #define _LIBRW_PRIV_H #include "silk.h" RCSIDENTVAR(rcsID_LIBRW_PRIV_H, "$SiLK: librw_priv.h 7291 2007-05-25 14:20:10Z mthomas $"); /* ** librw_priv.h ** ** For sharing of functions within librw (to avoid tons of extern's). ** ** THESE FUNCTIONS ARE FOR INTERNAL USE BY LIBRW. ** */ #include "sksite.h" #include "utils.h" #include "rwpack.h" #include "bits.h" /* macros to swap the bytes in place */ #define SWAP_DATA32(d) *((uint32_t*)(d)) = BSWAP32(*(uint32_t*)(d)) #define SWAP_DATA16(d) *((uint16_t*)(d)) = BSWAP16(*(uint16_t*)(d)) #define RWIO_READ(rwio, buffer, sz) \ (((ssize_t)(sz)) != _ioRead((rwio), (buffer), ((size_t)(sz)))) #define RWIO_WRITE(rwio, buffer, sz) \ (((ssize_t)(sz)) != _ioWrite((rwio), (buffer), ((size_t)(sz)))) /* * Macros to read and write small amounts of uncompressed data, * such as file headers. Return 0 for a complete read/write, * non-zero otherwise. */ #define MAX_PKTS 1048576 /* 2^20 */ #define PKTS_DIVISOR 64 /* * We store the packet count in a 20 bit value. When the packet * count is larger than that, we divide the value by the * PKTS_DIVISOR and store the result. That gives an absolute max * of 67,100,864 packets. */ #define BPP_BITS 6 #define BPP_PRECN 64 /* 2^BPP_BITS */ #define BPP_PRECN_DIV_2 32 /* 2^BPP_BITS/2 */ #define MAX_START_TIME 4096 /* 2^12 */ /* * We pack flows by their start time into hourly files. The file's * hour is stored in the header; each record's start time is offset * from that and stored in 12 bits. */ #define MAX_ELAPSED_TIME 4096 /* 2^12 */ #define MAX_ELAPSED_TIME_OLD 2048 /* 2^11 */ /* * The elapsed time is the offset from the record's start time. We * assume the router flushes flows at least once an hour, though in * practice CISCO flushes every 30 mintues. The elapsed time is * stored in 11 or 12 bits, depending on file format. */ /* Web classification utilities */ #define WEBP(p) ((p) == 80 || (p) == 443 || (p) == 8080) /* * isweb = WEBP(p) * * Return a true value if port 'p' is a "web" port; false otherwise */ #define WEBPORT(p) \ (((p) == 80) \ ? 0 \ : (((p) == 443) \ ? 1 \ : (((p) == 8080) \ ? 2 \ : 3))) /* * encoding = WEBPORT(p) * * Encode the port 'p' into a value suitable for storing in the * wPort field of an FT_RWWWW record. */ #define EXPAND_WEBPORT(p) \ (((p) == 0) \ ? 80 \ : (((p) == 1) \ ? 443 \ : (((p) == 2) \ ? 8080 \ : 0))) /* * decoding = EXPAND_WEBPORT(p) * * Decode the port 'p' from the value stored in the wPort field in * an FT_RWWWW record. */ #define IS_WEB_PORT(p) WEBP(p) /* * isweb = IS_WEB_PORT(p) * * Alias for WEBP */ #define IS_WEB(r) \ (((r).prot == 6) && (IS_WEB_PORT((r).srcport) || IS_WEB_PORT((r).dstport))) /* * isweb = IS_WEB(r) * * Return a true value if the flow 'r' represents a web flow; * return false otherwise. 'r' is an rwrec, NOT an rwrec*. */ /* *** rwio.c *** */ ssize_t _ioRead( rwIOStruct_t *rwIOS, void *buf, size_t count); /* * Read 'count' bytes from the stream in 'rwIOS' and copy them into * 'buf'. Returns the number of bytes actually read, or -1 to * indicate an error. */ int _ioHandleHeaderPadding( rwIOStruct_t *rwIOS); /* * Read or write the number of bytes required to make the header an * even multiple of the record size. Return LIBRW_OK on success, * or an error code. */ /* ***** rwheaders.c ***** */ int _headersAppendArgv( rwIOStruct_t *rwIOS, int argc, char * const *argv); /* * Add a new command line---of 'argc' elements in the 'argv' * array---to the command line history stored in the header * associated with the 'rwIOS' stream. Update the length of the * header. * * Returns LIBRW_OK on success, or non-zero otherwise. */ int _headersAppendFromFile( rwIOStruct_t *rwIOS, const sk_cmd_history_t *src_hist); /* * Append the command line history 'src_hist' from a source file to * the command line history stored in the header for the stream in * 'rwIOS', and update the length of the header. * * Returns LIBRW_OK on success, or non-zero otherwise. */ int _headersCreateV1(rwIOStruct_t *rwIOS); /* * Expand the existing genericHeader on 'rwIOS' to a header * required by an FT_RWFILTER file; i.e., filterHeaderV1. * * Returns LIBRW_OK on success, or non-zero otherwise. */ int _headersDestroyV1(rwIOStruct *rwIOS); /* * Destroy the filterHeaderV1 on 'rwIOS' and set the 'hdr' field on * 'rwIOS' to NULL. Returns LIBRW_OK. */ int _headersReadV1(rwIOStruct_t *rwIOS); /* * Read the portion of the header for an FT_RWFILTER file that * follows the genericHeader. * * Returns LIBRW_OK on success, or non-zero otherwise. */ int _headersWriteV1(rwIOStruct_t *rwIOS, const rwRec *rwrec); /* * Write the complete header for an FT_RWFILTER file, including the * genericHeader. * * Returns LIBRW_OK on success, or non-zero otherwise. */ /* ***** rwpackedfileheader.c ***** */ int _packedfileheaderCreateV0(rwIOStruct_t *rwIOS); /* * Complete the setup of the file described by *rwIOS: Grow the * header to hold an rwFileHeaderV0* and copy the startTime into * the header structure. Returns LIBRW_OK on success and non-zero * on failure: memory-allocation. */ int _packedfileheaderReadV0(rwIOStruct_t *rwIOS); /* * Assumes the cursor is at the end of the genericHeader and reads * the remainder of a FileHeaderV0. This involves reading the * start time (byte swapping it if necessary, and reading any * header padding in V2 or greater files. This function leaves the * cursor at the start of the data. Will realloc() the header * contained in the rwIOS. * * Returns LIBRW_OK if successful. */ int _packedfileheaderWriteV0( rwIOStruct_t *rwIOS, const rwGenericRec_V3 *rwrec); /* * Write the entire header for the rwIOS, a rwFileHeaderV0*, to the * underlying file or stream--with the start time in the * appropriate byte order--and pad the header to an even multiple * of the record size if required. Returns LIBRW_OK on success and * non-zero on failure. Use 'rwrec', if provided, to set the start * time of the file if the start-time had not been previously set. */ int _packedfileheaderSetSTime( rwIOStruct_t *rwIOS, uint32_t start_time, int round_to_hour); /* * status = _packedfileheaderSetSTime(rwIOS, start_time, round_to_hour); * * Sets the start time on the 'rwIOS' to the given 'start_time', * rounding the time down to the hour (floor) if 'round_to_hour' is * non-zero. Will expand the header on 'rwIOS' to a rwFileHeaderV0 * if required. Return LIBRW_OK on succcess, or non-zero on * failure: allocation. */ /* ***** Functions exported from each rwio.c file ***** */ int _augmentedioPrepareRead(rwIOStruct_t *rwIOS); int _augroutingioPrepareRead(rwIOStruct_t *rwIOS); int _augsnmpoutioPrepareRead(rwIOStruct_t *rwIOS); int _augwebioPrepareRead(rwIOStruct_t *rwIOS); int _filterioPrepareRead(rwIOStruct_t *rwIOS); int _genericioPrepareRead(rwIOStruct_t *rwIOS); int _notroutedioPrepareRead(rwIOStruct_t *rwIOS); int _routedioPrepareRead(rwIOStruct_t *rwIOS); int _splitioPrepareRead(rwIOStruct_t *rwIOS); int _wwwioPrepareRead(rwIOStruct_t *rwIOS); /* * status = _ioPrepareRead(rwIOS) * * DO NOT CALL DIRECTLY. FOR INTERNAL LIBRW USE * * One of these functions should be called after a file has been * opened for reading or appending and the genericHeader has been * read. The function does whatever is required to prepare the file * for reading records of the specified type: checks that the version * is valid for this type of file, sets record and header lengths, * sets reading and writing function pointers appropriately for the * file's type and version, reads the remainder of the header, and * prints the entire header to the copyInputFD if required. * * Returns LIBRW_OK on success; otherwise returns error code on * failure: bad version, unable to read header. */ int _augmentedioPrepareWrite(rwIOStruct_t *rwIOS); int _augroutingioPrepareWrite(rwIOStruct_t *rwIOS); int _augsnmpoutioPrepareWrite(rwIOStruct_t *rwIOS); int _augwebioPrepareWrite(rwIOStruct_t *rwIOS); int _filterioPrepareWrite(rwIOStruct_t *rwIOS); int _genericioPrepareWrite(rwIOStruct_t *rwIOS); int _notroutedioPrepareWrite(rwIOStruct_t *rwIOS); int _routedioPrepareWrite(rwIOStruct_t *rwIOS); int _splitioPrepareWrite(rwIOStruct_t *rwIOS); int _wwwioPrepareWrite(rwIOStruct_t *rwIOS); /* * status = _ioPrepareWrite(rwIOS); * * DO NOT CALL DIRECTLY. FOR INTERNAL LIBRW USE * * One of these functions should be called after a file has been * opened for writing. The function does whatever is required to * prepare the file for writing records of the specified type: * checks that the version is valid for this type of file, sets * record and header lengths, sets reading and writing function * pointers appropriately for the file's type and version, and * creates---but does not write---the entire header. * * Returns LIBRW_OK on success; otherwise returns an error code on * failure: bad version, unable to read header. */ uint16_t _augmentedioGetRecLen(fileVersion_t); uint16_t _augroutingioGetRecLen(fileVersion_t); uint16_t _augsnmpoutioGetRecLen(fileVersion_t); uint16_t _augwebioGetRecLen(fileVersion_t); uint16_t _filterioGetRecLen(fileVersion_t); uint16_t _genericioGetRecLen(fileVersion_t); uint16_t _notroutedioGetRecLen(fileVersion_t); uint16_t _routedioGetRecLen(fileVersion_t); uint16_t _splitioGetRecLen(fileVersion_t); uint16_t _wwwioGetRecLen(fileVersion_t); /* * length = _ioGetRecLen(version); * * Return the on-disk length in bytes of records of the specified * type and vresion; or return 0 if the specified version is not * defined for the given type. */ /* ***** rwpack.c ***** */ int _packPackBytesPackets( uint32_t *bpp_out, uint32_t *pkts_out, uint32_t *pflag_out, const rwGenericRec_V3 *rwrec); /* * Uses fields from the rwRec pointed to by 'rwrec' to compute the * bytes-per-packet ('bpp'), packets ('pkts'), and * packets-multiplier ('pflag') field required by the packed file * formats FILTER, SPLIT, WWW, ROUTED, and NOTROUTED. * * The parameters 'bpp', 'pkts', and 'pflag' will be the values to * store in the packed file format; i.e., they will be the values * that _packUnpackBytesPackets() can read; they will be in native * byte order. * * Specifically, 'pkts' is either the packet count or the packet * count divided by the PKTS_DIVISOR when 'pflag' is non-zero. * 'bpp' is the bytes-per-packet ratio given by a 14 bit value and * a 6 bit fractional part. * * This function returns LIBRW_OK on success, or the following to * indicate an error: LIBRW_ERR_PKTS_ZERO-the 'pkts' field on rwrec * is 0; LIBRW_ERR_PKTS_OVRFLO-the 'pkts' value is too large to * store in the packed file format. */ void _packUnpackBytesPackets( rwGenericRec_V3 *rwrec, uint32_t bpp, uint32_t pkts, uint32_t pflag); /* * Does the reverse of _packPackBytesPackets(): Fills in the * 'bytes', 'packets', and 'bpp' fields of the rwRec pointed to by * 'rwrec'. All values are expected to be in native byte order. * * This function does no error checking. */ void _packPackProtoFlags( uint8_t *is_tcp_out, uint8_t *prot_flags_out, uint8_t *tcp_state_out, uint8_t *rest_flags_out, const rwGenericRec_V3 *rwrec); /* * Uses fields from the rwRec pointed to by 'rwrec' to compute the * values pointed to by these variables: * * is_tcp_out - 1 if the flow is TCP (proto==6); 0 otherwise * * prot_flags_out - protocol when is_tcp==0; bitwise OR of TCP * flags on ALL packages when is_tcp==1 and tcp_state!=0; TCP * flags on FIRST packet when is_tcp==1 and tcp_state!=0 * * tcp_state_out - value of tcp_state field on the rwrec * * rest_flags_out - the flags reported by the flow collector * when is_tcp==0 (even though there are no flags to report); * empty when is_tcp==1 and tcp_state==0; bitwise OR of TCP * flags on all but the first packet when is_tcp==1 and * tcp_state!=0. * * The output variables prot_flags, tcp_state, and rest_flags will * be the values to store in the packed file format; is_tcp can be * stored in a single bit. The values can be read by the * _packUnpackProtoFlags() function. * * This function should never fail, and thus has no return value. */ void _packUnpackProtoFlags( rwGenericRec_V3 *rwrec, uint8_t is_tcp, uint8_t prot_flags, uint8_t tcp_state, uint8_t rest_flags); /* * Does the reverse of _packPackProtoFlags(): Fills in the 'proto', * 'flags', 'init_flags', 'rest_flags', and 'tcp_state' fields on * the rwRec pointed to by 'rwrec'. All values are expected to be * in native byte order. * * This function does no error checking. */ int _packPackSbbPef( uint32_t *sbb_out, uint32_t *pef_out, const rwGenericRec_V3 *rwrec, uint32_t file_start_time); /* * Uses fields from the rwRec pointed to by 'rwrec' to compute the * 'sbb' and 'pef' fields used when packing SPLIT, WWW, ROUTED, and * NOTROUTED V1 and V2 files. 'file_start_time' is the time value * stored in the header--record times are offset from that time. * * Uses the sTime, elapsed, pkts, bytes in the rwrec to compute * these values. Any millisec values for sTime and/or elapsed on * the rwRec are ingored.. * * sbb and pef are returned in native byte order. * * Returns 0 on success or non-zero on these failures: rwrec's sTime * is earlier than the 'file_start_time' or is too large; elapsed * time is too large; packets field is zero or too large. */ void _packUnpackSbbPef( rwGenericRec_V3 *rwrec, uint32_t file_start_time, const uint32_t *sbb, const uint32_t *pef); /* * Does the reverse of _packPackSbbPef(): Fills in the 'sTime', * 'elapsed', 'bytes', 'pkts', and 'bpp' fields on the rwRec * pointed to by 'rwrec'. All values are expected to be in native * byte order. * * This function does no error checking. */ int _packPackTimeBytesPktsFlags( uint32_t *pkts_stime_out, uint32_t *bbe_out, uint32_t *msec_flags_out, const rwGenericRec_V3 *rwrec, uint32_t file_start_time); /* * Computes the 'pkts_stime', 'bbe', and 'msec_flags' fields used * when packing into various formats. * * Uses the sTime, elapsed, pkts, and bytes fields in the rwRec * pointed to by rwrec to compute these values. 'file_start_time' * is the hour stored in the file's header---record times are * offset from it. * * sbb and pef are returned in native byte order. * * Returns 0 on success or non-zero on these failures: rwrec's sTime * is earlier than rwIOS's sTime or is too large; elapsed time * is too large; packets field is too large. */ void _packUnpackTimeBytesPktsFlags( rwGenericRec_V3 *rwrec, uint32_t file_start_time, const uint32_t *pkts_stime, const uint32_t *bbe, const uint32_t *msec_flags); /* * Does the reverse of _packPackSbbPef(): Fills in the 'sTime', * 'elapsed', 'sTime_msec, 'elapsed_msec', 'bytes', 'pkts', and * 'bpp' fields on the rwRec pointed to by 'rwrec'. All values are * expected to be in native byte order. * * This function does no error checking. */ #endif /* _LIBRW_PRIV_H */ /* ** Local Variables: ** mode:c ** indent-tabs-mode:nil ** c-basic-offset:4 ** End: */