/* ** Copyright (C) 2001-2007 by Carnegie Mellon University. ** ** @OPENSOURCE_HEADER_START@ ** ** Use of the SILK system and related source code is subject to the terms ** of the following licenses: ** ** GNU Public License (GPL) Rights pursuant to Version 2, June 1991 ** Government Purpose License Rights (GPLR) pursuant to DFARS 252.225-7013 ** ** NO WARRANTY ** ** ANY INFORMATION, MATERIALS, SERVICES, INTELLECTUAL PROPERTY OR OTHER ** PROPERTY OR RIGHTS GRANTED OR PROVIDED BY CARNEGIE MELLON UNIVERSITY ** PURSUANT TO THIS LICENSE (HEREINAFTER THE "DELIVERABLES") ARE ON AN ** "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY ** KIND, EITHER EXPRESS OR IMPLIED AS TO ANY MATTER INCLUDING, BUT NOT ** LIMITED TO, WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE, ** MERCHANTABILITY, INFORMATIONAL CONTENT, NONINFRINGEMENT, OR ERROR-FREE ** OPERATION. CARNEGIE MELLON UNIVERSITY SHALL NOT BE LIABLE FOR INDIRECT, ** SPECIAL OR CONSEQUENTIAL DAMAGES, SUCH AS LOSS OF PROFITS OR INABILITY ** TO USE SAID INTELLECTUAL PROPERTY, UNDER THIS LICENSE, REGARDLESS OF ** WHETHER SUCH PARTY WAS AWARE OF THE POSSIBILITY OF SUCH DAMAGES. ** LICENSEE AGREES THAT IT WILL NOT MAKE ANY WARRANTY ON BEHALF OF ** CARNEGIE MELLON UNIVERSITY, EXPRESS OR IMPLIED, TO ANY PERSON ** CONCERNING THE APPLICATION OF OR THE RESULTS TO BE OBTAINED WITH THE ** DELIVERABLES UNDER THIS LICENSE. ** ** Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie ** Mellon University, its trustees, officers, employees, and agents from ** all claims or demands made against them (and any related losses, ** expenses, or attorney's fees) arising out of, or relating to Licensee's ** and/or its sub licensees' negligent use or willful misuse of or ** negligent conduct or willful misconduct regarding the Software, ** facilities, or other rights or assistance granted by Carnegie Mellon ** University under this License, including, but not limited to, any ** claims of product liability, personal injury, death, damage to ** property, or violation of any laws or regulations. ** ** Carnegie Mellon University Software Engineering Institute authored ** documents are sponsored by the U.S. Department of Defense under ** Contract F19628-00-C-0003. Carnegie Mellon University retains ** copyrights in all material produced under this contract. The U.S. ** Government retains a non-exclusive, royalty-free license to publish or ** reproduce these documents, or allow others to do so, for U.S. ** Government purposes only pursuant to the copyright license under the ** contract clause at 252.227.7013. ** ** @OPENSOURCE_HEADER_END@ */ #ifndef _RWPACK_H #define _RWPACK_H #include "silk.h" RCSIDENTVAR(rcsID_RWPACK_H, "$SiLK: rwpack.h 8269 2007-08-03 18:54:48Z mthomas $"); #include "utils.h" #include "skiobuf.h" #include "v5pdu.h" #include "skstringmap.h" #define SK_MAX_RECORD_SIZE 64 /* * The maximum size of a SiLK Flow record. */ #define SK_SNMP_INDEX_LIMIT 65536 /* * Limit of SNMP interface index values */ typedef uint8_t flowtypeID_t; /* * A flowtype is a class/type pair. It has a unique name and unique ID. */ typedef uint16_t sensorID_t; /* * Type to hold a sensor ID. Usually, a sensor is a router or * other flow collector. */ typedef uint8_t classID_t; /* * Type to hold a class ID. A class is not actually stored in packed * records (see flowtype). */ typedef uint8_t sensorgroupID_t; /* * Type to hold a sensor group ID. This is not actually stored in * packged records. */ typedef uint8_t fileFormat_t; /* * Various SiLK file formats. These are listed in silk_files.h. */ typedef uint8_t fileVersion_t; /* * A version of the file format. */ typedef uint8_t sk_compmethod_t; /* * The method used to write the data section of a file. */ #define SK_MAX_NUM_FLOWTYPES ((flowtypeID_t)64) /* * The maximum number of flowtypes that may be allocated. (All * valid flowtype IDs must be less than this number.) */ #define SK_INVALID_FLOWTYPE ((flowtypeID_t)0xFF) /* * The value for an invalid or unrecognized flow-type value */ #define SK_MAX_NUM_SENSORS ((sensorID_t)0xFFFF) /* * The maximum number of sensors that may be allocated. (All valid * sensor IDs must be less than this number. */ #define SK_INVALID_SENSOR ((sensorID_t)0xFFFF) /* * The value for an invalid or unrecognized sensor. */ #define SK_MAX_NUM_CLASSES ((classID_t)32) /* * The maximum number of classes that may be allocatd. (All valid * class IDs must be less than this number.) */ #define SK_INVALID_CLASS ((classID_t)0xFF) /* * The value for an invalid or unrecognized class. */ #define SK_MAX_NUM_SENSORGROUPS ((sensorgroupID_t)0xFF) /* * The maximum number of sensorgroups that may be allocated. (All * valid sensorgroup IDs must be less than this number.) */ #define SK_INVALID_SENSORGROUP ((sensorgroupID_t)0xFF) /* * The value for an invalid or unrecognized sensor. */ #define SK_FILE_VERSION_ANY ((fileVersion_t)0xFF) /* * Value meaning that any version is valid */ #define SK_INVALID_COMPMETHOD ((sk_compmethod_t)0xFF) /* * The value for an invalid or unrecognized compression method */ #define SK_MAX_STRLEN_FLOWTYPE 32 /* * The strlen() of the names of flowtypes, classes, and types will * be this size or less. Add 1 to allow for the NUL byte. */ #define SK_MAX_STRLEN_FILE_FORMAT 32 /* * The strlen() of the names of file formats will be this size or * less. */ #define SK_MAX_STRLEN_SENSOR 24 /* * The maximum length of a sensor name, not including the final * NUL. */ /* ** This is the generic record returned from ANY packed type. */ typedef struct rwGenericRec_V3 { ipUnion sIP; /* 0- 3 Source IP */ ipUnion dIP; /* 4- 7 Destination IP */ uint16_t sPort; /* 8- 9 Source port */ uint16_t dPort; /* 10-11 Destination port */ ipUnion nhIP; /* 12-15 Router Next Hop IP */ uint16_t input; /* 16-17 Router incoming SNMP interface */ uint16_t output; /* 18-19 Router outgoing SNMP interface */ uint32_t sTime; /* 20-23 Start time of flow-epoch secs */ uint32_t elapsed; /* 24-27 Duration of flow */ uint32_t pkts; /* 28-31 Count of packets */ uint32_t bytes; /* 32-35 Count of bytes */ uint8_t proto; /* 36 IP protocol */ flowtypeID_t flow_type; /* 37 Class & Type info */ sensorID_t sID; /* 38-39 Sensor ID */ uint8_t flags; /* 40 OR of all flags (Netflow flags) */ uint8_t init_flags; /* 41 TCP flags in first packet * or blank for "legacy" data */ uint8_t rest_flags; /* 42 TCP flags on non-initial packet * or blank for "legacy" data */ uint8_t tcp_state; /* 43 TCP state machine info (below)*/ uint32_t bpp; /* 44-47 Bytes-per-Pkt--eliminate roundoff */ uint16_t sTime_msec; /* 48-49 Start time fraction (in millisec) */ uint16_t elapsed_msec;/* 50-51 Elapsed time fraction (in msec) */ uint16_t application; /* 52-53 Indication of type of traffic */ } rwGenericRec_V3; #define rwRec rwGenericRec_V3 typedef rwRec * rwRecPtr; /* ** Values for tcp_state value in rwGeneric and packed formats */ /* No additional TCP-state machine information is available */ #define SK_TCPSTATE_NO_INFO 0x00 /* Additional TCP-state machine information is available. This must * be set for every TCP flow where the init_flags and rest_flags * fields are set. */ #define SK_TCPSTATE_EXPANDED 0x01 /* Flow received packets following the FIN packet that were not ACK or * RST packets. */ #define SK_TCPSTATE_FIN_FOLLOWED_NOT_ACK 0x08 /* Flow ends prematurely due to a timeout by the collector. */ #define SK_TCPSTATE_TIMEOUT_KILLED 0x20 /* Flow is a continuation of a previous flow that was killed * prematurely due to a timeout by the collector. */ #define SK_TCPSTATE_TIMEOUT_STARTED 0x40 /* * Macros to decode the ICMP type and code values. * * In NetFlow, Cisco has traditionally encoded the ICMP type and * code in the DESTPORT field as (type << 8 | code); however, some * routers encode it in the SRCPORT field as (code << 8 | type) * (basically byteswapping the 32-bit "SRCPORT DESTPORT" value). * * Our current packing code will modify these odd SRCPORT encodings * to be in the more traditional DESTPORT encoding when the record * is initially packed, though these macros will work for the * window before our packing code was modified. If the DESTPORT is * non-zero, we assume it contains the ICMP values; otherwise, we * look at the SRCPORT values. Note that these macros do not check * the protocol. */ #define RWREC_ICMP_TYPE(r) \ ((unsigned)(0xFF & (((r)->dPort > 0) \ ? ((r)->dPort >> 8) \ : ((r)->sPort)))) #define RWREC_ICMP_CODE(r) \ ((unsigned)(0xFF & (((r)->dPort > 0) \ ? ((r)->dPort) \ : ((r)->sPort >> 8)))) #define rwRecGetIcmpType(r) \ ((unsigned)(0xFF & (((r)->dPort > 0) \ ? ((r)->dPort >> 8) \ : ((r)->sPort)))) #define rwRecSetIcmpType(r, in_v) \ ((r)->dPort = ((((in_v) & 0xFF) << 8) | ((r)->dPort & 0xFF)) #define rwRecMemGetIcmpType(r, out_vp) \ do { \ uint8_t _tmp08 = (uint8_t)rwRecGetIcmpType(r); \ *out_vp = _tmp08; \ } while(0) #define rwRecGetIcmpCode(r) \ ((unsigned)(0xFF & (((r)->dPort > 0) \ ? ((r)->dPort) \ : ((r)->sPort >> 8)))) #define rwRecSetIcmpCode(r, in_v) \ ((r)->dPort = (((r)->dPort & 0xFF00) | ((in_v) & 0xFF)) #define rwRecMemGetIcmpCode(r, out_vp) \ do { \ uint8_t _tmp08 = (uint8_t)rwRecGetIcmpCode(r); \ *out_vp = _tmp08; \ } while(0) #define rwRecGetSIP(r) ((r)->sIP.ipnum) #define rwRecSetSIP(r, in_v) (((r)->sIP.ipnum) = (in_v)) #define rwRecMemGetSIP(r, out_vp) memcpy((out_vp), &((r)->sIP.ipnum), 4) #define rwRecMemSetSIP(r, in_vp) memcpy(&((r)->sIP.ipnum), (in_vp), 4) #define rwRecMemCmpSIP(r, vp) memcmp(&((r)->sIP.ipnum), (vp), 4) #define rwRecGetDIP(r) ((r)->dIP.ipnum) #define rwRecSetDIP(r, in_v) (((r)->dIP.ipnum) = (in_v)) #define rwRecMemGetDIP(r, out_vp) memcpy((out_vp), &((r)->dIP.ipnum), 4) #define rwRecMemSetDIP(r, in_vp) memcpy(&((r)->dIP.ipnum), (in_vp), 4) #define rwRecMemCmpDIP(r, vp) memcmp(&((r)->dIP.ipnum), (vp), 4) #define rwRecGetSPort(r) ((r)->sPort) #define rwRecSetSPort(r, in_v) (((r)->sPort) = (in_v)) #define rwRecMemGetSPort(r, out_vp) memcpy((out_vp), &((r)->sPort), 2) #define rwRecMemSetSPort(r, in_vp) memcpy(&((r)->sPort), (in_vp), 2) #define rwRecMemCmpSPort(r, vp) memcmp(&((r)->sPort), (vp), 2) #define rwRecGetDPort(r) ((r)->dPort) #define rwRecSetDPort(r, in_v) (((r)->dPort) = (in_v)) #define rwRecMemGetDPort(r, out_vp) memcpy((out_vp), &((r)->dPort), 2) #define rwRecMemSetDPort(r, in_vp) memcpy(&((r)->dPort), (in_vp), 2) #define rwRecMemCmpDPort(r, vp) memcmp(&((r)->dPort), (vp), 2) #define rwRecGetProto(r) ((r)->proto) #define rwRecSetProto(r, in_v) (((r)->proto) = (in_v)) #define rwRecMemGetProto(r, out_vp) memcpy((out_vp), &((r)->proto), 1) #define rwRecMemSetProto(r, in_vp) memcpy(&((r)->proto), (in_vp), 1) #define rwRecMemCmpProto(r, vp) memcmp(&((r)->proto), (vp), 1) #define rwRecGetPkts(r) ((r)->pkts) #define rwRecSetPkts(r, in_v) (((r)->pkts) = (in_v)) #define rwRecMemGetPkts(r, out_vp) memcpy((out_vp), &((r)->pkts), 4) #define rwRecMemSetPkts(r, in_vp) memcpy(&((r)->pkts), (in_vp), 4) #define rwRecMemCmpPkts(r, vp) memcmp(&((r)->pkts), (vp), 4) #define rwRecGetBytes(r) ((r)->bytes) #define rwRecSetBytes(r, in_v) (((r)->bytes) = (in_v)) #define rwRecMemGetBytes(r, out_vp) memcpy((out_vp), &((r)->bytes), 4) #define rwRecMemSetBytes(r, in_vp) memcpy(&((r)->bytes), (in_vp), 4) #define rwRecMemCmpBytes(r, vp) memcmp(&((r)->bytes), (vp), 4) #define rwRecGetFlags(r) ((r)->flags) #define rwRecSetFlags(r, in_v) (((r)->flags) = (in_v)) #define rwRecMemGetFlags(r, out_vp) memcpy((out_vp), &((r)->flags), 1) #define rwRecMemSetFlags(r, in_vp) memcpy(&((r)->flags), (in_vp), 1) #define rwRecMemCmpFlags(r, vp) memcmp(&((r)->flags), (vp), 1) #define rwRecGetSecondsSTime(r) ((r)->sTime) #define rwRecSetSecondsSTime(r, in_v) (((r)->sTime) = (in_v)) #define rwRecMemGetSecondsSTime(r, out_vp) memcpy((out_vp), &((r)->sTime), 4) #define rwRecMemSetSecondsSTime(r, in_vp) memcpy(&((r)->sTime), (in_vp), 4) #define rwRecMemCmpSecondsSTime(r, vp) memcmp(&((r)->sTime), (vp), 4) #define rwRecGetSecondsElapsed(r) ((r)->elapsed) #define rwRecSetSecondsElapsed(r, in_v) (((r)->elapsed) = (in_v)) #define rwRecMemGetSecondsElapsed(r, out_vp) \ memcpy((out_vp), &((r)->elapsed), 4) #define rwRecMemSetSecondsElapsed(r, in_vp) \ memcpy(&((r)->elapsed), (in_vp), 4) #define rwRecMemCmpSecondsElapsed(r, vp) \ memcpy(&((r)->elapsed), (vp), 4) #define rwRecGetSecondsETime(r) \ (((r)->sTime) + (r)->elapsed + \ ((((r)->sTime_msec + (r)->elapsed_msec) >= 1000) ? 1 : 0)) #define rwRecMemGetSecondsETime(r, out_vp) \ do { \ uint32_t _tmp32 = rwRecGetSecondsETime(r); \ memcpy((out_vp), &_tmp32, 4); \ } while(0) #define rwRecGetSTimeMSec(r) ((r)->sTime_msec) #define rwRecSetSTimeMSec(r, in_v) (((r)->sTime_msec) = (in_v)) #define rwRecMemGetSTimeMSec(r, out_vp) memcpy((out_vp), &((r)->sTime_msec), 2) #define rwRecMemSetSTimeMSec(r, in_vp) memcpy(&((r)->sTime_msec), (in_vp), 2) #define rwRecMemCmpSTimeMSec(r, vp) memcmp(&((r)->sTime_msec), (vp), 2) #define rwRecGetElapsedMSec(r) ((r)->elapsed_msec) #define rwRecSetElapsedMSec(r, in_v) (((r)->elapsed_msec) = (in_v)) #define rwRecMemGetElapsedMSec(r, out_vp) \ memcpy((out_vp), &((r)->elapsed_msec), 2) #define rwRecMemSetElapsedMSec(r, in_vp) \ memcpy(&((r)->elapsed_msec), (in_vp), 2) #define rwRecMemCmpElapsedMSec(r, vp) \ memcmp(&((r)->elapsed_msec), (vp), 2) #define rwRecGetETimeMSec(r) \ ((uint16_t)(((r)->sTime_msec + (r)->elapsed_msec) % 1000)) #define rwRecMemGetETimeMSec(r, out_vp) \ do { \ uint16_t _tmp16 = rwRecGetETimeMSec(r); \ memcpy((out_vp), &_tmp16, 2); \ } while(0) #if 0 #define rwRecMemGetEtime(r, out_tvp) \ do { \ uint32_t _tmp32; \ uint16_t _tmp16; \ memcpy(&_tmp32, &((r)->sTime), 4); \ memcpy(&_tmp16, &((r)->sTime_msec), 2); \ (out_tvp)->tv_sec = (time_t)_tmp32; \ (out_tvp)->tv_usec = (long)_tmp16 * 1000; \ memcpy(&_tmp32, &((r)->elapsed), 4); \ memcpy(&_tmp16, &((r)->elapsed_msec), 2); \ (out_tvp)->tv_sec += (time_t)_tmp32; \ (out_tvp)->tv_usec += (long)_tmp16 * 1000; \ if ((out_tvp)->tv_usec > 1000000) { \ ++(out_tvp)->tv_sec; \ (out_tvp)->tv_usec -= 1000000; \ } \ } while(0) #define rwRecGetMilliSTime(r) \ ((uint64_t)((r)->sTime) * 1000 + (uint64_t)((r)->sTime_msec)) #define rwRecGetMicroSTime(r) \ ((uint64_t)((r)->sTime) * 1000000 + (uint64_t)((r)->sTime_msec) * 1000) #define rwRecMemGetTimevalSTime(r, out_tvp) \ do { \ uint32_t _tmp32; \ uint16_t _tmp16; \ memcpy(&_tmp32, &((r)->sTime), 4); \ memcpy(&_tmp16, &((r)->sTime_msec), 2); \ (out_tvp)->tv_sec = (time_t)_tmp32; \ (out_tvp)->tv_usec = (long)_tmp16 * 1000; \ } while(0) #define rwRecMemSetSTime(r, in_tvp) \ do { \ uint32_t _tmp32 = (in_tvp)->tv_sec; \ uint16_t _tmp16 = (in_tvp)->tv_usec / 1000; \ memcpy(&((r)->sTime), &_tmp32, 4); \ memcpy(&((r)->sTime_msec), &_tmp16, 2); \ } while(0) #define rwRecGetElapsed_sec(r) ((r)->elapsed) #define rwRecMemGetElapsed(r, out_vp) memcpy((out_vp), &((r)->elapsed), 1) #define rwRecMemSetElapsed(r, in_vp) memcpy(&((r)->elapsed), (in_vp), 1) #define rwRecMemCmpElapsed(r, vp) memcmp(&((r)->elapsed), (vp), 1) #define rwRecMemSetSTime(r, in_vp) memcpy(&((r)->flags), (in_vp), 1) #define rwRecMemCmpSTime(r, vp) memcmp(&((r)->flags), (vp), 1) uint16_t sTime_msec; /* 48-49 Start time fraction (in millisec) */ uint16_t elapsed_msec;/* 50-51 Elapsed time fraction (in msec) */ #endif /* 0 */ #define rwRecGetSID(r) ((r)->sID) #define rwRecSetSID(r, in_v) (((r)->sID) = (in_v)) #define rwRecMemGetSID(r, out_vp) memcpy((out_vp), &((r)->sID), 2) #define rwRecMemSetSID(r, in_vp) memcpy(&((r)->sID), (in_vp), 2) #define rwRecMemCmpSID(r, vp) memcmp(&((r)->sID), (vp), 2) #define rwRecGetNhIP(r) ((r)->nhIP.ipnum) #define rwRecSetNhIP(r, in_v) (((r)->nhIP.ipnum) = (in_v)) #define rwRecMemGetNhIP(r, out_vp) memcpy((out_vp), &((r)->nhIP.ipnum), 4) #define rwRecMemSetNhIP(r, in_vp) memcpy(&((r)->nhIP.ipnum), (in_vp), 4) #define rwRecMemCmpNhIP(r, vp) memcmp(&((r)->nhIP.ipnum), (vp), 4) #define rwRecGetInput(r) ((r)->input) #define rwRecSetInput(r, in_v) (((r)->input) = (in_v)) #define rwRecMemGetInput(r, out_vp) memcpy((out_vp), &((r)->input), 2) #define rwRecMemSetInput(r, in_vp) memcpy(&((r)->input), (in_vp), 2) #define rwRecMemCmpInput(r, vp) memcmp(&((r)->input), (vp), 2) #define rwRecGetOutput(r) ((r)->output) #define rwRecSetOutput(r, in_v) (((r)->output) = (in_v)) #define rwRecMemGetOutput(r, out_vp) memcpy((out_vp), &((r)->output), 2) #define rwRecMemSetOutput(r, in_vp) memcpy(&((r)->output), (in_vp), 2) #define rwRecMemCmpOutput(r, vp) memcmp(&((r)->output), (vp), 2) #define rwRecGetInitFlags(r) ((r)->init_flags) #define rwRecSetInitFlags(r, in_v) (((r)->init_flags) = (in_v)) #define rwRecMemGetInitFlags(r, out_vp) memcpy((out_vp), &((r)->init_flags), 1) #define rwRecMemSetInitFlags(r, in_vp) memcpy(&((r)->init_flags), (in_vp), 1) #define rwRecMemCmpInitFlags(r, vp) memcmp(&((r)->init_flags), (vp), 1) #define rwRecGetRestFlags(r) ((r)->rest_flags) #define rwRecSetRestFlags(r, in_v) (((r)->rest_flags) = (in_v)) #define rwRecMemGetRestFlags(r, out_vp) memcpy((out_vp), &((r)->rest_flags), 1) #define rwRecMemSetRestFlags(r, in_vp) memcpy(&((r)->rest_flags), (in_vp), 1) #define rwRecMemCmpRestFlags(r, vp) memcmp(&((r)->rest_flags), (vp), 1) #define rwRecGetTcpState(r) ((r)->tcp_state) #define rwRecSetTcpState(r, in_v) (((r)->tcp_state) = (in_v)) #define rwRecMemGetTcpState(r, out_vp) memcpy((out_vp), &((r)->tcp_state), 1) #define rwRecMemSetTcpState(r, in_vp) memcpy(&((r)->tcp_state), (in_vp), 1) #define rwRecMemCmpTcpState(r, vp) memcmp(&((r)->tcp_state), (vp), 1) #define rwRecGetFlowType(r) ((r)->flow_type) #define rwRecSetFlowType(r, in_v) (((r)->flow_type) = (in_v)) #define rwRecMemGetFlowType(r, out_vp) memcpy((out_vp), &((r)->flow_type), 1) #define rwRecMemSetFlowType(r, in_vp) memcpy(&((r)->flow_type), (in_vp), 1) #define rwRecMemCmpFlowType(r, vp) memcmp(&((r)->flow_type), (vp), 1) #define rwRecGetApplication(r) ((r)->application) #define rwRecSetApplication(r, in_v) (((r)->application) = (in_v)) #define rwRecMemGetApplication(r, out_vp) \ memcpy((out_vp), &((r)->application), 2) #define rwRecMemSetApplication(r, in_vp) \ memcpy(&((r)->application), (in_vp), 2) #define rwRecMemCmpApplication(r, vp) \ memcpy(&((r)->application), (vp), 2) typedef enum { /* Used internally to mark stream as closed. Currently, * re-opening a stream is not supported. */ SK_RWIO_CLOSED = 0, /* Open an existing file for read. */ SK_RWIO_READ = 1, /* Open a new file for output; file must not exist unless output * is to a named pipe. */ SK_RWIO_WRITE = 2, /* Open an existing file for appending. Must be a regular file * that is not compressed. */ SK_RWIO_APPEND = 3 } rwio_mode_t; /* struct for rw file io */ struct _rwIOStruct; typedef struct _rwIOStruct rwIOStruct; typedef struct _rwIOStruct rwIOStruct_t; typedef struct _rwIOStruct* rwIOStructPtr; struct _rwIOStruct { /* Number of records read or written. For appending, this is the * number records added to the file. */ uint64_t recCount; /* Pointer to a function to convert an array of bytes into a record */ int (*rwUnpackFn)(rwIOStruct_t *, rwRec *, const uint8_t *); /* Pointer to a function to convert a record into an array of bytes */ int (*rwPackFn)(rwIOStruct_t *, const rwRec *, uint8_t *); /* Pointer to a function to byte-swap a record in place */ void (*rwSwapFn)(uint8_t *); /* Pointer to a function to write the file's header */ int (*writeHdrFn)(rwIOStruct_t *, const rwRec *); /* Path to the file; the rwIOStruct owns this memory */ char *fPath; /* The stream to copy the input to---for support of the --all-dest * switch */ rwIOStruct_t *copyInputFD; /* Pointer to the file's header: genericHeader plus whatever is * specific to the file format. Responsibility of the opener to * know what this is. */ void *hdr; /* The buffer used to read/write data. Also handles compression * of the records */ sk_iobuf_t *iobuf; #if HAVE_ZLIB_H /* When the entire file has been compressed, we use gzread/gzwrite * to process the file, this is interface to those functions */ gzFile gz; #endif /* An object to hold the parameter that caused the last error */ union { uint32_t num; const char *str; const rwRec *rec; } errobj; /* The errno from the last system call that failed */ int errnum; /* The file descriptor */ int _fd; /* Length of header as stored on disk not including any header * padding. */ uint32_t hdrLen; /* The fixed length of records of this type */ uint16_t recLen; /* Sensor ID of the file. Not used for FT_RWGENERIC or * FT_RWFILTER files. */ sensorID_t sID; /* Whether stream is read, write, append. */ rwio_mode_t io_mode; /* The flow-type (class+type information) of the file as defined * in silk_site_*.h. Not used for FT_RWGENERIC or FT_RWFILTER * files. */ flowtypeID_t flow_type; /* Whether the file is open */ unsigned isOpenFlag :1; /* Whether the header has been written */ unsigned didHdrFlag :1; /* Whether the bytes need to be swapped on read/write */ unsigned swapFlag :1; /* Whether we reached the end of this file */ unsigned eofFlag :1; /* Whether the stream is talking over MPI */ unsigned mpiFlag :1; /* Whether PVFS file system is in use */ unsigned pvfsFlag :1; }; /* Error codes that the rwio*() functions may return */ typedef enum { /* This indicates no error */ LIBRW_OK = 0, /* The following set of errors affect only the current record; * they occur when trying to write a record to a stream. */ /* The record's start time is less than the file's start time */ LIBRW_ERR_STIME_UNDRFLO = 17, /* The record's start time at least an hour greater than the * file's start time */ LIBRW_ERR_STIME_OVRFLO, /* The record's elapsed time is greater than space allocated for * duration in this file format */ LIBRW_ERR_ELPSD_OVRFLO, /* The record contains more than the number of packets allowed in * this file format */ LIBRW_ERR_PKTS_OVRFLO, /* The record contains a 0 value in the packets field. */ LIBRW_ERR_PKTS_ZERO, /* The records contains an SNMP value too large to fit into the * space allocated in this file format. */ LIBRW_ERR_SNMP_OVRFLO, /* The records contains a SensorID too large to fit into the space * allocated in this file format. */ LIBRW_ERR_SENSORID_OVRFLO, /* The record's IP protocol is not supported by the file's format; * for example, trying to store a non-TCP record in a FT_RWWWW * file. */ LIBRW_ERR_PROTO_MISMATCH, /* The record's "packets" value is greater than the "bytes" value. */ LIBRW_ERR_PKTS_GT_BYTES, /* ERRORS < 32 are not fatal. * Use the LIBRW_ERROR_IS_FATAL() macro below. */ /* The following set of errors can occur at any time and generally * indicate a fatal condition. */ /* The call to write() or fwrite() failed */ LIBRW_ERR_WRITE = 33, /* Caller passed an unexpected NULL value into a function; may * also be returned when function receives the empty string. */ LIBRW_ERR_NULL_ARGUMENT, /* The following set of errors are general errors that occur when * opening a SiLK file for read, write, or append. */ /* Cannot allocate memory; this should only occur during the * initial opening of a file and processing of its header. */ LIBRW_ERR_ALLOC = 65, /* The file has a type that does not support this operation. */ LIBRW_ERR_UNSUPPORT_TYPE, /* The file as a version that this version of librw does not know * how to handle */ LIBRW_ERR_UNSUPPORT_VERSION, /* Caller is attempting to read from stdout, write to stdin, * append to a stream that is not a regular file, or read from a * temporary file. */ LIBRW_ERR_UNSUPPORT_STREAM, /* Caller is attempting to process binary data on a terminal */ LIBRW_ERR_STREAM_TTY, /* The caller cannot change attributes on the rwIOStream; * typically this means the caller is attempting to change either * a stream opened for write to which the header has already been * written or a stream opened for read or append. */ LIBRW_ERR_ATTRIBUTE_FIXED, /* Caller is attempting to call open on a previously opened * stream. */ LIBRW_ERR_STREAM_OPEN, /* Caller is attempting to lock, read-from, or write-to an * unopened stream, i.e., a stream that has not yet been * opened. */ LIBRW_ERR_STREAM_NOT_OPEN, /* Caller is attempting to operate on a stream that has been * closed. Re-opening of streams is not supported. */ LIBRW_ERR_STREAM_CLOSED, /* Call to open(), fopen(), or popen() failed */ LIBRW_ERR_OPEN, /* Cannot open file descriptor as a file pointer */ LIBRW_ERR_FDOPEN, /* Call to mkstemp() failed. */ LIBRW_ERR_MKSTEMP, /* Unable to read data from a stream. */ LIBRW_ERR_READ, /* Unable to read data from a stream because there is no data. */ LIBRW_ERR_READ_EOF, /* The following set of errors occur when opening a file or stream * for writing. */ /* Cannot create a file for writing because a file already exists * at that location */ LIBRW_ERR_WRITE_FILE_EXISTS, /* Cannot create the directory path necessary to create the file * for writing */ LIBRW_ERR_MKDIRPATH, /* The following set of errors occur when opening an existing a * file for appending. */ /* File does not exist */ LIBRW_ERR_OPEN_NOEXIST, /* Cannot seek to end of file */ LIBRW_ERR_FSEEK, /* Attempt to compress to method that is not available---due to a * missing library---or that is not recognized. */ LIBRW_ERR_UNAVAILABLE_COMPRESS, /* File is compressed---cannot append to a compressed file */ LIBRW_ERR_UNSUPPORT_COMPRESS, /* File does not appear to be a SiLK file */ LIBRW_ERR_BAD_MAGIC, /* Attempting to open a file as a type different from the one that * was used to write the file. */ LIBRW_ERR_MISMATCH_TYPE, /* Attempting to open a file as a version different from the one * that was used to write the file. */ LIBRW_ERR_MISMATCH_VERSION, /* Attempting to open a file in a byte-order different from the * one that was used to write the file. */ LIBRW_ERR_MISMATCH_ENDIAN, /* Attempting to open a file with a start-time different from the * one that was used to write the file. */ LIBRW_ERR_MISMATCH_STARTIME, /* The following set of errors occur when opening a file for * writing or for appending. */ /* Cannot get an exclusive write lock on the file */ LIBRW_ERR_WLOCK, /* Cannot get a shared read lock on the file */ LIBRW_ERR_RLOCK, /* Error with underlying skIOBuf object */ LIBRW_ERR_IOBUF } librwErrorCodes_t; /* * LIBRW_ERROR_IS_FATAL(err); * * Evaluates to a true value if error is a fatal error, false otherwise. */ #define LIBRW_ERROR_IS_FATAL(err) ((err) >= 32) /* structure to hold a single command line invocation */ struct _cmd_invoc; typedef struct _cmd_invoc sk_cmd_invoc_t; typedef struct _cmd_history { /* Number of valid command lines in the cmd_array[] */ uint32_t cmd_count; /* The allocated size of the cmd_array[] */ uint32_t cmd_capacity; /* An array of command lines */ sk_cmd_invoc_t *cmd_array; } sk_cmd_history_t; struct _cmd_invoc { /* Number of bytes in this information set */ uint16_t cmd_length; /* The command line */ char *cmd_info; }; /* struct to hold the filter header itself for FT_RWFILTER. */ typedef struct filterHeaderV1 { /* The generic header */ genericHeader gHdr; /* Command line history */ sk_cmd_history_t cmdHist; } filterHeaderV1; typedef struct rwFileHeaderV0 { genericHeader gHdr; uint32_t fileSTime; } rwFileHeaderV0; /* * This is the header used in all "packed data" files which store * time as an offset from the "fileSTime" value in the header. * This includes the FT_RWROUTED, FT_RWNOTROUTED, FT_RWSPLIT, and * FT_RWWWW types. */ /* macros for accessing header fields */ #define rwGetIsBigEndian(a) (((genericHeader *)(a)->hdr)->isBigEndian) #define rwGetFileType(a) (((genericHeader *)(a)->hdr)->type) #define rwGetFileVersion(a) (((genericHeader *)(a)->hdr)->version) #define rwGetFileCompMethod(a) (((genericHeader *)(a)->hdr)->compMethod) #define rwGetFileSTime(a) (((rwFileHeaderV0 *)(a)->hdr)->fileSTime) #define rwGetFD(a) ((a)->_fd) #define rwGetFileName(a) ((a)->fPath) #define rwGetHeader(a) ((a)->hdr) #define rwGetRecordSize(a) ((a)->recLen) #define rwGetRecordCount(a) ((a)->recCount) #define rwGetFileByteorder(a) \ (rwGetIsBigEndian(a) ? SILK_ENDIAN_BIG : SILK_ENDIAN_LITTLE) uint32_t rwGetHeaderLength( const rwIOStruct_t *rwIOS); int rwioCreate( rwIOStruct **rwIOS, const char *pathname, rwio_mode_t read_write_append); /* * Create a new rwio Stream at the location pointed to by rwIOS. * The new stream is bound to the file at 'pathname', and is * created for read (SK_RWIO_READ), write (SK_RWIO_WRITE), or * append (SK_RWIO_APPEND) according to the value in * 'read_write_append'. * * For write, the file's format is set to the default version of * FT_RWGENERIC using the machine's native byte order and the * default compression specified when SiLK was configured. * * The function returns a value defined in librwErrorCodes_t. */ int rwioSetHeaderFromFile( rwIOStruct_t *rwIOS, const rwIOStruct_t *source_ios); /* * Copy the file headers from the 'source_ios' to 'rwIOS'. 'rwIOS' * should be created for write but not yet open. * * The function returns a value defined in librwErrorCodes_t. */ int rwioSetFileType( rwIOStruct *rwIOS, uint8_t file_type); /* * Set the file format that will be used by 'rwIOS' to 'file_type'. * 'rwIOS' should be created for write but not yet open. * * Available file formats are specified in silk_files.h; not all * formats are supported by rwio. * * The function returns a value defined in librwErrorCodes_t. */ int rwioSetFileVersion( rwIOStruct *rwIOS, fileVersion_t file_version); /* * Set the version of the file format that will be used by 'rwIOS' * to 'file_version'. 'rwIOS' should be created for write but not * yet open. * * The function returns a value defined in librwErrorCodes_t. */ int rwioSetCompression( rwIOStruct *rwIOS, sk_compmethod_t comp_method); /* * Set the compression method that will be used by 'rwIOS' to * 'comp_method'. 'rwIOS' should be created for write but not yet * open. * * Supported compression methods are defined in silk_files.h. * Whether a method is available depends on whether the library was * available when SiLK was compiled. * * The function returns a value defined in librwErrorCodes_t. */ int rwioSetFileByteorder( rwIOStruct *rwIOS, silk_endian_t byte_order); int rwioSetFileSTime( rwIOStruct *rwIOS, uint32_t start_time, int use_floor_hour); int rwioAppendToHistoryFromFile( rwIOStruct *rwIOS, const rwIOStruct *source_rwIOS); int rwioAppendToHistoryArgv( rwIOStruct *rwIOS, int argc, char * const *argv); int rwioOpen( rwIOStruct *rwIOS); /* * Open the pathname that 'rwIOS' is bound to. For an 'rwIOS' that * were created for read or append, this will read the file's * header. * * The function returns a value defined in librwErrorCodes_t. */ int rwioFDOpen( rwIOStruct *rwIOS, int file_desc); /* * Bind 'rwIOS' to the previously opened file descriptor * 'file_desc'. For an 'rwIOS' that were created for read or * append, this will read the file's header. * * The function returns a value defined in librwErrorCodes_t. */ int rwioMakeTemp( rwIOStruct *rwIOS); /* * Use the pathname specified when 'rwIOS' was created as a * template to mkstemp(3) to open a uniquely named temporary file. * * The function returns a value defined in librwErrorCodes_t. */ int rwioLockFile( rwIOStruct *rwIOS); /* * Create an advisory file lock on 'rwIOS'. If 'rwIOS' was created * for read, the lock is shared; for append or write, the lock is * exclusive. */ int rwioMakeDirectory( rwIOStruct *rwIOS); /* * Create any components of the directory path that would be * required to open the pathname that is bound to 'rwIOS'. */ int rwioWriteHeader( rwIOStruct *rwIOS); int rwioFlush( rwIOStruct *rwIOS); int rwioClose( rwIOStruct *rwIOS); int rwioDestroy( rwIOStruct **rwIOS); void rwioPrintLastErr( rwIOStruct *rwIOS, int error_code, sk_msg_fn_t errfn); int rwioReadRecord( rwIOStruct_t *rwIOS, rwRec *rwRP); int rwioWriteRecord( rwIOStruct_t *rwIOS, const rwRec *rwRP); ssize_t _ioWrite( rwIOStruct_t *rwIOS, const void *buf, size_t count); /* * Write 'count' bytes from specified 'buf' byte array to the * stream in 'rwIOS'. Returns the number of bytes actually * written, or -1 to indicate an error. */ int rwioSkipRecords( rwIOStruct_t *rwIOS, size_t skip_count); /* Older API */ #define rwRead(a, b) (!(rwioReadRecord((a), (b)))) #define rwWrite(a, b) rwioWriteRecord((a), (b)) #define rwSkip(a, b) rwioSkipRecords((a), (b)) rwIOStruct *rwOpenFile(const char *fPath, rwIOStruct *copyInput); /* * rwIOSPtr = rwOpenFile(filename, copy_stream); * * Open the given file 'filename' and based on the header * information setup everything we need to read through the file * always returning rwRec. * * If 'copy_stream' is not-NULL, it should be an open rwIOStruct* * stream to copy the input to. * * Return a pointer to a new struct to contain all the information * needed to traverse the file. Return NULL on error. * * If 'filename' is "stdin", then read binary input stream from * stdin. */ #define rwCloseFile(rwio) \ (((rwio) == NULL) ? LIBRW_OK : rwioDestroy(&(rwio))) /* *** Printing rwRec as ASCII *** */ /* Number of fields we can print; should be one more than the last ID * in rwrec_printable_fields_t */ #define RWREC_PRINTABLE_FIELD_COUNT 25 /* Maximum width of the name of a field */ #define RWREC_PRINTABLE_MAX_NAME_LEN 16 /* An identifier for each field */ typedef enum { RWREC_FIELD_SIP, RWREC_FIELD_DIP, RWREC_FIELD_SPORT, RWREC_FIELD_DPORT, RWREC_FIELD_PROTO, RWREC_FIELD_PKTS, RWREC_FIELD_BYTES, RWREC_FIELD_FLAGS, RWREC_FIELD_STIME, RWREC_FIELD_ELAPSED, RWREC_FIELD_ETIME, RWREC_FIELD_SID, RWREC_FIELD_INPUT, RWREC_FIELD_OUTPUT, RWREC_FIELD_NHIP, RWREC_FIELD_INIT_FLAGS, RWREC_FIELD_REST_FLAGS, RWREC_FIELD_TCP_STATE, RWREC_FIELD_APPLICATION, RWREC_FIELD_FTYPE_CLASS, RWREC_FIELD_FTYPE_TYPE, RWREC_FIELD_STIME_MSEC, RWREC_FIELD_ETIME_MSEC, RWREC_FIELD_ELAPSED_MSEC, RWREC_FIELD_ICMP_TYPE_CODE } rwrec_printable_fields_t; /* A type for printing records as ASCII. Created via * rwAsciiStreamCreate() */ typedef struct _rwAsciiStream rwAsciiStream_t; extern int rwAsciiStreamCreate(rwAsciiStream_t **stream); /* * Create a new output stream for printing rwRec records in a human * readable form. Store the newly allocated rwAsciiStream_t in the * memory pointed to by **stream. Return 0 on success or non-zero if * allocation fails. * * The caller may immediately call rwAsciiPrintRec() to print records * to the stream, or the caller may use the functions listed below to * change the defaults. Calling an rwAsciiSet*() function once * titles or records have been printed may result in strange output. * * The defaults are (function that modifies this behavior): * -- Output is sent to the standard output (rwAsciiSetOutputHandle) * -- All fields are printed (rwAsciiSetFields) * -- Column titles are printed before first record (rwAsciiSetNoTitles) * -- Fields are printed in a columnar format (rwAsciiSetNoColumns) * -- Delimiter between columns is '|' * -- A newline is printed after final column (rwAsciiSetNoNewline) * -- IPs are printed in dotted quad (rwAsciiSetIntegerIps, * rwAsciiSetZeroPadIps) * -- Times are printed in human readable form (rwAsciiSetEpochTime) * -- Sensor names are printed (rwAsciiSetIntegerSensors) * -- No special handling of ICMP (rwAsciiSetIcmpTypeCode) */ extern void rwAsciiStreamDestroy(rwAsciiStream_t **stream); /* * Free all memory associated with the stream. It is the caller's * responsibility to fflush() the underlying file pointer. */ extern void rwAsciiPrintRec( rwAsciiStream_t *stream, const rwRec *rwrec); /* * Print 'rwrec' in a human-readable form to 'stream'. Will print * the column titles when the stream is configured to have titles and * the titles have not yet been printed. */ extern void rwAsciiPrintTitles(rwAsciiStream_t *stream); /* * Print the column titles when the stream is configured to have * titles and they have not been printed. */ extern void rwAsciiSetOutputHandle(rwAsciiStream_t *stream, FILE *fh); /* * Configure the 'stream' to print the output to 'fh'. If 'fh' is * NULL, stdout is used. */ extern int rwAsciiSetFields( rwAsciiStream_t *stream, const uint32_t *field_list, uint32_t field_count); /* * Configure the 'stream' to print the fields listed in 'field_list', * which is an array of 'field_count' elements. If field_list is * NULL or field_count is 0, returns non-zero. Any previously * configured field_list will be lost. */ extern void rwAsciiSetNoTitles(rwAsciiStream_t *stream); /* * Configure the 'stream' not to print titles before the first record * of output. */ extern void rwAsciiSetDelimiter(rwAsciiStream_t *stream, char delimiter); /* * Configure the 'stream' to put the character 'delimiter' between * columns. Does not effect the columnar setting. */ extern void rwAsciiSetNoColumns(rwAsciiStream_t *stream); /* * Configure the 'stream' not to use fixed size columns. This avoids * extra whitespace, but makes the output difficult for humans to * read. */ extern void rwAsciiSetNoNewline(rwAsciiStream_t *stream); /* * Configure the 'stream' not to print a newline after the final field. */ extern void rwAsciiSetIntegerIps(rwAsciiStream_t *stream); /* * Configure the 'stream' to print IP addresses as integers (0) * instead of as dotted quad (0.0.0.0). This function will override * rwAsciiSetZeroPadIps(). */ extern void rwAsciiSetZeroPadIps(rwAsciiStream_t *stream); /* * Configure the 'stream' to print each octet of an IP with extra * 0's; e.g., print "10.1.2.3" as "010.001.002.003"; useful for * sorting with text-based sortes. rwAsciiSetIntegerIps() will * override this function. */ extern void rwAsciiSetEpochTime(rwAsciiStream_t *stream); /* * Configure the 'stream' to print times as seconds since the UNIX * epoch. Usually times are printed as YYYY/MM/DD:hh:mm:ss. */ extern void rwAsciiSetIntegerSensors(rwAsciiStream_t *stream); /* * Configure the 'stream' to print sensors as integers. Usually the * name of the sensor is printed. */ extern void rwAsciiSetIcmpTypeCode(rwAsciiStream_t *stream); /* * Configure the 'stream' to use a slightly different output for ICMP * records (proto==1); when active and an ICMP record is given * (proto==1), the sPort and dPort columns will hold the ICMP type * and code, respectively. */ extern void rwAsciiGetFieldName( char *buf, size_t buf_len, rwrec_printable_fields_t field_id); /* * Put the first 'buf_len'-1 characters of the name of the * field/column denoted by 'field_id' into the buffer 'buf'. The * caller should ensure that 'buf' is 'buf_len' characters long. * * A 'buf' of at least RWREC_PRINTABLE_MAX_NAME_LEN characters will * be large enough to hold the entire field name. */ extern int rwAsciiFlush(rwAsciiStream_t *stream); /* * Call flush() on the I/O object that 'stream' wraps. */ typedef enum { RW_ASCII_DUPES_KEEP = 0, RW_ASCII_DUPES_REMOVE_SILENT, RW_ASCII_DUPES_REMOVE_WARN, RW_ASCII_DUPES_ERROR } rwascii_dupes_t; void rwAsciiFieldMapPrintUsage(FILE *fh, sk_stringmap_t *field_map); int rwAsciiFieldMapAddDefaultFields(sk_stringmap_t **field_map); int rwAsciiFieldMapParseFields( uint32_t **field_list, uint32_t *field_count, const char *input, const sk_stringmap_t *field_map, rwascii_dupes_t handle_dupes); extern int rwAsciiParseFieldList( rwrec_printable_fields_t **field_ids, uint32_t *field_count, const char *field_string); /* NOT IMPLEMENTED FULLY */ #endif /* _RWPACK_H */ /* ** Local variables: ** mode:c ** indent-tabs-mode:nil ** c-basic-offset:4 ** End: */