/*
 * scamper_file_warts.c
 *
 * the Waikato ARTS file format replacement
 *
 * $Id: scamper_file_warts.c,v 1.104 2007/05/23 05:15:08 mjl Exp $
 *
 * Copyright (C) 2004-2007 The University of Waikato
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, version 2.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */

#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>

#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/ip_icmp.h>
#include <netinet/ip6.h>
#include <netinet/icmp6.h>

#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <stdio.h>

#if defined(__APPLE__)
#include <stdint.h>
#endif

#include <assert.h>

#if defined(DMALLOC)
#include <dmalloc.h>
#endif

#include "scamper_addr.h"
#include "scamper_list.h"
#include "scamper_tlv.h"
#include "scamper_trace.h"
#include "scamper_ping.h"
#include "scamper_file.h"
#include "scamper_file_warts.h"

#include "mjl_splaytree.h"
#include "utils.h"

#define WARTS_MAGIC 0x1205

/*
 * trace attributes: 2 bytes each.
 * the first 4 bits are the type, the second 12 bits are the length
 */
#define WARTS_TRACE_ATTR_HDR(type, len) ((type << 12) | len)
#define WARTS_TRACE_ATTR_HDR_TYPE(hdr)  ((hdr >> 12) & 0xf)
#define WARTS_TRACE_ATTR_HDR_LEN(hdr)    (hdr & 0x0fff)
#define WARTS_TRACE_ATTR_EOF     0x0000
#define WARTS_TRACE_ATTR_PMTUD   0x1

/* how many entries to grow the table by each time */
#define WARTS_ADDR_TABLEGROW  1000
#define WARTS_LIST_TABLEGROW  1
#define WARTS_CYCLE_TABLEGROW 1

/*
 * warts_addr / warts_list / warts_cycle
 *
 * these structures associate a scamper structure with an id number used
 * to represent the structure on disk.
 */
typedef struct warts_addr
{
  scamper_addr_t *addr;
  uint32_t id;
} warts_addr_t;
typedef struct warts_list
{
  scamper_list_t *list;
  uint32_t id;
} warts_list_t;
typedef struct warts_cycle
{
  scamper_cycle_t *cycle;
  uint32_t id;
} warts_cycle_t;

/*
 * warts_hdr
 *
 * this object is written at the start of every object.
 * the magic field is a special integer value that signifies a new warts
 * record.
 * the type field says what type of record follows.
 * the length field reports the length of the following record.
 */
typedef struct warts_hdr
{
  uint16_t magic;
  uint16_t type;
  uint32_t len;
} warts_hdr_t;

/*
 * warts_state
 *
 * warts keeps state of lists, cycles, and addresses declared in a warts
 * file.  each resource is stored either in a tree (for fast searching) or
 * a table (for fast indexing).  when a file is open for writing, the tree
 * is used.  when a file is open for reading, the table is used.  each null
 * entry is used for the first ([0]) entry in the corresponding table.
 */
typedef struct warts_state
{
  /* list state */
  uint32_t          list_count;
  splaytree_t      *list_tree;
  warts_list_t    **list_table;
  warts_list_t      list_null;

  /* cycle state */
  uint32_t          cycle_count;
  splaytree_t      *cycle_tree;
  warts_cycle_t   **cycle_table;
  warts_cycle_t     cycle_null;

  /* address state */
  uint32_t          addr_count;
  splaytree_t      *addr_tree;
  warts_addr_t    **addr_table;
  warts_addr_t      addr_null;

} warts_state_t;

/*
 * warts_var
 *
 * warts often stores optional items of data with each object.  it does
 * this by declaring an array of bits that declare which optional bits of
 * data will be stored.  the warts_var structure is a convenient way of
 * encouraging the code for each object to be consistent.
 *
 * the id field corresponds to a bit
 * the size field records how large the field is stored on disk; -1 is variable
 * the tlv_id field records the id for a scamper_tlv_t if the data item is
 * stored optionally in the data structure itself.
 */
typedef struct warts_var
{
  int     id;
  ssize_t size;
  int     tlv_id;
} warts_var_t;
#define WARTS_VAR_COUNT(array) (sizeof(array)/sizeof(warts_var_t))
#define WARTS_VAR_MFB(array) ((WARTS_VAR_COUNT(array) / 7) + \
			      (WARTS_VAR_COUNT(array) % 7 == 0 ? 0 : 1))

/*
 * warts_param_reader
 *
 */
typedef struct warts_param_reader
{
  void  *data;
  int  (*read)(const uint8_t *, uint16_t *, const uint16_t, void *, void *);
  void  *param;
} warts_param_reader_t;

typedef struct warts_param_writer
{
  const void *data;
  void (*write)(uint8_t *, uint16_t *, const uint16_t, const void *, void *);
  void  *param;
} warts_param_writer_t;

/*
 * the optional bits of a list structure
 */
#define WARTS_LIST_DESCR      1              /* description of list */
#define WARTS_LIST_MONITOR    2              /* canonical name of monitor */
static const warts_var_t list_vars[] =
{
  {WARTS_LIST_DESCR,   -1, -1},
  {WARTS_LIST_MONITOR, -1, -1},
};
static const unsigned int list_vars_mfb = WARTS_VAR_MFB(list_vars);

/*
 * the optional bits of a cycle start structure
 */
#define WARTS_CYCLE_STOP_TIME 1              /* time at which cycle ended */
#define WARTS_CYCLE_HOSTNAME  2              /* hostname at cycle point */
static const warts_var_t cycle_vars[] =
{
  {WARTS_CYCLE_STOP_TIME,  4, -1},
  {WARTS_CYCLE_HOSTNAME,  -1, -1},
};
static const unsigned int cycle_vars_mfb = WARTS_VAR_MFB(cycle_vars);

/*
 * the optional bits of a trace structure
 */
#define WARTS_TRACE_LIST_ID   1              /* list id assigned by warts */
#define WARTS_TRACE_CYCLE_ID  2              /* cycle id assigned by warts */
#define WARTS_TRACE_ADDR_SRC  3              /* source address key */
#define WARTS_TRACE_ADDR_DST  4              /* destination address key */
#define WARTS_TRACE_START     5              /* start timestamp */
#define WARTS_TRACE_STOP_R    6              /* stop reason */
#define WARTS_TRACE_STOP_D    7              /* stop data */
#define WARTS_TRACE_FLAGS     8              /* flags */
#define WARTS_TRACE_ATTEMPTS  9              /* attempts */
#define WARTS_TRACE_HOPLIMIT  10             /* hoplimit */
#define WARTS_TRACE_TYPE      11             /* type */
#define WARTS_TRACE_PROBE_S   12             /* probe size */
#define WARTS_TRACE_PORT_SRC  13             /* source port */
#define WARTS_TRACE_PORT_DST  14             /* destination port */
#define WARTS_TRACE_FIRSTHOP  15             /* first hop */
#define WARTS_TRACE_TOS       16             /* type of service bits */
#define WARTS_TRACE_WAIT      17             /* how long to wait per probe */
#define WARTS_TRACE_LOOPS     18             /* max loops before stopping */
#define WARTS_TRACE_HOPCOUNT  19             /* hop_count */
static const warts_var_t trace_vars[] =
{
  {WARTS_TRACE_LIST_ID,  4, -1},
  {WARTS_TRACE_CYCLE_ID, 4, -1},
  {WARTS_TRACE_ADDR_SRC, 4, -1},
  {WARTS_TRACE_ADDR_DST, 4, -1},
  {WARTS_TRACE_START,    8, -1},
  {WARTS_TRACE_STOP_R,   1, -1},
  {WARTS_TRACE_STOP_D,   1, -1},
  {WARTS_TRACE_FLAGS,    1, -1},
  {WARTS_TRACE_ATTEMPTS, 1, -1},
  {WARTS_TRACE_HOPLIMIT, 1, -1},
  {WARTS_TRACE_TYPE,     1, -1},
  {WARTS_TRACE_PROBE_S,  2, -1},
  {WARTS_TRACE_PORT_SRC, 2, -1},
  {WARTS_TRACE_PORT_DST, 2, -1},
  {WARTS_TRACE_FIRSTHOP, 1, -1},
  {WARTS_TRACE_TOS,      1, -1},
  {WARTS_TRACE_WAIT,     1, -1},
  {WARTS_TRACE_LOOPS,    1, -1},
  {WARTS_TRACE_HOPCOUNT, 2, -1},
};
static const unsigned int trace_vars_mfb = WARTS_VAR_MFB(trace_vars);

/*
 * the optional bits of a trace pmtud structure
 */
#define WARTS_TRACE_PMTUD_IFMTU  1           /* interface mtu */
#define WARTS_TRACE_PMTUD_PMTU   2           /* path mtu */
#define WARTS_TRACE_PMTUD_OUTMTU 3           /* mtu to gateway */
static const warts_var_t pmtud_vars[] =
{
  {WARTS_TRACE_PMTUD_IFMTU,  2, -1},
  {WARTS_TRACE_PMTUD_PMTU,   2, -1},
  {WARTS_TRACE_PMTUD_OUTMTU, 2, SCAMPER_TRACE_PMTUD_TLV_OUTMTU},
};
static const unsigned int pmtud_vars_mfb = WARTS_VAR_MFB(pmtud_vars);

/*
 * the optional bits of a trace hop structure
 */
#define WARTS_TRACE_HOP_ADDR         1       /* address id */
#define WARTS_TRACE_HOP_PROBE_TTL    2       /* probe ttl */
#define WARTS_TRACE_HOP_REPLY_TTL    3       /* reply ttl */
#define WARTS_TRACE_HOP_FLAGS        4       /* flags */
#define WARTS_TRACE_HOP_PROBE_ID     5       /* probe id */
#define WARTS_TRACE_HOP_RTT          6       /* round trip time */
#define WARTS_TRACE_HOP_ICMP_TC      7       /* icmp type / code */
#define WARTS_TRACE_HOP_PROBE_SIZE   8       /* probe size */
#define WARTS_TRACE_HOP_REPLY_SIZE   9       /* reply size */
#define WARTS_TRACE_HOP_REPLY_IPID   10      /* ipid of reply packet */
#define WARTS_TRACE_HOP_REPLY_IPTOS  11      /* tos bits of reply packet */
#define WARTS_TRACE_HOP_NHMTU        12      /* next hop mtu in ptb message */
#define WARTS_TRACE_HOP_INNER_IPLEN  13      /* ip->len from inside icmp */
#define WARTS_TRACE_HOP_INNER_IPTTL  14      /* ip->ttl from inside icmp */
#define WARTS_TRACE_HOP_TCP_FLAGS    15      /* tcp->flags of reply packet */
static const warts_var_t hop_vars[] =
{
  {WARTS_TRACE_HOP_ADDR,        4, -1},
  {WARTS_TRACE_HOP_PROBE_TTL,   1, -1},
  {WARTS_TRACE_HOP_REPLY_TTL,   1, -1},
  {WARTS_TRACE_HOP_FLAGS,       1, -1},
  {WARTS_TRACE_HOP_PROBE_ID,    1, -1},
  {WARTS_TRACE_HOP_RTT,         4, -1},
  {WARTS_TRACE_HOP_ICMP_TC,     2, -1},
  {WARTS_TRACE_HOP_PROBE_SIZE,  2, -1},
  {WARTS_TRACE_HOP_REPLY_SIZE,  2, -1},
  {WARTS_TRACE_HOP_REPLY_IPID,  2, SCAMPER_TRACE_HOP_TLV_REPLY_IPID},
  {WARTS_TRACE_HOP_REPLY_IPTOS, 1, SCAMPER_TRACE_HOP_TLV_REPLY_IPTOS},
  {WARTS_TRACE_HOP_NHMTU,       2, SCAMPER_TRACE_HOP_TLV_NHMTU},
  {WARTS_TRACE_HOP_INNER_IPLEN, 2, SCAMPER_TRACE_HOP_TLV_INNER_IPLEN},
  {WARTS_TRACE_HOP_INNER_IPTTL, 1, SCAMPER_TRACE_HOP_TLV_INNER_IPTTL},
  {WARTS_TRACE_HOP_TCP_FLAGS,   1, -1},
};
static const unsigned int hop_vars_mfb = WARTS_VAR_MFB(hop_vars);

/*
 * the optional bits of a ping structure
 */
#define WARTS_PING_LIST_ID         1
#define WARTS_PING_CYCLE_ID        2
#define WARTS_PING_ADDR_SRC        3
#define WARTS_PING_ADDR_DST        4
#define WARTS_PING_START           5
#define WARTS_PING_STOP_R          6
#define WARTS_PING_STOP_D          7
#define WARTS_PING_PATTERN_LEN     8
#define WARTS_PING_PATTERN_BYTES   9
#define WARTS_PING_PROBE_COUNT    10
#define WARTS_PING_PROBE_SIZE     11
#define WARTS_PING_PROBE_WAIT     12
#define WARTS_PING_PROBE_TTL      13
#define WARTS_PING_REPLY_COUNT    14
#define WARTS_PING_PING_SENT      15
static const warts_var_t ping_vars[] =
{
  {WARTS_PING_LIST_ID,        4, -1},
  {WARTS_PING_CYCLE_ID,       4, -1},
  {WARTS_PING_ADDR_SRC,       4, -1},
  {WARTS_PING_ADDR_DST,       4, -1},
  {WARTS_PING_START,          8, -1},
  {WARTS_PING_STOP_R,         1, -1},
  {WARTS_PING_STOP_D,         1, -1},
  {WARTS_PING_PATTERN_LEN,    2, -1},
  {WARTS_PING_PATTERN_BYTES, -1, -1},
  {WARTS_PING_PROBE_COUNT,    2, -1},
  {WARTS_PING_PROBE_SIZE,     2, -1},
  {WARTS_PING_PROBE_WAIT,     1, -1},
  {WARTS_PING_PROBE_TTL,      1, -1},
  {WARTS_PING_REPLY_COUNT,    2, -1},
  {WARTS_PING_PING_SENT,      2, -1},
};
static const unsigned int ping_vars_mfb = WARTS_VAR_MFB(ping_vars);

#define WARTS_PING_REPLY_ADDR        1
#define WARTS_PING_REPLY_FLAGS       2
#define WARTS_PING_REPLY_REPLY_TTL   3
#define WARTS_PING_REPLY_REPLY_SIZE  4
#define WARTS_PING_REPLY_ICMP_TC     5
#define WARTS_PING_REPLY_RTT         6
#define WARTS_PING_REPLY_PROBE_ID    7
static const warts_var_t ping_reply_vars[] =
{
  {WARTS_PING_REPLY_ADDR,       4, -1},
  {WARTS_PING_REPLY_FLAGS,      1, -1},
  {WARTS_PING_REPLY_REPLY_TTL,  1, -1},
  {WARTS_PING_REPLY_REPLY_SIZE, 2, -1},
  {WARTS_PING_REPLY_ICMP_TC,    2, -1},
  {WARTS_PING_REPLY_RTT,        4, -1},
  {WARTS_PING_REPLY_PROBE_ID,   2, -1},
};
static const unsigned int ping_reply_vars_mfb = WARTS_VAR_MFB(ping_reply_vars);

typedef struct warts_hop_state
{
  scamper_trace_hop_t *hop;
  uint32_t             addr;
  uint8_t              flags[WARTS_VAR_MFB(hop_vars)];
  uint16_t             flags_len;
  uint16_t             params_len;
} warts_hop_state_t;

typedef struct warts_ping_reply_state
{
  scamper_ping_reply_t *reply;
  uint32_t              addr;
  uint8_t               flags[WARTS_VAR_MFB(ping_reply_vars)];
  uint16_t              flags_len;
  uint16_t              params_len;
} warts_ping_reply_state_t;

/*
 * set_flag
 *
 * small routine to set a flag bit.  this exists because the 8th bit of
 * each byte used for flags is used to indicate when another set of flags
 * follows the byte.
 */
static void set_flag(uint8_t *flags, const int id, int *max_id)
{
  int i, j;

  assert(id > 0);

  if(id % 7 == 0)
    {
      i = (id / 7) - 1;
      j = 7;
    }
  else
    {
      i = id / 7;
      j = id % 7;
    }

  flags[i] |= (0x1 << (j-1));

  if(max_id != NULL && *max_id < id) *max_id = id;

  return;
}

/*
 * fold_flags
 *
 * go through and set each link bit in the flag set, as appropriate.
 * conveniently return the count of the number of bytes required to store
 * the flags.
 */
static uint16_t fold_flags(uint8_t *flags, const int max_id)
{
  uint16_t i, j;

  /* if no flags are set, it is still a requirement to include a zero byte */
  if(max_id == 0)
    {
      return 1;
    }

  /* figure out how many bytes have been used */
  j = max_id / 7;
  if((max_id % 7) != 0) j++;

  /*
   * j has to be greater than zero by the above logic.  however, the for
   * loop below will go bananas if it is not
   */
  assert(j > 0);

  /* skip through and set the 'more flags' bit for all flag bytes necessary */
  for(i=0; i<j-1; i++)
    {
      flags[i] |= 0x80;
    }

  return j;
}

static void insert_uint16(uint8_t *buf, uint16_t *off, const uint16_t len,
			  const void *vin, void *param)
{
  uint16_t tmp = htons(*((const uint16_t *)vin));

  assert(len - *off >= 2);

  memcpy(&buf[*off], &tmp, 2);
  *off += 2;
  return;
}

static void insert_uint16_tlv(uint8_t *buf, uint16_t *off, const uint16_t len,
			      const void *vin, void *param)
{
  const scamper_tlv_t *tlv = scamper_tlv_get((const scamper_tlv_t *)vin,
					     *((uint8_t *)param));
  assert(tlv != NULL);
  assert(tlv->tlv_len == 2);
  insert_uint16(buf, off, len, &tlv->tlv_val_16, NULL);
  return;
}

static void insert_uint32(uint8_t *buf, uint16_t *off, const uint16_t len,
			  const void *vin, void *param)
{
  uint32_t tmp = htonl(*((const uint32_t *)vin));

  assert(len - *off >= 4);

  memcpy(&buf[*off], &tmp, 4);
  *off += 4;
  return;
}

static void insert_byte(uint8_t *buf, uint16_t *off, const uint16_t len,
			const void *vin, void *param)
{
  assert(len - *off >= 1);
  buf[(*off)++] = *((const uint8_t *)vin);
  return;
}

static void insert_byte_tlv(uint8_t *buf, uint16_t *off, const uint16_t len,
			    const void *vin, void *param)
{
  const scamper_tlv_t *tlv = scamper_tlv_get((const scamper_tlv_t *)vin,
					     *((uint8_t *)param));
  assert(tlv != NULL);
  assert(tlv->tlv_len == 1);
  insert_byte(buf, off, len, &tlv->tlv_val_8, NULL);
  return;
}

static void insert_bytes_uint16(uint8_t *buf, uint16_t *off,
				const uint16_t len,const void *vin,void *param)
{
  uint16_t *count = (uint16_t *)param;

  assert(len - *off >= *count);

  memcpy(buf + *off, vin, *count);
  *off += *count;
  return;
}

static void insert_string(uint8_t *buf, uint16_t *off, const uint16_t len,
			  const void *vin, void *param)
{
  const char *in = (const char *)vin;
  uint8_t c;
  int i = 0;

  do
    {
      assert(len - *off > 0);
      buf[(*off)++] = c = in[i++];
    }
  while(c != '\0');

  return;
}

/*
 * insert_timeval
 *
 * this function may cause trouble in the future with timeval struct members
 * changing types and so on.
 */
static void insert_timeval(uint8_t *buf, uint16_t *off, const uint16_t len,
			   const void *vin, void *param)
{
  uint32_t t32;

  assert(len - *off >= 8);

  t32 = htonl(((const struct timeval *)vin)->tv_sec);
  memcpy(buf + *off, &t32, 4); *off += 4;
  
  t32 = htonl(((const struct timeval *)vin)->tv_usec);
  memcpy(buf + *off, &t32, 4); *off += 4;

  return;
}

static void insert_rtt(uint8_t *buf, uint16_t *off, const uint16_t len,
		       const void *vin, void *param)
{
  const struct timeval *tv = (const struct timeval *)vin;
  uint32_t t32 = (tv->tv_sec * 1000000) + tv->tv_usec;

  insert_uint32(buf, off, len, &t32, NULL);
  return;
}

static int extract_string(const uint8_t *buf, uint16_t *off,
			  const uint16_t len, void *vout, void *param)
{
  char **out = (char **)vout;
  uint16_t i;

  for(i=*off; i<len; i++)
    {
      /* scan for the null terminator */
      if(buf[i] == '\0')
	{
	  if((*out = memdup(buf+*off, (size_t)(i-*off+1))) == NULL)
	    {
	      return -1;
	    }

	  *off = i+1;
	  return 0;
	}
    }

  return -1;
}

static int extract_uint32(const uint8_t *buf, uint16_t *off,
			  const uint16_t len, void *vout, void *param)
{
  uint32_t *out = (uint32_t *)vout;

  if(len - *off < 4)
    {
      return -1;
    }

  memcpy(out, buf + *off, 4); *off += 4;
  *out = ntohl(*out);
  return 0;
}

static int extract_uint16(const uint8_t *buf, uint16_t *off,
			  const uint16_t len, void *vout, void *param)
{
  uint16_t *out = (uint16_t *)vout;

  if(len - *off < 2)
    {
      return -1;
    }

  memcpy(out, buf + *off, 2); *off += 2;
  *out = ntohs(*out);
  return 0;
}

static int extract_uint16_tlv(const uint8_t *buf, uint16_t *off,
			      const uint16_t len, void *vout, void *param)
{
  scamper_tlv_t **tlvs = (scamper_tlv_t **)vout;
  uint8_t *type = (uint8_t *)param;
  uint16_t t16;

  if(extract_uint16(buf, off, len, &t16, NULL) != 0)
    {
      return -1;
    }

  if(scamper_tlv_set(tlvs, *type, 2, &t16) == NULL)
    {
      return -1;
    }

  return 0;
}

static int extract_byte(const uint8_t *buf, uint16_t *off,
			const uint16_t len, void *vout, void *param)
{
  uint8_t *out = (uint8_t *)vout;

  if(len - *off < 1)
    {
      return -1;
    }

  *out = buf[(*off)++];
  return 0;  
}

static int extract_bytes_uint16(const uint8_t *buf, uint16_t *off,
				const uint16_t len, void *vout, void *param)
{
  uint8_t **out = (uint8_t **)vout;
  uint16_t *req = (uint16_t *)param;

  if(len - *off < *req)
    {
      return -1;
    }

  if(*req == 0)
    {
      *out = NULL;
    }
  else
    {
      if((*out = malloc(*req)) == NULL)
	{
	  return -1;
	}

      memcpy(*out, buf + *off, *req);
      *off += *req;
    }

  return 0;
}

static int extract_byte_tlv(const uint8_t *buf, uint16_t *off,
			    const uint16_t len, void *vout, void *param)
{
  scamper_tlv_t **tlvs = (scamper_tlv_t **)vout;
  uint8_t *type = (uint8_t *)param;
  uint8_t  t8;

  if(extract_byte(buf, off, len, &t8, NULL) != 0)
    {
      return -1;
    }

  if(scamper_tlv_set(tlvs, *type, 1, &t8) == NULL)
    {
      return -1;
    }

  return 0;
}

static int extract_addr(const uint8_t *buf, uint16_t *off,
			const uint16_t len, void *vout, void *param)
{
  warts_state_t *state = (warts_state_t *)param;
  scamper_addr_t **addr = (scamper_addr_t **)vout;
  uint32_t id;

  if(extract_uint32(buf, off, len, &id, NULL) != 0)
    {
      return -1;
    }

  if(id >= state->addr_count)
    {
      return -1;
    }

  *addr = scamper_addr_use(state->addr_table[id]->addr);
  return 0;
}

static int extract_list(const uint8_t *buf, uint16_t *off,
			const uint16_t len, void *vout, void *param)
{
  warts_state_t *state = (warts_state_t *)param;
  scamper_list_t **list = (scamper_list_t **)vout;
  uint32_t id;

  if(extract_uint32(buf, off, len, &id, NULL) != 0)
    {
      return -1;
    }

  if(id >= state->list_count)
    {
      return -1;
    }

  *list = scamper_list_use(state->list_table[id]->list);
  return 0;
}

static int extract_cycle(const uint8_t *buf, uint16_t *off,
			 const uint16_t len, void *vout, void *param)
{
  warts_state_t *state = (warts_state_t *)param;
  scamper_cycle_t **cycle = (scamper_cycle_t **)vout;
  uint32_t id;

  if(extract_uint32(buf, off, len, &id, NULL) != 0)
    {
      return -1;
    }

  if(id >= state->cycle_count)
    {
      return -1;
    }

  *cycle = scamper_cycle_use(state->cycle_table[id]->cycle);
  return 0;
}

static int extract_timeval(const uint8_t *buf, uint16_t *off,
			   const uint16_t len, void *vout, void *param)
{
  struct timeval *tv = (struct timeval *)vout;
  uint32_t t32;

  if(extract_uint32(buf, off, len, &t32, NULL) != 0)
    {
      return -1;
    }
  tv->tv_sec = t32;

  if(extract_uint32(buf, off, len, &t32, NULL) != 0)
    {
      return -1;
    }
  tv->tv_usec = t32;

  return 0;
}

static int extract_rtt(const uint8_t *buf, uint16_t *off, const uint16_t len,
		       void *vout, void *param)
{
  struct timeval *tv = (struct timeval *)vout;
  uint32_t t32;

  if(extract_uint32(buf, off, len, &t32, NULL) != 0)
    {
      return -1;
    }

  tv->tv_sec  = t32 / 1000000;
  tv->tv_usec = t32 % 1000000;
  return 0;
}

static int warts_params_read(const uint8_t *buf, uint16_t *off, uint16_t len,
			     warts_param_reader_t *handlers, int handler_cnt)
{
  warts_param_reader_t *handler;
  const uint8_t *flags = &buf[*off];
  uint16_t flags_len, params_len, final_off;
  int      i, j, id;

  /* if there are no flags set at all, then there's nothing left to do */
  if(flags[0] == 0)
    {
      (*off)++;
      return 0;
    }

  /* figure out how long the flags block is */
  flags_len = 0;
  while((buf[*off] & 0x80) != 0 && *off < len)
    {
      (*off)++; flags_len++;
    }
  flags_len++; (*off)++;
  if(*off > len) goto err;

  /* the length field */
  if(extract_uint16(buf, off, len, &params_len, NULL) != 0)
    {
      goto err;
    }

  /*
   * this calculation is required so we handle the case where we have
   * new parameters that we don't know how to handle (i.e. so we can skip
   * over them)
   */
  final_off = *off + params_len;

  /* read all flag bytes */
  for(i=0; i<flags_len; i++)
    {
      /* if no flags are set in this byte, then skip over it */
      if((flags[i] & 0x7f) == 0)
	{
	  continue;
	}

      /* try each bit in this byte */
      for(j=0; j<7; j++)
	{
	  /* if this flag is unset, then skip the rest of the loop */
	  if((flags[i] & (0x1 << j)) == 0)
	    {
	      continue;
	    }

	  /*
	   * if the id is greater than we have handlers for, then we've
	   * got to the end of what we can parse.
	   */
	  if((id = (i*7)+j) >= handler_cnt)
	    {
	      goto done;
	    }

	  handler = &handlers[id];
	  if(handler->read(buf, off, len, handler->data, handler->param) == -1)
	    {
	      goto err;
	    }
	}
    }

 done:
  *off = final_off;
  return 0;

 err:
  return -1;  
}

static void warts_params_write(uint8_t *buf, uint16_t *off,
			       const uint16_t len,
			       const uint8_t *flags,
			       const uint16_t flags_len,
			       const uint16_t params_len,
			       const warts_param_writer_t *handlers,
			       const int handler_cnt)
{
  int i, j, id;
  uint16_t tmp;

  /* write the flag bytes out */
  tmp = flags_len;
  insert_bytes_uint16(buf, off, len, flags, &tmp);

  /*
   * if there are flags specified, then write the parameter length out.
   * otherwise, there are no parameters to write, so we are done.
   */
  if(flags[0] != 0)
    {
      insert_uint16(buf, off, len, &params_len, NULL);
    }
  else
    {
      assert(params_len == 0);
      return;
    }

  /* handle writing the parameter for each flight out */
  for(i=0; i<flags_len; i++)
    {
      /* skip flag bytes where no flags are set */
      if((flags[i] & 0x7f) == 0)
	{
	  continue;
	}

      /* try each flag bit in the byte */
      for(j=0; j<7; j++)
	{
	  /* skip over unset flags */
	  if((flags[i] & (0x1 << j)) == 0)
	    {
	      continue;
	    }

	  /* this is the parameter id for the flag */
	  id = (i*7)+j;

	  /*
	   * if the id is greater than we have handlers for, then either there
	   * is some code missing, or there is a bug.
	   */
	  assert(id < handler_cnt);

	  /* actually write the data out */
	  handlers[id].write(buf,off,len,handlers[id].data,handlers[id].param);
	}
    }

  return;
}

/*
 * warts_hdr_read
 *
 */
static int warts_hdr_read(const scamper_file_t *sf, warts_hdr_t *hdr)
{
  const uint16_t len = 8;
  int      fd = scamper_file_getfd(sf);
  uint8_t  buf[len];
  size_t   rc;
  int      ret;
  uint16_t off = 0;

  if((ret = read_wrap(fd, buf, &rc, len)) != 0)
    {
      /* have we hit the eof? */
      if(rc == 0 && ret == -2)
	{
	  return 0;
	}

      fprintf(stderr, "warts_hdr_read: read %d of %d bytes\n", (int)rc, len);
      return -1;
    }

  /* these three statements are guaranteed not to fail... */
  extract_uint16(buf, &off, len, &hdr->magic, NULL);
  extract_uint16(buf, &off, len, &hdr->type, NULL);
  extract_uint32(buf, &off, len, &hdr->len, NULL);

  assert(off == len);

  return 1;
}

/*
 * warts_hdr_write
 *
 */
static int warts_hdr_write(const scamper_file_t *sf,
			   const uint16_t type, const uint32_t len)
{
  const uint16_t hdr_magic = WARTS_MAGIC;
  const uint16_t hdr_len = 8;
  int      fd = scamper_file_getfd(sf);
  uint16_t off = 0;
  off_t    pos;
  uint8_t  buf[hdr_len];
  size_t   wc;

  insert_uint16(buf, &off, hdr_len, &hdr_magic, NULL);
  insert_uint16(buf, &off, hdr_len, &type, NULL);
  insert_uint32(buf, &off, hdr_len, &len, NULL);

  assert(off == hdr_len);

  /* treat stdout differently, since we can't seek or truncate it */
  if(fd != 1)
    {
      if((pos = lseek(fd, 0, SEEK_CUR)) == (off_t)-1)
	{
	  return -1;
	}

      if(write_wrap(fd, buf, &wc, hdr_len) != 0)
	{
	  /* truncate if a partial header was written */
	  if(wc != 0) ftruncate(fd, pos);
	  return -1;
	}
    }
  else
    {
      if(write_wrap(fd, buf, &wc, hdr_len) != 0)
	{
	  return -1;
	}
    }

  return 0;
}

static uint8_t *warts_read(const scamper_file_t *sf, const warts_hdr_t *hdr)
{
  uint8_t *buf;
  int      fd;

  if((fd = scamper_file_getfd(sf)) == -1)
    {
      return NULL;
    }

  if((buf = (void *)malloc(hdr->len)) == NULL)
    {
      return NULL;
    }

  if(read_wrap(fd, buf, NULL, hdr->len) != 0)
    {
      free(buf);
      return NULL;
    }

  return buf;
}

/*
 * warts_write
 *
 * this function will write a record to disk, appending a warts_header
 * on the way out to the disk.  if the write fails for whatever reason
 * (as in the disk is full and only a partial recrd can be written), then
 * the write will be retracted in its entirety.
 */
static int warts_write(const scamper_file_t *sf, const uint8_t type,
		       const void *buf, const size_t len)
{
  off_t  off = 0;
  int    fd;

  fd = scamper_file_getfd(sf);
  if(fd != 1 && (off = lseek(fd, 0, SEEK_CUR)) == (off_t)-1)
    {
      return -1;
    }

  if(warts_hdr_write(sf, type, len) == -1)
    {
      return -1;
    }

  if(write_wrap(fd, buf, NULL, len) != 0)
    {
      /*
       * if we could not write the buf out, then truncate the warts file
       * at the hdr we just wrote out above.
       */
      if(fd != 1) ftruncate(fd, off);
      return -1;
    }

  return 0;
}

static int warts_addr_cmp(const void *a, const void *b)
{
  const warts_addr_t *wa = (const warts_addr_t *)a;
  const warts_addr_t *wb = (const warts_addr_t *)b;
  return scamper_addr_cmp(wa->addr, wb->addr);
}

static warts_addr_t *warts_addr_alloc(scamper_addr_t *addr, uint32_t id)
{
  warts_addr_t *wa;
  if((wa = malloc_zero(sizeof(warts_addr_t))) != NULL)
    {
      wa->addr = scamper_addr_use(addr);
      wa->id = id;
    }
  return wa;
}

static void warts_addr_free(warts_addr_t *wa)
{
  if(wa->addr != NULL) scamper_addr_free(wa->addr);
  free(wa);
  return;
}

/*
 * warts_addr_read
 *
 * read an address structure out of the file and record it in the splay
 * tree of addresses.
 *
 * each address record consists of
 *   - an id assigned to the address, modulo 255
 *   - the address family the address belongs to
 *   - the address [length determined by record length]
 */
static scamper_addr_t *warts_addr_read(const scamper_file_t *sf,
				       const warts_hdr_t *hdr)
{
  warts_state_t  *state = scamper_file_getstate(sf);
  scamper_addr_t *addr = NULL;
  warts_addr_t   *wa = NULL, **table;
  uint8_t        *buf = NULL;
  size_t          size;

  /* the data has to be at least 3 bytes long to be valid */
  assert(hdr->len > 2);

  if((state->addr_count % WARTS_ADDR_TABLEGROW) == 0)
    {
      size = sizeof(warts_addr_t*)*(state->addr_count + WARTS_ADDR_TABLEGROW);
      if((table = realloc(state->addr_table, size)) == NULL)
	{
	  goto err;
	}
      state->addr_table = table;
    }

  /* read the address record from the file */
  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  /*
   * sanity check that the warts id recorded in the file matches what we
   * think it should be.
   */
  if(state->addr_count % 255 != buf[0])
    {
      goto err;
    }

  /* allocate a scamper address using the record read from disk */
  if((addr = scamper_addr_alloc(buf[1], buf+2)) == NULL)
    {
      goto err;
    }

  /* finally, allocate the warts address to wrap the scamper address */
  if((wa = warts_addr_alloc(addr, state->addr_count)) == NULL)
    {
      goto err;
    }

  state->addr_table[state->addr_count++] = wa;
  scamper_addr_free(addr);
  free(buf);

  return addr;

 err:
  if(addr != NULL) scamper_addr_free(addr);
  if(wa != NULL) warts_addr_free(wa);
  if(buf != NULL) free(buf);
  return NULL;
}

/*
 * warts_addr_write
 *
 * the address passed in does not have an id associated with it yet, so
 * allocate one and write it to disk.
 */
static int warts_addr_write(const scamper_file_t *sf,
			    scamper_addr_t *addr, uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_addr_t  *wa = NULL;
  uint8_t        tmp[18], *buf = NULL;
  size_t         size;

  /* create the warts addr struct to associate an id with an address */
  if((wa = warts_addr_alloc(addr, state->addr_count)) == NULL)
    {
      goto err;
    }

  /*
   * create the structure to write to disk.
   *
   * where possible, use the tmp buffer on the stack, rather than allocating
   * a buffer with malloc.
   */
  if((size = scamper_addr_size(addr) + 2) <= sizeof(tmp))
    {
      buf = tmp;
    }
  else if((buf = malloc(size)) == NULL)
    {
      goto err;
    }
  buf[0] = wa->id % 255;
  buf[1] = addr->type;
  memcpy(buf+2, addr->addr, scamper_addr_size(addr));

  /* insert the warts_addr into a tree so it can be found quickly */
  if(splaytree_insert(state->addr_tree, wa) == NULL)
    {
      goto err;
    }

  /* write the warts address record to disk */
  if(warts_write(sf, SCAMPER_FILE_OBJ_ADDR, buf, size) == -1)
    {
      goto err;
    }
  if(buf != NULL && buf != tmp) free(buf);

  state->addr_count++;
  *id = wa->id;
  return 0;

 err:
  if(wa != NULL)
    {
      splaytree_remove_item(state->addr_tree, wa);
      warts_addr_free(wa);
    }
  if(buf != NULL && buf != tmp) free(buf);
  return -1;
}

/*
 * warts_addr_getid
 *
 */
static int warts_addr_getid(const scamper_file_t *sf,
			    scamper_addr_t *sa, uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_addr_t   findme, *wa;

  if(sa == NULL)
    {
      *id = 0;
      return 0;
    }

  findme.addr = sa;
  findme.id = 0;
  if((wa = splaytree_find(state->addr_tree, &findme)) != NULL)
    {
      *id = wa->id;
      return 0;
    }

  if(warts_addr_write(sf, sa, id) == 0)
    {
      return 0;
    }

  return -1;
}

static int warts_list_cmp(const void *va, const void *vb)
{
  const warts_list_t *wa = (const warts_list_t *)va;
  const warts_list_t *wb = (const warts_list_t *)vb;
  return scamper_list_cmp(wa->list, wb->list);
}

static warts_list_t *warts_list_alloc(scamper_list_t *list, uint32_t id)
{
  warts_list_t *wl;
  if((wl = malloc_zero(sizeof(warts_list_t))) != NULL)
    {
      wl->list = scamper_list_use(list);
      wl->id = id;
    }
  return wl;
}

static void warts_list_free(warts_list_t *wl)
{
  if(wl->list != NULL) scamper_list_free(wl->list);
  free(wl);
  return;
}

/*
 * warts_list_params
 *
 * put together an outline of the optional bits for a list structure,
 * including the flags structure that sits at the front, and the size (in
 * bytes) of the various parameters that will be optionally included in the
 * file.
 */
static void warts_list_params(const scamper_list_t *list, uint8_t *flags,
			      uint16_t *flags_len, uint16_t *params_len)
{
  int max_id = 0;

  /* unset all the flags */
  memset(flags, 0, list_vars_mfb);
  *params_len = 0;

  if(list->descr != NULL)
    {
      set_flag(flags, WARTS_LIST_DESCR,   &max_id);
      *params_len += strlen(list->descr) + 1;
    }

  if(list->monitor != NULL)
    {
      set_flag(flags, WARTS_LIST_MONITOR, &max_id);
      *params_len += strlen(list->monitor) + 1;
    }

  *flags_len = fold_flags(flags, max_id);

  return;
}

/*
 * warts_list_params_read
 *
 */
static int warts_list_params_read(scamper_list_t *list,
				  uint8_t *buf, uint16_t *off, uint16_t len)
{
  warts_param_reader_t handlers[] = {
    {&list->descr,   extract_string, NULL}, /* WARTS_LIST_DESCR   */
    {&list->monitor, extract_string, NULL}, /* WARTS_LIST_MONITOR */
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static void warts_list_params_write(const scamper_list_t *list,
				    uint8_t *buf, uint16_t *off,
				    const uint16_t len,
				    const uint8_t *flags,
				    const uint16_t flags_len,
				    const uint16_t params_len)
{
  warts_param_writer_t handlers[] = {
    {list->descr,   insert_string, NULL}, /* WARTS_LIST_DESCR */
    {list->monitor, insert_string, NULL}, /* WARTS_LIST_MONITOR */
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  warts_params_write(buf, off, len, flags, flags_len, params_len, handlers,
		     handler_cnt);
  return;
}

/*
 * warts_list_read
 *
 * each list record consists of
 *   - a 4 byte id assigned to the list by warts
 *   - a 4 byte list id assigned by a human
 *   - the name of the list
 *   - optional parameters (e.g. list description, monitor)
 */
static scamper_list_t *warts_list_read(const scamper_file_t *sf,
				       const warts_hdr_t *hdr)
{
  warts_state_t *state = scamper_file_getstate(sf);
  scamper_list_t *list = NULL;
  warts_list_t *wl = NULL, **table;
  uint8_t  *buf = NULL;
  size_t    size;
  uint16_t  i = 0;
  uint32_t  id;

  /*
   * must at least include the warts list id, the human-assigned list-id,
   * a name, and some amount of flags + parameters
   */
  if(hdr->len < 4 + 4 + 2 + 1)
    {
      goto err;
    }

  if((state->list_count % WARTS_LIST_TABLEGROW) == 0)
    {
      size = sizeof(warts_list_t *)*(state->list_count + WARTS_LIST_TABLEGROW);
      if((table = realloc(state->list_table, size)) == NULL)
	{
	  goto err;
	}
      state->list_table = table;
    }

  /* read the list record from the file */
  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  /* preallocate an empty list structure */
  if((list = malloc_zero(sizeof(scamper_list_t))) == NULL)
    {
      goto err;
    }
  list->refcnt = 1;

  /*
   * sanity check that the warts id recorded in the file matches what we
   * think it should be.
   */
  if(extract_uint32(buf, &i, hdr->len, &id, NULL) != 0 ||
     id != state->list_count)
    {
      goto err;
    }

  /* get the list id (assigned by a human) and name */
  if(extract_uint32(buf, &i, hdr->len, &list->id, NULL) != 0 ||
     extract_string(buf, &i, hdr->len, &list->name, NULL) != 0)
    {
      goto err;
    }

  if(warts_list_params_read(list, buf, &i, hdr->len) != 0)
    {
      goto err;
    }

  if((wl = warts_list_alloc(list, state->list_count)) == NULL)
    {
      goto err;
    }

  state->list_table[state->list_count++] = wl;
  scamper_list_free(list);
  free(buf);

  return list;

 err:
  if(list != NULL) scamper_list_free(list);
  if(wl != NULL)   warts_list_free(wl);
  if(buf != NULL)  free(buf);
  return NULL;
}

/*
 * warts_list_write
 *
 * take a list structure and write it to disk.  update the state held, too
 */
static int warts_list_write(const scamper_file_t *sf, scamper_list_t *list,
			    uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_list_t *wl = NULL;
  uint8_t  *buf = NULL;
  uint8_t   flags[list_vars_mfb];
  uint16_t  off = 0, len;
  uint16_t  name_len, flags_len, params_len;

  /* we require a list name */
  if(list->name == NULL)
    {
      goto err;
    }

  /* allocate a warts wrapping structure for the list */
  if((wl = warts_list_alloc(list, state->list_count)) == NULL)
    {
      goto err;
    }

  /* figure out how large the record will be */
  name_len = strlen(list->name) + 1;
  warts_list_params(list, flags, &flags_len, &params_len);
  len = 4 + 4 + name_len + flags_len + params_len;
  if(params_len != 0) len += 2;

  /* allocate the record */
  if((buf = malloc(len)) == NULL)
    {
      goto err;
    }

  /* list id assigned by warts */
  insert_uint32(buf, &off, len, &wl->id, NULL);

  /* list id assigned by a person */
  insert_uint32(buf, &off, len, &list->id, NULL);

  /* list name */
  insert_bytes_uint16(buf, &off, len, list->name, &name_len);

  /* copy in the flags for any parameters */
  warts_list_params_write(list, buf, &off, len, flags, flags_len, params_len);

  assert(off == len);

  if(splaytree_insert(state->list_tree, wl) == NULL)
    {
      goto err;
    }

  /* write the list record to disk */
  if(warts_write(sf, SCAMPER_FILE_OBJ_LIST, buf, len) == -1)
    {
      goto err;
    }

  state->list_count++;
  *id = wl->id;
  free(buf);
  return 0;

 err:
  if(wl != NULL)
    {
      splaytree_remove_item(state->list_tree, wl);
      warts_list_free(wl);
    }
  if(buf != NULL) free(buf);
  return -1;
}

/*
 * warts_list_getid
 *
 * given a scamper_list structure, return the id to use internally to
 * uniquely identify it.  allocate the id if necessary.
 */
static int warts_list_getid(const scamper_file_t *sf, scamper_list_t *list,
			    uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_list_t findme, *wl;

  if(list == NULL)
    {
      *id = 0;
      return 0;
    }

  /* see if there is a tree entry for this list */
  findme.list = list;
  if((wl = splaytree_find(state->list_tree, &findme)) != NULL)
    {
      *id = wl->id;
      return 0;
    }

  /* no tree entry, so write it to a file and return the assigned id */
  if(warts_list_write(sf, list, id) == 0)
    {
      return 0;
    }

  return -1;
}

static int warts_cycle_cmp(const void *va, const void *vb)
{
  const warts_cycle_t *a = (const warts_cycle_t *)va;
  const warts_cycle_t *b = (const warts_cycle_t *)vb;
  return scamper_cycle_cmp(a->cycle, b->cycle);
}

static warts_cycle_t *warts_cycle_alloc(scamper_cycle_t *cycle, uint32_t id)
{
  warts_cycle_t *wc;
  if((wc = malloc_zero(sizeof(warts_cycle_t))) != NULL)
    {
      wc->cycle = scamper_cycle_use(cycle);
      wc->id = id;
    }
  return wc;
}

static void warts_cycle_free(warts_cycle_t *cycle)
{
  if(cycle->cycle != NULL) scamper_cycle_free(cycle->cycle);
  free(cycle);
  return;
}

static void warts_cycle_params(const scamper_cycle_t *cycle, uint8_t *flags,
			       uint16_t *flags_len, uint16_t *params_len)
{
  int max_id;

  /* unset all the flags, reset max_id */
  memset(flags, 0, cycle_vars_mfb);
  max_id = 0;

  *params_len = 0;

  if(cycle->hostname != NULL)
    {
      set_flag(flags, WARTS_CYCLE_HOSTNAME, &max_id);
      *params_len += strlen(cycle->hostname) + 1;
    }

  if(cycle->stop_time != 0)
    {
      set_flag(flags, WARTS_CYCLE_STOP_TIME, &max_id);
      *params_len += 4;
    }

  /* figure out how many bytes the flags will require */
  *flags_len = fold_flags(flags, max_id);

  return;
}

static void warts_cycle_params_write(const scamper_cycle_t *cycle,
				     uint8_t *buf, uint16_t *off,
				     const uint16_t len,
				     const uint8_t *flags,
				     const uint16_t flags_len,
				     const uint16_t params_len)
{
  warts_param_writer_t handlers[] = {
    {&cycle->stop_time, insert_uint32, NULL}, /* WARTS_CYCLE_STOP_TIME */
    {cycle->hostname,   insert_string, NULL}, /* WARTS_CYCLE_HOSTNAME */
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  warts_params_write(buf, off, len, flags, flags_len, params_len, handlers,
		     handler_cnt);
  return;
}

static int warts_cycle_params_read(scamper_cycle_t *cycle,
				   uint8_t *buf, uint16_t *off, uint16_t len)
{
  warts_param_reader_t handlers[] = {
    {&cycle->stop_time, extract_uint32, NULL}, /* WARTS_CYCLE_STOP_TIME */
    {&cycle->hostname,  extract_string, NULL}, /* WARTS_CYCLE_HOSTNAME */
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

/*
 * warts_cycle_read
 *
 * 4 byte cycle id (assigned by warts from counter)
 * 4 byte list id (assigned by warts)
 * 4 byte cycle id (assigned by human)
 * 4 byte time since the epoch, representing start time of the cycle
 * 1 byte flags (followed by optional data items)
 */
static scamper_cycle_t *warts_cycle_read(const scamper_file_t *sf,
					 const warts_hdr_t *hdr)
{
  warts_state_t *state = scamper_file_getstate(sf);
  scamper_cycle_t *cycle = NULL;
  warts_cycle_t *wc = NULL, **table;
  size_t   size;
  uint8_t *buf = NULL;
  uint32_t id;
  uint16_t off = 0;

  /* ensure the cycle_start object is large enough to be valid */
  if(hdr->len < 4 + 4 + 4 + 4 + 1)
    {
      goto err;
    }

  if((state->cycle_count % WARTS_CYCLE_TABLEGROW) == 0)
    {
      size = sizeof(warts_list_t *)*(state->cycle_count+WARTS_CYCLE_TABLEGROW);
      if((table = realloc(state->cycle_table, size)) == NULL)
	{
	  goto err;
	}
      state->cycle_table = table;
    }

  /* read the cycle_start structure out of the file */
  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  /*
   * sanity check that the warts id recorded in the file matches what we
   * think it should be.
   */
  if(extract_uint32(buf, &off, hdr->len, &id, NULL) != 0 ||
     id != state->cycle_count)
    {
      goto err;
    }

  /* the _warts_ list id for the cycle */
  if(extract_uint32(buf, &off, hdr->len, &id, NULL) != 0 ||
     id >= state->list_count)
    {
      goto err;
    }

  if((cycle = scamper_cycle_alloc(state->list_table[id]->list)) == NULL)
    {
      goto err;
    }

  /*
   * the second 4 bytes is the actual cycle id assigned by a human.
   * the third 4 bytes is seconds since the epoch.
   */
  if(extract_uint32(buf, &off, hdr->len, &cycle->id, NULL) != 0 ||
     extract_uint32(buf, &off, hdr->len, &cycle->start_time, NULL) != 0)
    {
      goto err;
    }

  if(warts_cycle_params_read(cycle, buf, &off, hdr->len) != 0)
    {
      goto err;
    }

  if((wc = warts_cycle_alloc(cycle, state->cycle_count)) == NULL)
    {
      goto err;
    }

  state->cycle_table[state->cycle_count++] = wc;
  scamper_cycle_free(cycle);
  free(buf);

  return cycle;

 err:
  if(cycle != NULL)
    {
      if(cycle->list != NULL) scamper_list_free(cycle->list);
      free(cycle);
    }
  if(buf != NULL) free(buf);
  return NULL;
}

/*
 * warts_cycle_write
 *
 * write out a cycle record.  depending on whether the type is a start point,
 * or a cycle definition, some
 *
 * 4 byte cycle id (assigned by warts from counter)
 * 4 byte list id (assigned by warts)
 * 4 byte cycle id (assigned by human)
 * 4 byte time since the epoch, representing start time of the cycle
 * 1 byte flags (followed by optional data items)
 */
static int warts_cycle_write(const scamper_file_t *sf, scamper_cycle_t *cycle,
			     const int type, uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_cycle_t *wc = NULL;
  uint32_t warts_list_id;
  uint8_t *buf = NULL;
  uint8_t  flags[cycle_vars_mfb];
  uint16_t flags_len, params_len;
  uint16_t off = 0, len;

  /* find the list associated w/ the cycle, as we require the warts list id */
  if(warts_list_getid(sf, cycle->list, &warts_list_id) == -1)
    {
      goto err;
    }

  /* allocate warts_cycle wrapping struct to associate a warts-assigned id */
  if((wc = warts_cycle_alloc(cycle, state->cycle_count)) == NULL)
    {
      goto err;
    }

  /* figure out the shape the optional parameters will take */
  warts_cycle_params(cycle, flags, &flags_len, &params_len);

  /* allocate a temporary buf for recording the cycle */
  len = 4 + 4 + 4 + 4 + flags_len + params_len;
  if(params_len != 0) len += 2;
  if((buf = malloc(len)) == NULL)
    {
      goto err;
    }

  /* cycle and list ids, assigned by warts from counters */
  insert_uint32(buf, &off, len, &wc->id, NULL);
  insert_uint32(buf, &off, len, &warts_list_id, NULL);

  /* human cycle id, timestamp */
  insert_uint32(buf, &off, len, &cycle->id, NULL);
  insert_uint32(buf, &off, len, &cycle->start_time, NULL);

  /* copy in the optionally-included parameters */
  warts_cycle_params_write(cycle, buf,&off,len, flags, flags_len, params_len);

  assert(off == len);

  if(splaytree_insert(state->cycle_tree, wc) == NULL)
    {
      goto err;
    }

  if(warts_write(sf, type, buf, len) == -1)
    {
      goto err;
    }

  if(id != NULL) *id = wc->id;
  state->cycle_count++;
  free(buf);

  return 0;

 err:
  if(wc != NULL)
    {
      splaytree_remove_item(state->cycle_tree, wc);
      warts_cycle_free(wc);
    }
  if(buf != NULL) free(buf);
  return -1;
}

/*
 * warts_cycle_stop_read
 *
 * a cycle_stop record consists of the cycle id (assigned by warts from a
 * counter), a timestamp, and some optional parameters.
 *
 * the want parameter specifies if the cycle record is required by the caller.
 */
static scamper_cycle_t *warts_cycle_stop_read(const scamper_file_t *sf,
					      const warts_hdr_t *hdr,
					      const int want)
{
  warts_state_t *state = scamper_file_getstate(sf);
  scamper_cycle_t *cycle;
  uint16_t  off = 0;
  uint32_t  id;
  uint8_t  *buf = NULL;

  if(hdr->len < 4 + 4 + 1)
    {
      goto err;
    }

  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  /*
   * get an index into the stored cycles.
   *
   * if the id does not make sense (is larger than any cycle currently
   * defined, or is the null cycle entry, or there is no current cycle
   * for this id) then we have a problem...
   */
  if(extract_uint32(buf, &off, hdr->len, &id, NULL) != 0 || 
     id >= state->cycle_count || id == 0 || state->cycle_table[id] == NULL)
    {
      goto err;
    }

  /* embed the stop timestamp with the cycle object */
  cycle = state->cycle_table[id]->cycle;
  if(extract_uint32(buf, &off, hdr->len, &cycle->stop_time, NULL) != 0)
    {
      goto err;
    }

  /*
   * don't need the cycle in the array any longer; get a reference to the
   * cycle, since the caller may want the structure.  free the warts_cycle
   * wrapping structure, though
   */
  if(want != 0) scamper_cycle_use(cycle);
  warts_cycle_free(state->cycle_table[id]);
  state->cycle_table[id] = NULL;

  free(buf);

  return cycle;

 err:
  if(buf != NULL) free(buf);
  return NULL;
}

static int warts_cycle_getid(const scamper_file_t *sf, scamper_cycle_t *cycle,
			     uint32_t *id)
{
  warts_state_t *state = scamper_file_getstate(sf);
  warts_cycle_t findme, *wc;

  /* if no cycle is specified, we use the special value zero */
  if(cycle == NULL)
    {
      *id = 0;
      return 0;
    }

  /* see if there is an entry for this cycle */
  findme.cycle = cycle;
  if((wc = splaytree_find(state->cycle_tree, &findme)) != NULL)
    {
      *id = wc->id;
      return 0;
    }

  if(warts_cycle_write(sf, cycle, SCAMPER_FILE_OBJ_CYCLE_DEF, id) == 0)
    {
      return 0;
    }

  return -1;
}

/*
 * warts_cycle_stop_write
 *
 * this function writes a record denoting the end of the cycle pointed to
 * by the cycle parameter.
 * it writes
 *  the 4 byte cycle id assigned by warts
 *  the 4 byte stop time
 *  where applicable, additional parameters
 */
static int warts_cycle_stop_write(const scamper_file_t *sf,
				  scamper_cycle_t *cycle)
{
  uint32_t wc_id;
  uint8_t *buf = NULL;
  uint16_t off = 0, len;
  uint8_t  flag = 0;

  assert(cycle != NULL);

  if(warts_cycle_getid(sf, cycle, &wc_id) != 0)
    {
      goto err;
    }

  len = 4 + 4 + 1;
  if((buf = malloc(len)) == NULL)
    {
      goto err;
    }

  insert_uint32(buf, &off, len, &wc_id, NULL);
  insert_uint32(buf, &off, len, &cycle->stop_time, NULL);
  insert_byte(buf, &off, len, &flag, NULL);

  assert(off == len);

  if(warts_write(sf, SCAMPER_FILE_OBJ_CYCLE_STOP, buf, len) == -1)
    {
      goto err;
    }

  free(buf);
  return 0;

 err:
  if(buf != NULL) free(buf);
  return -1;
}

static void warts_trace_params(const scamper_trace_t *trace, uint8_t *flags,
			       uint16_t *flags_len, uint16_t *params_len)
{
  int max_id = 0;

  /* unset all the flags possible */
  memset(flags, 0, trace_vars_mfb);
  *params_len = 0;

  /* for now, we include the base data items */
  set_flag(flags, WARTS_TRACE_LIST_ID,  &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_CYCLE_ID, &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_ADDR_SRC, &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_ADDR_DST, &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_START,    &max_id); *params_len += 8;
  set_flag(flags, WARTS_TRACE_STOP_R,   &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_STOP_D,   &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_FLAGS,    &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_ATTEMPTS, &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOPLIMIT, &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_TYPE,     &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_PROBE_S,  &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_PORT_SRC, &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_PORT_DST, &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_FIRSTHOP, &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_TOS,      &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_WAIT,     &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_LOOPS,    &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOPCOUNT, &max_id); *params_len += 2;

  *flags_len = fold_flags(flags, max_id);

  return;
}

static int warts_trace_params_read(scamper_trace_t *trace,warts_state_t *state,
				   uint8_t *buf, uint16_t *off, uint16_t len)
{
  warts_param_reader_t handlers[] = {
    {&trace->list,        extract_list,    state}, /* WARTS_TRACE_LIST_ID */
    {&trace->cycle,       extract_cycle,   state}, /* WARTS_TRACE_CYCLE_ID */
    {&trace->src,         extract_addr,    state}, /* WARTS_TRACE_ADDR_SRC */
    {&trace->dst,         extract_addr,    state}, /* WARTS_TRACE_ADDR_DST */
    {&trace->start,       extract_timeval, NULL},  /* WARTS_TRACE_START */
    {&trace->stop_reason, extract_byte,    NULL},  /* WARTS_TRACE_STOP_R */
    {&trace->stop_data,   extract_byte,    NULL},  /* WARTS_TRACE_STOP_D */
    {&trace->flags,       extract_byte,    NULL},  /* WARTS_TRACE_FLAGS */
    {&trace->attempts,    extract_byte,    NULL},  /* WARTS_TRACE_ATTEMPTS */
    {&trace->hoplimit,    extract_byte,    NULL},  /* WARTS_TRACE_HOPLIMIT */
    {&trace->type,        extract_byte,    NULL},  /* WARTS_TRACE_TYPE */
    {&trace->probe_size,  extract_uint16,  NULL},  /* WARTS_TRACE_PROBE_S */
    {&trace->sport,       extract_uint16,  NULL},  /* WARTS_TRACE_PORT_SRC */
    {&trace->dport,       extract_uint16,  NULL},  /* WARTS_TRACE_PORT_DST */
    {&trace->firsthop,    extract_byte,    NULL},  /* WARTS_TRACE_FIRSTHOP */
    {&trace->tos,         extract_byte,    NULL},  /* WARTS_TRACE_TOS */
    {&trace->wait,        extract_byte,    NULL},  /* WARTS_TRACE_WAIT */
    {&trace->loops,       extract_byte,    NULL},  /* WARTS_TRACE_LOOPS */
    {&trace->hop_count,   extract_uint16,  NULL},  /* WARTS_TRACE_HOPCOUNT */
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static int warts_trace_params_write(const scamper_trace_t *trace,
				    const scamper_file_t *sf,
				    uint8_t *buf, uint16_t *off,
				    const uint16_t len,
				    const uint8_t *flags,
				    const uint16_t flags_len,
				    const uint16_t params_len)
{
  uint32_t list_id, cycle_id, src_id, dst_id;
  warts_param_writer_t handlers[] = {
    {&list_id,            insert_uint32,  NULL},
    {&cycle_id,           insert_uint32,  NULL},
    {&src_id,             insert_uint32,  NULL},
    {&dst_id,             insert_uint32,  NULL},
    {&trace->start,       insert_timeval, NULL},
    {&trace->stop_reason, insert_byte,    NULL},
    {&trace->stop_data,   insert_byte,    NULL},
    {&trace->flags,       insert_byte,    NULL},
    {&trace->attempts,    insert_byte,    NULL},
    {&trace->hoplimit,    insert_byte,    NULL},
    {&trace->type,        insert_byte,    NULL},
    {&trace->probe_size,  insert_uint16,  NULL},
    {&trace->sport,       insert_uint16,  NULL},
    {&trace->dport,       insert_uint16,  NULL},
    {&trace->firsthop,    insert_byte,    NULL},
    {&trace->tos,         insert_byte,    NULL},
    {&trace->wait,        insert_byte,    NULL},
    {&trace->loops,       insert_byte,    NULL},
    {&trace->hop_count,   insert_uint16,  NULL},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  if(warts_list_getid(sf,  trace->list,  &list_id)  == -1) return -1;
  if(warts_cycle_getid(sf, trace->cycle, &cycle_id) == -1) return -1;
  if(warts_addr_getid(sf,  trace->src,   &src_id)   == -1) return -1;
  if(warts_addr_getid(sf,  trace->dst,   &dst_id)   == -1) return -1;

  warts_params_write(buf, off, len, flags, flags_len, params_len, handlers,
		     handler_cnt);
  return 0;
}

static void warts_pmtud_params(const scamper_trace_t *trace, uint8_t *flags,
			       uint16_t *flags_len, uint16_t *params_len)
{
  static const int tlv_idx[] = {
    WARTS_TRACE_PMTUD_OUTMTU, /* SCAMPER_TRACE_PMTUD_TLV_OUTMTU */
  };
  scamper_tlv_t *tlv;
  int max_id = 0;

  /* unset all the flags possible */
  memset(flags, 0, pmtud_vars_mfb);
  *params_len = 0;

  /* for now, we include the base data items */
  set_flag(flags, WARTS_TRACE_PMTUD_IFMTU, &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_PMTUD_PMTU, &max_id);  *params_len += 2;

  /* go through the TLVs and decide which flags to set */
  for(tlv = trace->pmtud->tlvs; tlv != NULL; tlv = tlv->tlv_next)
    {
      set_flag(flags, tlv_idx[tlv->tlv_type-1], &max_id);
      *params_len += tlv->tlv_len;
    }

  *flags_len = fold_flags(flags, max_id);

  return;
}

static int warts_pmtud_read(scamper_trace_t *trace,
			    const uint8_t *buf, uint16_t *off, uint16_t len)
{
  uint8_t outmtu = SCAMPER_TRACE_PMTUD_TLV_OUTMTU;
  warts_param_reader_t handlers[] = {
    {&trace->pmtud->ifmtu, extract_uint16,     NULL},
    {&trace->pmtud->pmtu,  extract_uint16,     NULL},
    {&trace->pmtud->tlvs,  extract_uint16_tlv, &outmtu},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static void warts_pmtud_write(const scamper_trace_t *trace, uint8_t *buf,
			      uint16_t *off, uint16_t len, uint8_t *flags,
			      uint16_t flags_len, uint16_t params_len)
{
  uint16_t outmtu;
  warts_param_writer_t handlers[] = {
    {&trace->pmtud->ifmtu, insert_uint16, NULL},
    {&trace->pmtud->pmtu,  insert_uint16, NULL},
    {&outmtu,              insert_uint16, NULL},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  SCAMPER_TRACE_PMTUD_GET_OUTMTU(trace->pmtud, outmtu);

  warts_params_write(buf, off, len, flags, flags_len, params_len, handlers,
		     handler_cnt);
  return;
}

static int warts_hop_read_icmp_tc(const uint8_t *buf, uint16_t *off,
				  uint16_t len, void *vout, void *param)
{
  scamper_trace_hop_t *hop = (scamper_trace_hop_t *)vout;

  if(len - *off < 2)
    {
      return -1;
    }

  hop->hop_icmp_type = buf[(*off)++];
  hop->hop_icmp_code = buf[(*off)++];

  return 0;
}

static void warts_hop_write_icmp_tc(uint8_t *buf, uint16_t *off,
				    const uint16_t len, const void *vin,
				    void *param)
{
  const scamper_trace_hop_t *hop = (const scamper_trace_hop_t *)vin;

  assert(len - *off >= 2);

  buf[(*off)++] = hop->hop_icmp_type;
  buf[(*off)++] = hop->hop_icmp_code;

  return;
}

static int warts_hop_read_probe_id(const uint8_t *buf, uint16_t *off,
				   uint16_t len, void *vout, void *param)
{
  uint8_t *out = (uint8_t *)vout;

  if(len - *off < 1)
    {
      return -1;
    }

  *out = buf[(*off)++] + 1;

  return 0;
}

static void warts_hop_write_probe_id(uint8_t *buf, uint16_t *off,
				     const uint16_t len, const void *vin,
				     void *param)
{
  assert(len - *off >= 1);
  buf[(*off)++] = *((const uint8_t *)vin) - 1;
  return;
}

static void warts_hop_params(const scamper_trace_hop_t *hop, uint8_t *flags,
			     uint16_t *flags_len, uint16_t *params_len)
{
  static const int tlv_idx[] = {
    WARTS_TRACE_HOP_REPLY_IPID,  /* SCAMPER_TRACE_HOP_TLV_REPLY_IPID */
    WARTS_TRACE_HOP_REPLY_IPTOS, /* SCAMPER_TRACE_HOP_TLV_REPLY_IPTOS */
    WARTS_TRACE_HOP_NHMTU,       /* SCAMPER_TRACE_HOP_TLV_NHMTU */
    WARTS_TRACE_HOP_INNER_IPLEN, /* SCAMPER_TRACE_HOP_TLV_INNER_IPLEN */
    WARTS_TRACE_HOP_INNER_IPTTL, /* SCAMPER_TRACE_HOP_TLV_INNER_IPTTL */
  };
  scamper_tlv_t *tlv;
  int max_id = 0;

  /* unset all the flags possible */
  memset(flags, 0, hop_vars_mfb);
  *params_len = 0;

  /* for now, we include all the base data items */
  set_flag(flags, WARTS_TRACE_HOP_ADDR,       &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_HOP_PROBE_TTL,  &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOP_REPLY_TTL,  &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOP_FLAGS,      &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOP_PROBE_ID,   &max_id); *params_len += 1;
  set_flag(flags, WARTS_TRACE_HOP_RTT,        &max_id); *params_len += 4;
  set_flag(flags, WARTS_TRACE_HOP_PROBE_SIZE, &max_id); *params_len += 2;
  set_flag(flags, WARTS_TRACE_HOP_REPLY_SIZE, &max_id); *params_len += 2;

  if((hop->hop_flags & SCAMPER_TRACE_HOP_FLAG_TCP) == 0)
    {
      set_flag(flags, WARTS_TRACE_HOP_ICMP_TC,    &max_id); *params_len += 2;
    }
  else
    {
      set_flag(flags, WARTS_TRACE_HOP_TCP_FLAGS,  &max_id); *params_len += 1;
    }

  /* go through the TLVs and decide which flags to set */
  for(tlv = hop->hop_tlvs; tlv != NULL; tlv = tlv->tlv_next)
    {
      assert(tlv->tlv_type-1 < (int)(sizeof(tlv_idx)/sizeof(int)));
      set_flag(flags, tlv_idx[tlv->tlv_type-1], &max_id);
      *params_len += tlv->tlv_len;
    }

  *flags_len = fold_flags(flags, max_id);

  return;
}

static int warts_hop_read(scamper_trace_hop_t *hop, warts_state_t *state,
			  const uint8_t *buf, uint16_t *off, uint16_t len)
{
  uint8_t types[] = {
    SCAMPER_TRACE_HOP_TLV_REPLY_IPID,
    SCAMPER_TRACE_HOP_TLV_REPLY_IPTOS,
    SCAMPER_TRACE_HOP_TLV_NHMTU,
    SCAMPER_TRACE_HOP_TLV_INNER_IPLEN,
    SCAMPER_TRACE_HOP_TLV_INNER_IPTTL,
  };

  warts_param_reader_t handlers[] = {
    {&hop->hop_addr,       extract_addr,            state},
    {&hop->hop_probe_ttl,  extract_byte,            NULL},
    {&hop->hop_reply_ttl,  extract_byte,            NULL},
    {&hop->hop_flags,      extract_byte,            NULL},
    {&hop->hop_probe_id,   warts_hop_read_probe_id, NULL},
    {&hop->hop_rtt,        extract_rtt,             NULL},
    {hop,                  warts_hop_read_icmp_tc,  NULL},
    {&hop->hop_probe_size, extract_uint16,          NULL},
    {&hop->hop_reply_size, extract_uint16,          NULL},
    {&hop->hop_tlvs,       extract_uint16_tlv,      &types[0]},
    {&hop->hop_tlvs,       extract_byte_tlv,        &types[1]},
    {&hop->hop_tlvs,       extract_uint16_tlv,      &types[2]},
    {&hop->hop_tlvs,       extract_uint16_tlv,      &types[3]},
    {&hop->hop_tlvs,       extract_byte_tlv,        &types[4]},
    {&hop->hop_tcp_flags,  extract_byte,            NULL},
  };

  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static void warts_hop_write(const warts_hop_state_t *state,
			    uint8_t *buf, uint16_t *off, uint16_t len)
{
  scamper_trace_hop_t *hop = state->hop;

  uint8_t types[] = {
    SCAMPER_TRACE_HOP_TLV_REPLY_IPID,
    SCAMPER_TRACE_HOP_TLV_REPLY_IPTOS,
    SCAMPER_TRACE_HOP_TLV_NHMTU,
    SCAMPER_TRACE_HOP_TLV_INNER_IPLEN,
    SCAMPER_TRACE_HOP_TLV_INNER_IPTTL,
  };

  warts_param_writer_t handlers[] = {
    {&state->addr,         insert_uint32,            NULL},
    {&hop->hop_probe_ttl,  insert_byte,              NULL},
    {&hop->hop_reply_ttl,  insert_byte,              NULL},
    {&hop->hop_flags,      insert_byte,              NULL},
    {&hop->hop_probe_id,   warts_hop_write_probe_id, NULL},
    {&hop->hop_rtt,        insert_rtt,               NULL},
    {hop,                  warts_hop_write_icmp_tc,  NULL},
    {&hop->hop_probe_size, insert_uint16,            NULL},
    {&hop->hop_reply_size, insert_uint16,            NULL},
    {hop->hop_tlvs,        insert_uint16_tlv,        &types[0]},
    {hop->hop_tlvs,        insert_byte_tlv,          &types[1]},
    {hop->hop_tlvs,        insert_uint16_tlv,        &types[2]},
    {hop->hop_tlvs,        insert_uint16_tlv,        &types[3]},
    {hop->hop_tlvs,        insert_byte_tlv,          &types[4]},
    {&hop->hop_tcp_flags,  insert_byte,              NULL},
  };

  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  warts_params_write(buf, off, len, state->flags, state->flags_len,
		     state->params_len, handlers, handler_cnt);
  return;
}

static int warts_hops_read(scamper_trace_hop_t **hops, warts_state_t *state,
			   const uint8_t *buf, uint16_t *off, uint16_t len,
			   uint16_t count)
{
  scamper_trace_hop_t *head = NULL, *hop = NULL;
  int i;

  for(i=0; i<count; i++)
    {
      /*
       * the hop list is stored in a linked list; add each new hop to the
       * end of the list
       */
      if(hop != NULL)
	{
	  hop->hop_next = scamper_trace_hop_alloc();
	  hop = hop->hop_next;
	}
      else
	{
	  head = hop = scamper_trace_hop_alloc();
	}

      /* could not allocate an empty hop structure ... */
      if(hop == NULL)
	{
	  goto err;
	}

      if(warts_hop_read(hop, state, buf, off, len) != 0)
	{
	  goto err;
	}
    }

  *hops = head;

  return 0;

 err:
  while(head != NULL)
    {
      hop = head;
      head = head->hop_next;
      scamper_trace_hop_free(hop);
    }
  return -1;
}

/*
 * warts_trace_read
 *
 */
static scamper_trace_t *warts_trace_read(const scamper_file_t *sf,
					 const warts_hdr_t *hdr)
{
  warts_state_t       *state = scamper_file_getstate(sf);
  scamper_trace_t     *trace = NULL;
  uint8_t             *buf = NULL;
  uint16_t             off=0, i;
  scamper_trace_hop_t *hops = NULL;
  scamper_trace_hop_t *hop;
  uint16_t             count;
  int                  max_ttl;
  uint8_t              type;
  uint16_t             len;

  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  if((trace = scamper_trace_alloc()) == NULL)
    {
      goto err;
    }

  /* read the trace's parameters */
  if(warts_trace_params_read(trace, state, buf, &off, hdr->len) != 0)
    {
      goto err;
    }

  /*
   * the next two bytes tell us how many scamper_hops to read out of trace
   * if we did not get any responses, we are done.
   */
  if(extract_uint16(buf, &off, hdr->len, &count, NULL) != 0)
    {
      goto err;
    }

  /* read all the hop records */
  if(warts_hops_read(&hops, state, buf, &off, hdr->len, count) != 0)
    {
      goto err;
    }

  /* work out the maximum ttl probed with that got a response */
  max_ttl = 0;
  for(i=0, hop = hops; i < count; i++)
    {
      if(hop->hop_probe_ttl > max_ttl)
	{
	  max_ttl = hop->hop_probe_ttl;
	}
      hop = hop->hop_next;
    }

  /*
   * if the hop_count field was provided in the file, then
   * make sure it makes sense based on the hop data we've just scanned
   */
  if(trace->hop_count != 0)
    {
      if(trace->hop_count < max_ttl)
	{
	  goto err;
	}
    }
  else
    {
      trace->hop_count = max_ttl;
    }

  /* allocate enough hops to string the trace together */
  if(scamper_trace_hops_alloc(trace, trace->hop_count) == -1)
    {
      goto err;
    }

  if(hops == NULL)
    {
      assert(count == 0);
      goto done;
    }

  /*
   * now loop through the hops array stored in this procedure
   * and assemble the responses into trace->hops.
   */
  trace->hops[hops->hop_probe_ttl-1] = hop = hops;
  while(hop->hop_next != NULL) 
    {
      if(hop->hop_probe_ttl != hop->hop_next->hop_probe_ttl)
	{
	  i = hop->hop_next->hop_probe_ttl-1;
	  trace->hops[i] = hop->hop_next;
	  hop->hop_next = NULL;
	  hop = trace->hops[i];
	}
      else hop = hop->hop_next;
    }
  hops = NULL;

  for(;;)
    {
      if(extract_uint16(buf, &off, hdr->len, &i, NULL) != 0)
	{
	  goto err;
	}
      if(i == WARTS_TRACE_ATTR_EOF)
	{
	  break;
	}

      type = WARTS_TRACE_ATTR_HDR_TYPE(i);
      len  = WARTS_TRACE_ATTR_HDR_LEN(i);

      /* XXX: this code should all be in warts_pmtud_read */
      if(type == WARTS_TRACE_ATTR_PMTUD)
	{
	  if((trace->pmtud=malloc_zero(sizeof(scamper_trace_pmtud_t))) == NULL)
	    {
	      goto err;
	    }

	  if(warts_pmtud_read(trace, buf, &off, hdr->len) != 0)
	    {
	      goto err;
	    }

	  if(extract_uint16(buf, &off, hdr->len, &count, NULL) != 0)
	    {
	      goto err;
	    }
	  if(count != 0)
	    {
	      if(warts_hops_read(&hops,state, buf, &off, hdr->len, count) != 0)
		{
		  goto err;
		}

	      trace->pmtud->hops = hops;
	    }
	}

      off += len;
    }

  assert(off == hdr->len);

 done:
  free(buf);
  return trace;

 err:
  if(hops != NULL) free(hops);
  if(buf != NULL) free(buf);
  if(trace != NULL) scamper_trace_free(trace);
  return NULL;
}

static int warts_hop_state(const scamper_file_t *sf, scamper_trace_hop_t *hop,
			   warts_hop_state_t *state, uint16_t *len)
{
  /* for each hop, remember the address id */
  if(warts_addr_getid(sf, hop->hop_addr, &state->addr) == -1)
    {
      return -1;
    }

  /* for each hop, figure out how much space it will take up */
  warts_hop_params(hop, state->flags, &state->flags_len, &state->params_len);

  /* store the actual hop record with the state structure too */
  state->hop = hop;

  /* increase length required for the trace record */
  *len += state->flags_len + 2 + state->params_len;

  return 0;
}

static int warts_trace_write(const scamper_file_t *sf,
			     const scamper_trace_t *trace)
{
  scamper_trace_hop_t *hop;
  uint8_t             *buf = NULL;
  uint8_t              trace_flags[trace_vars_mfb];
  uint16_t             trace_flags_len, trace_params_len;
  warts_hop_state_t   *hop_state = NULL;
  uint16_t             hop_recs;
  uint8_t              pmtud_flags[pmtud_vars_mfb];
  uint16_t             pmtud_flags_len = 0, pmtud_params_len = 0;
  warts_hop_state_t   *pmtud_state = NULL;
  uint16_t             pmtud_recs = 0;
  uint16_t             pmtud_len = 0;
  uint16_t             junk16;
  uint16_t             off = 0, len;
  size_t               size;
  int                  i, j;

  assert(trace != NULL);

  /* figure out which trace data items we'll store in this record */
  warts_trace_params(trace, trace_flags, &trace_flags_len, &trace_params_len);

  /*
   * this represents the length of the trace's flags and parameters, and the
   * 2-byte field that records the number of hop records that follow
   */
  len = trace_flags_len + 2 + trace_params_len + 2;

  /* for each hop, figure out what is going to be stored in this record */
  if((hop_recs = scamper_trace_hop_count(trace)) > 0)
    {
      size = hop_recs * sizeof(warts_hop_state_t);
      if((hop_state = (warts_hop_state_t *)malloc(size)) == NULL)
	{
	  goto err;
	}

      for(i=0, j=0; i<trace->hop_count; i++)
	{
	  for(hop = trace->hops[i]; hop != NULL; hop = hop->hop_next)
	    {
	      /* record basic hop state */
	      if(warts_hop_state(sf, hop, &hop_state[j++], &len) == -1)
		{
		  goto err;
		}
	    }
	}
    }

  /* figure out how much space we need for PMTUD data, if we have it */
  if(trace->pmtud != NULL)
    {
      /* figure out what the structure of the pmtud header looks like */
      warts_pmtud_params(trace,pmtud_flags,&pmtud_flags_len,&pmtud_params_len);

      /* count the number of hop records */
      pmtud_recs = scamper_trace_pmtud_hop_count(trace);

      /* allocate an array of address indexes for the pmtud hop addresses */
      size = pmtud_recs * sizeof(warts_hop_state_t);
      if((pmtud_state = (warts_hop_state_t *)malloc(size)) == NULL)
	{
	  goto err;
	}

      /* flags + params + number of hop records for pmtud structure */
      pmtud_len = pmtud_flags_len + 2 + pmtud_params_len + 2;

      /* record hop state for each pmtud hop */
      for(hop = trace->pmtud->hops, j=0; hop != NULL; hop = hop->hop_next)
	{
	  if(warts_hop_state(sf, hop, &pmtud_state[j++], &pmtud_len) == -1)
	    {
	      goto err;
	    }
	}

      len += (2 + pmtud_len); /* 2 = size of attribute header */
    }

  len += 2; /* EOF */

  if((buf = malloc(len)) == NULL)
    {
      goto err;
    }

  /* write trace parameters */
  if(warts_trace_params_write(trace, sf, buf, &off, len, trace_flags,
			      trace_flags_len, trace_params_len) == -1)
    {
      goto err;
    }

  /* hop record count */
  insert_uint16(buf, &off, len, &hop_recs, NULL);

  /* write each traceroute hop record */
  for(i=0; i<hop_recs; i++)
    {
      warts_hop_write(&hop_state[i], buf, &off, len);
    }
  if(hop_state != NULL)
    {
      free(hop_state);
      hop_state = NULL;
    }

  /* write the PMTUD data */
  if(trace->pmtud != NULL)
    {
      junk16 = WARTS_TRACE_ATTR_HDR(WARTS_TRACE_ATTR_PMTUD, pmtud_len);
      insert_uint16(buf, &off, len, &junk16, NULL);
		    
      /* write details of the pmtud measurement */
      warts_pmtud_write(trace, buf, &off, len,
			pmtud_flags, pmtud_flags_len, pmtud_params_len);

      /* write the number of hop records */
      insert_uint16(buf, &off, len, &pmtud_recs, NULL);

      for(i=0; i<pmtud_recs; i++)
	{
	  warts_hop_write(&pmtud_state[i], buf, &off, len);
	}
      if(pmtud_state != NULL)
	{
	  free(pmtud_state);
	  pmtud_state = NULL;
	}
    }

  /* write the end of trace attributes header */
  junk16 = WARTS_TRACE_ATTR_EOF;
  insert_uint16(buf, &off, len, &junk16, NULL);

  assert(off == len);

  if(warts_write(sf, SCAMPER_FILE_OBJ_TRACE, buf, len) == -1)
    {
      goto err;
    }

  free(buf);
  return 0;

 err:
  if(buf != NULL) free(buf);
  if(hop_state != NULL) free(hop_state);
  if(pmtud_state != NULL) free(pmtud_state);
  return -1;
}

static void warts_ping_reply_params(const scamper_ping_reply_t *reply,
				    uint8_t *flags, uint16_t *flags_len,
				    uint16_t *params_len)
{
  int max_id = 0;

  /* unset all the flags possible */
  memset(flags, 0, ping_reply_vars_mfb);
  *params_len = 0;

  set_flag(flags, WARTS_PING_REPLY_ADDR,       &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_REPLY_FLAGS,      &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_REPLY_REPLY_TTL,  &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_REPLY_REPLY_SIZE, &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_REPLY_ICMP_TC,    &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_REPLY_RTT,        &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_REPLY_PROBE_ID,   &max_id); *params_len += 2;

  *flags_len = fold_flags(flags, max_id);

  return;
}

static int warts_ping_reply_state(const scamper_file_t *sf,
				  scamper_ping_reply_t *reply,
				  warts_ping_reply_state_t *state,
				  uint16_t *len)
{
  if(warts_addr_getid(sf, reply->addr, &state->addr) == -1)
    {
      return -1;
    }

  warts_ping_reply_params(reply, state->flags,
			  &state->flags_len,&state->params_len);

  state->reply = reply;

  *len += state->flags_len + 2 + state->params_len;

  return 0;
}

static int warts_ping_reply_read_icmptc(const uint8_t *buf, uint16_t *off,
					uint16_t len, void *vout, void *param)
{
  scamper_ping_reply_t *reply = (scamper_ping_reply_t *)vout;

  if(len - *off < 2)
    {
      return -1;
    }

  reply->icmp_type = buf[(*off)++];
  reply->icmp_code = buf[(*off)++];

  return 0;
}

static void warts_ping_reply_write_icmptc(uint8_t *buf, uint16_t *off,
					  const uint16_t len, const void *vin,
					  void *param)
{
  assert(len - *off >= 2);

  buf[(*off)++] = ((const scamper_ping_reply_t *)vin)->icmp_type;
  buf[(*off)++] = ((const scamper_ping_reply_t *)vin)->icmp_code;

  return;
}

static int warts_ping_reply_read(scamper_ping_reply_t *reply,
				 warts_state_t *state, const uint8_t *buf,
				 uint16_t *off, uint16_t len)
{
  warts_param_reader_t handlers[] = {
    {&reply->addr,       extract_addr,                 state},
    {&reply->flags,      extract_byte,                 NULL},
    {&reply->reply_ttl,  extract_byte,                 NULL},
    {&reply->reply_size, extract_uint16,               NULL},
    {reply,              warts_ping_reply_read_icmptc, NULL},
    {&reply->rtt,        extract_rtt,                  NULL},
    {&reply->probe_id,   extract_uint16,               NULL},
  };

  const int handler_cnt = sizeof(handlers) / sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static void warts_ping_reply_write(const warts_ping_reply_state_t *state,
				   uint8_t *buf, uint16_t *off, uint16_t len)
{
  scamper_ping_reply_t *reply = state->reply;

  warts_param_writer_t handlers[] = {
    {&state->addr,       insert_uint32,                 NULL},
    {&reply->flags,      insert_byte,                   NULL},
    {&reply->reply_ttl,  insert_byte,                   NULL},
    {&reply->reply_size, insert_uint16,                 NULL},
    {reply,              warts_ping_reply_write_icmptc, NULL},
    {&reply->rtt,        insert_rtt,                    NULL},
    {&reply->probe_id,   insert_uint16,                 NULL},
  };

  const int handler_cnt = sizeof(handlers) / sizeof(warts_param_writer_t);

  warts_params_write(buf, off, len, state->flags, state->flags_len,
		     state->params_len, handlers, handler_cnt);
  return;
}

static void warts_ping_params(const scamper_ping_t *ping, uint8_t *flags,
			      uint16_t *flags_len, uint16_t *params_len)
{
  uint16_t pad_len = ping->pattern_len;
  int max_id = 0;

  /* unset all the flags possible */
  memset(flags, 0, ping_vars_mfb);
  *params_len = 0;

  set_flag(flags, WARTS_PING_LIST_ID,       &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_CYCLE_ID,      &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_ADDR_SRC,      &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_ADDR_DST,      &max_id); *params_len += 4;
  set_flag(flags, WARTS_PING_START,         &max_id); *params_len += 8;
  set_flag(flags, WARTS_PING_STOP_R,        &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_STOP_D,        &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_PATTERN_LEN,   &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_PATTERN_BYTES, &max_id); *params_len += pad_len;
  set_flag(flags, WARTS_PING_PROBE_COUNT,   &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_PROBE_SIZE,    &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_PROBE_WAIT,    &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_PROBE_TTL,     &max_id); *params_len += 1;
  set_flag(flags, WARTS_PING_REPLY_COUNT,   &max_id); *params_len += 2;
  set_flag(flags, WARTS_PING_PING_SENT,     &max_id); *params_len += 2;

  *flags_len = fold_flags(flags, max_id);

  return;
}

static int warts_ping_params_read(scamper_ping_t *ping, warts_state_t *state,
				  uint8_t *buf, uint16_t *off, uint16_t len)
{
  warts_param_reader_t handlers[] = {
    {&ping->list,          extract_list,         state},
    {&ping->cycle,         extract_cycle,        state},
    {&ping->src,           extract_addr,         state},
    {&ping->dst,           extract_addr,         state},
    {&ping->start,         extract_timeval,      NULL},
    {&ping->stop_reason,   extract_byte,         NULL},
    {&ping->stop_data,     extract_byte,         NULL},
    {&ping->pattern_len,   extract_uint16,       NULL},
    {&ping->pattern_bytes, extract_bytes_uint16, &ping->pattern_len},
    {&ping->probe_count,   extract_uint16,       NULL},
    {&ping->probe_size,    extract_uint16,       NULL},
    {&ping->probe_wait,    extract_byte,         NULL},
    {&ping->probe_ttl,     extract_byte,         NULL},
    {&ping->reply_count,   extract_uint16,       NULL},
    {&ping->ping_sent,     extract_uint16,       NULL},
  };
  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_reader_t);

  return warts_params_read(buf, off, len, handlers, handler_cnt);
}

static int warts_ping_params_write(const scamper_ping_t *ping,
				   const scamper_file_t *sf,
				   uint8_t *buf, uint16_t *off,
				   const uint16_t len,
				   const uint8_t *flags,
				   const uint16_t flags_len,
				   const uint16_t params_len)
{
  uint32_t list_id, cycle_id, src_id, dst_id;
  uint16_t pad_len = ping->pattern_len;
  warts_param_writer_t handlers[] = {
    {&list_id,             insert_uint32,       NULL},
    {&cycle_id,            insert_uint32,       NULL},
    {&src_id,              insert_uint32,       NULL},
    {&dst_id,              insert_uint32,       NULL},
    {&ping->start,         insert_timeval,      NULL},
    {&ping->stop_reason,   insert_byte,         NULL},
    {&ping->stop_data,     insert_byte,         NULL},
    {&ping->pattern_len,   insert_uint16,       NULL},
    {&ping->pattern_bytes, insert_bytes_uint16, &pad_len},
    {&ping->probe_count,   insert_uint16,       NULL},
    {&ping->probe_size,    insert_uint16,       NULL},
    {&ping->probe_wait,    insert_byte,         NULL},
    {&ping->probe_ttl,     insert_byte,         NULL},
    {&ping->reply_count,   insert_uint16,       NULL},
    {&ping->ping_sent,     insert_uint16,       NULL},
  };

  const int handler_cnt = sizeof(handlers)/sizeof(warts_param_writer_t);

  if(warts_list_getid(sf,  ping->list,  &list_id)  == -1) return -1;
  if(warts_cycle_getid(sf, ping->cycle, &cycle_id) == -1) return -1;
  if(warts_addr_getid(sf,  ping->src,   &src_id)   == -1) return -1;
  if(warts_addr_getid(sf,  ping->dst,   &dst_id)   == -1) return -1;

  warts_params_write(buf, off, len, flags, flags_len, params_len, handlers,
		     handler_cnt);
  return 0;
}

static scamper_ping_t *warts_ping_read(const scamper_file_t *sf,
				       const warts_hdr_t *hdr)
{
  warts_state_t *state = scamper_file_getstate(sf);
  scamper_ping_t *ping = NULL;
  uint8_t *buf = NULL;
  uint16_t off = 0;
  uint16_t i;
  scamper_ping_reply_t *reply;
  uint16_t reply_count;

  if((buf = warts_read(sf, hdr)) == NULL)
    {
      goto err;
    }

  if((ping = scamper_ping_alloc()) == NULL)
    {
      goto err;
    }

  if(warts_ping_params_read(ping, state, buf, &off, hdr->len) != 0)
    {
      goto err;
    }

  /*
   * determine how many replies to read.
   * if there are no replies, then we are done.
   */
  if(extract_uint16(buf, &off, hdr->len, &reply_count, NULL) != 0)
    {
      goto err;
    }
  if(reply_count == 0) goto done;

  /* allocate the ping_replies array */
  if(scamper_ping_replies_alloc(ping, ping->ping_sent) != 0)
    {
      goto err;
    }

  /* for each reply, read it and insert it into the ping structure */
  for(i=0; i<reply_count; i++)
    {
      if((reply = scamper_ping_reply_alloc()) == NULL)
	{
	  goto err;
	}

      if(warts_ping_reply_read(reply, state, buf, &off, hdr->len) != 0)
	{
	  goto err;
	}

      if(scamper_ping_reply_append(ping, reply) != 0)
	{
	  goto err;
	}
    }

  assert(off == hdr->len);

 done:
  free(buf);
  return ping;

 err:
  if(buf != NULL) free(buf);
  if(ping != NULL) scamper_ping_free(ping);
  return NULL;
}

static int warts_ping_write(const scamper_file_t *sf,
			    const scamper_ping_t *ping)
{
  warts_ping_reply_state_t *reply_state = NULL;
  scamper_ping_reply_t *reply;
  uint8_t *buf = NULL;
  uint8_t  flags[ping_vars_mfb];
  uint16_t len, flags_len, params_len, off = 0;
  uint16_t reply_count;
  size_t   size;
  int      i, j;

  assert(ping != NULL);

  /* figure out which ping data items we'll store in this record */
  warts_ping_params(ping, flags, &flags_len, &params_len);

  /* length of the ping's flags, parameters, and number of reply records */
  len = flags_len + 2 + params_len + 2;

  if((reply_count = scamper_ping_reply_count(ping)) > 0)
    {
      size = reply_count * sizeof(warts_ping_reply_state_t);
      if((reply_state = (warts_ping_reply_state_t *)malloc(size)) == NULL)
	{
	  goto err;
	}

      for(i=0, j=0; i<ping->ping_sent; i++)
	{
	  for(reply=ping->ping_replies[i]; reply != NULL; reply = reply->next)
	    {
	      if(warts_ping_reply_state(sf,reply,&reply_state[j++],&len) == -1)
		{
		  goto err;
		}
	    }
	}
    }

  if((buf = malloc(len)) == NULL)
    {
      goto err;
    }

  if(warts_ping_params_write(ping, sf, buf, &off, len, flags, flags_len,
			     params_len) == -1)
    {
      goto err;
    }

  /* reply record count */
  insert_uint16(buf, &off, len, &reply_count, NULL);

  /* write each ping reply record */
  for(i=0; i<reply_count; i++)
    {
      warts_ping_reply_write(&reply_state[i], buf, &off, len);
    }
  if(reply_state != NULL)
    {
      free(reply_state);
      reply_state = NULL;
    }

  assert(off == len);

  if(warts_write(sf, SCAMPER_FILE_OBJ_PING, buf, len) == -1)
    {
      goto err;
    }

  free(buf);
  return 0;

 err:
  if(buf != NULL) free(buf);
  return -1;
}

/*
 * scamper_file_warts_read
 *
 */
int scamper_file_warts_read(scamper_file_t *sf, scamper_file_filter_t *filter,
			    uint16_t *type, void **data)
{
  scamper_list_t  *list;
  scamper_cycle_t *cycle;
  scamper_addr_t  *addr;
  warts_hdr_t      hdr;
  int              fd;
  int              isfilter;
  int              tmp;

  fd = scamper_file_getfd(sf);

  for(;;)
    {
      /* read the header for the next record from the file */
      if((tmp = warts_hdr_read(sf, &hdr)) == 0)
	{
	  /* EOF */
	  *data = NULL;
	  break;
	}
      else if(tmp == -1)
	{
	  /* partial record */
	  return -1;
	}

      /* if the header does not pass a basic sanity check, then give up */
      if(hdr.magic != WARTS_MAGIC)
	{
	  return -1;
	}

      /* does the caller wants to know about this type? */
      if((isfilter = scamper_file_filter_isset(filter, hdr.type)) == 1)
	{
	  *type = hdr.type;
	}

      if(hdr.type == SCAMPER_FILE_OBJ_ADDR)
	{
	  if((addr = warts_addr_read(sf, &hdr)) == NULL) return -1;
	  if(isfilter == 1) {*data = scamper_addr_use(addr); return 0; }
	}
      else if(hdr.type == SCAMPER_FILE_OBJ_LIST)
	{
	  if((list = warts_list_read(sf, &hdr)) == NULL) return -1;
	  if(isfilter == 1) {*data = scamper_list_use(list); return 0; }
	}
      else if(hdr.type == SCAMPER_FILE_OBJ_CYCLE_DEF ||
	      hdr.type == SCAMPER_FILE_OBJ_CYCLE_START)
	{
	  if((cycle = warts_cycle_read(sf, &hdr)) == NULL) return -1;
	  if(isfilter == 1) {*data = scamper_cycle_use(cycle); return 0; }
	}
      else if(hdr.type == SCAMPER_FILE_OBJ_CYCLE_STOP)
	{
	  if((cycle = warts_cycle_stop_read(sf, &hdr, isfilter)) == NULL)
	    {
	      return -1;
	    }
	  if(isfilter == 1) {*data = cycle; return 0; }
	}
      else if(isfilter == 0)
	{
	  /* reader doesn't care what the data is, and neither do we */
	  if(lseek(fd, hdr.len, SEEK_CUR) == -1) return -1;
	}
      else if(hdr.type == SCAMPER_FILE_OBJ_TRACE)
	{
	  if((*data = warts_trace_read(sf, &hdr)) == NULL) return -1;
	  return 0;
	}
      else if(hdr.type == SCAMPER_FILE_OBJ_PING)
	{
	  if((*data = warts_ping_read(sf, &hdr)) == NULL) return -1;
	  return 0;
	}
      else
	{
	  /* we don't know about this object, skip it */
	  if(lseek(fd, hdr.len, SEEK_CUR) == -1) return -1;
	}
    }

  return 0;
}

int scamper_file_warts_write_ping(const scamper_file_t *sf,
				  const scamper_ping_t *ping)
{
  return warts_ping_write(sf, ping);
}

int scamper_file_warts_write_trace(const scamper_file_t *sf,
				   const scamper_trace_t *trace)
{
  return warts_trace_write(sf, trace);
}

int scamper_file_warts_write_cycle_start(const scamper_file_t *sf,
					 scamper_cycle_t *c)
{
  return warts_cycle_write(sf, c, SCAMPER_FILE_OBJ_CYCLE_START, NULL);
}

int scamper_file_warts_write_cycle_stop(const scamper_file_t *sf,
					scamper_cycle_t *c)
{
  return warts_cycle_stop_write(sf, c);
}

/*
 * scamper_file_warts_init_read
 *
 * initialise the scamper_file_t's state structure so that it is all set
 * for reading.  the first entry of the list and cycle tables is pre-set
 * to be null for data objects that don't have associated list/cycle
 * objects.
 */
int scamper_file_warts_init_read(scamper_file_t *sf)
{
  warts_state_t *state;
  size_t size;

  if((state = (warts_state_t *)malloc_zero(sizeof(warts_state_t))) == NULL)
    {
      goto err;
    }

  size = sizeof(warts_addr_t *) * WARTS_ADDR_TABLEGROW;
  if((state->addr_table = malloc(size)) == NULL)
    {
      goto err;
    }
  state->addr_table[0] = &state->addr_null;
  state->addr_count = 1;

  size = sizeof(warts_list_t *) * WARTS_LIST_TABLEGROW;
  if((state->list_table = malloc(size)) == NULL)
    {
      goto err;
    }
  state->list_table[0] = &state->list_null;
  state->list_count = 1;

  size = sizeof(warts_cycle_t *) * WARTS_CYCLE_TABLEGROW;
  if((state->cycle_table = malloc(size)) == NULL)
    {
      goto err;
    }
  state->cycle_table[0] = &state->cycle_null;
  state->cycle_count = 1;

  scamper_file_setstate(sf, state);
  return 0;

 err:
  if(state != NULL)
    {
      if(state->addr_table != NULL) free(state->addr_table);
      if(state->list_table != NULL) free(state->list_table);
      if(state->cycle_table != NULL) free(state->cycle_table);
      free(state);
    }
  return -1;
}

/*
 * scamper_file_warts_init_write
 *
 * get the scamper_file_t object ready to write warts objects and keep state
 */
int scamper_file_warts_init_write(scamper_file_t *sf)
{
  warts_state_t *state = NULL;

  if((state = (warts_state_t *)malloc_zero(sizeof(warts_state_t))) == NULL)
    {
      goto err;
    }

  if((state->addr_tree = splaytree_alloc(warts_addr_cmp)) == NULL)
    {
      goto err;
    }
  state->addr_count = 1;

  if((state->list_tree = splaytree_alloc(warts_list_cmp)) == NULL)
    {
      goto err;
    }
  state->list_count = 1;

  if((state->cycle_tree = splaytree_alloc(warts_cycle_cmp)) == NULL)
    {
      goto err;
    }
  state->cycle_count = 1;

  scamper_file_setstate(sf, state);

  return 0;

 err:
  if(state != NULL)
    {
      if(state->addr_tree != NULL)  splaytree_free(state->addr_tree, NULL);
      if(state->list_tree != NULL)  splaytree_free(state->list_tree, NULL);
      if(state->cycle_tree != NULL) splaytree_free(state->cycle_tree, NULL);
      free(state);
    }
  return -1;
}

/*
 * scamper_file_warts_init_append
 *
 * go through the file and form the address, list, and cycle dictionaries
 */
int scamper_file_warts_init_append(scamper_file_t *sf)
{
  warts_state_t *state;
  warts_hdr_t    hdr;
  int            i, fd;
  uint32_t       j;

  /* init the warts structures as if we were reading the file */
  if(scamper_file_warts_init_read(sf) == -1)
    {
      return -1;
    }

  fd = scamper_file_getfd(sf);

  for(;;)
    {
      if((i = warts_hdr_read(sf, &hdr)) == -1)
	{
	  return -1;
	}
      else if(i == 0) break;

      if(hdr.magic != WARTS_MAGIC || hdr.type == 0)
	{
	  return -1;
	}

      switch(hdr.type)
	{
	case SCAMPER_FILE_OBJ_ADDR:
	  if(warts_addr_read(sf, &hdr) == NULL) return -1;
	  break;

	case SCAMPER_FILE_OBJ_LIST:
	  if(warts_list_read(sf, &hdr) == NULL) return -1;
	  break;

	case SCAMPER_FILE_OBJ_CYCLE_START:
	case SCAMPER_FILE_OBJ_CYCLE_DEF:
	  if(warts_cycle_read(sf, &hdr) == NULL) return -1;
	  break;

	case SCAMPER_FILE_OBJ_CYCLE_STOP:
	  if(warts_cycle_stop_read(sf, &hdr, 0) == NULL) return -1;
	  break;

	default:
	  if(lseek(fd, hdr.len, SEEK_CUR) == -1)
	    {
	      return -1;
	    }
	  break;
	}      
    }

  /* get the state structure created in init_read */
  state = scamper_file_getstate(sf);

  /*
   * all the addresses are in a table.  put them into a splay tree so we can
   * find them quickly, and then trash the address table
   */
  if((state->addr_tree = splaytree_alloc(warts_addr_cmp)) == NULL)
    {
      return -1;
    }
  for(j=1; j<state->addr_count; j++)
    {
      if(splaytree_insert(state->addr_tree, state->addr_table[j]) == NULL)
	{
	  return -1;
	}
    }
  free(state->addr_table); state->addr_table = NULL;

  if((state->list_tree = splaytree_alloc(warts_list_cmp)) == NULL)
    {
      return -1;
    }
  for(j=1; j<state->list_count; j++)
    {
      if(splaytree_insert(state->list_tree, state->list_table[j]) == NULL)
	{
	  return -1;
	}
    }
  free(state->list_table); state->list_table = NULL;

  if((state->cycle_tree = splaytree_alloc(warts_cycle_cmp)) == NULL)
    {
      return -1;
    }
  for(j=1; j<state->cycle_count; j++)
    {
      /* don't install finished cycles into the splaytree */
      if(state->cycle_table[j] == NULL)
	{
	  continue;
	}

      if(splaytree_insert(state->cycle_tree, state->cycle_table[j]) == NULL)
	{
	  return -1;
	}
    }
  free(state->cycle_table); state->cycle_table = NULL;

  return 0;
}

int scamper_file_warts_is(const scamper_file_t *sf)
{
  uint16_t magic16;
  int fd = scamper_file_getfd(sf);

  if(lseek(fd, 0, SEEK_SET) == -1)
    {
      return 0;
    }

  if(read_wrap(fd, &magic16, NULL, sizeof(magic16)) != 0)
    {
      return 0;
    }

  if(ntohs(magic16) == WARTS_MAGIC)
    {
      if(lseek(fd, 0, SEEK_SET) == -1)
	{
	  return 0;
	}
      return 1;
    }

  return 0;
}

static void warts_free_state(splaytree_t *tree, void **table,
			     unsigned int count, splaytree_free_t free_cb)
{
  unsigned int i;

  if(table != NULL)
    {
      for(i=1; i<count; i++)
	{
	  if(table[i] != NULL)
	    {
	      free_cb(table[i]);
	    }
	}
      free(table);
    }
  if(tree != NULL)
    {
      splaytree_free(tree, free_cb);
    }

  return;
}

void scamper_file_warts_free_state(scamper_file_t *sf)
{
  warts_state_t *state;

  /* there may not actually be state allocated with the file ... */
  if((state = scamper_file_getstate(sf)) == NULL)
    {
      return;
    }

  warts_free_state(state->list_tree,
		   (void **)state->list_table, state->list_count,
		   (splaytree_free_t)warts_list_free);

  warts_free_state(state->cycle_tree,
		   (void **)state->cycle_table, state->cycle_count,
		   (splaytree_free_t)warts_cycle_free);

  warts_free_state(state->addr_tree,
		   (void **)state->addr_table, state->addr_count,
		   (splaytree_free_t)warts_addr_free);

  free(state);

  return;
}


syntax highlighted by Code2HTML, v. 0.9.1