/* ** Copyright (C) 2006-2007 by Carnegie Mellon University. ** ** @OPENSOURCE_HEADER_START@ ** ** Use of the SILK system and related source code is subject to the terms ** of the following licenses: ** ** GNU Public License (GPL) Rights pursuant to Version 2, June 1991 ** Government Purpose License Rights (GPLR) pursuant to DFARS 252.225-7013 ** ** NO WARRANTY ** ** ANY INFORMATION, MATERIALS, SERVICES, INTELLECTUAL PROPERTY OR OTHER ** PROPERTY OR RIGHTS GRANTED OR PROVIDED BY CARNEGIE MELLON UNIVERSITY ** PURSUANT TO THIS LICENSE (HEREINAFTER THE "DELIVERABLES") ARE ON AN ** "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY ** KIND, EITHER EXPRESS OR IMPLIED AS TO ANY MATTER INCLUDING, BUT NOT ** LIMITED TO, WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE, ** MERCHANTABILITY, INFORMATIONAL CONTENT, NONINFRINGEMENT, OR ERROR-FREE ** OPERATION. CARNEGIE MELLON UNIVERSITY SHALL NOT BE LIABLE FOR INDIRECT, ** SPECIAL OR CONSEQUENTIAL DAMAGES, SUCH AS LOSS OF PROFITS OR INABILITY ** TO USE SAID INTELLECTUAL PROPERTY, UNDER THIS LICENSE, REGARDLESS OF ** WHETHER SUCH PARTY WAS AWARE OF THE POSSIBILITY OF SUCH DAMAGES. ** LICENSEE AGREES THAT IT WILL NOT MAKE ANY WARRANTY ON BEHALF OF ** CARNEGIE MELLON UNIVERSITY, EXPRESS OR IMPLIED, TO ANY PERSON ** CONCERNING THE APPLICATION OF OR THE RESULTS TO BE OBTAINED WITH THE ** DELIVERABLES UNDER THIS LICENSE. ** ** Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie ** Mellon University, its trustees, officers, employees, and agents from ** all claims or demands made against them (and any related losses, ** expenses, or attorney's fees) arising out of, or relating to Licensee's ** and/or its sub licensees' negligent use or willful misuse of or ** negligent conduct or willful misconduct regarding the Software, ** facilities, or other rights or assistance granted by Carnegie Mellon ** University under this License, including, but not limited to, any ** claims of product liability, personal injury, death, damage to ** property, or violation of any laws or regulations. ** ** Carnegie Mellon University Software Engineering Institute authored ** documents are sponsored by the U.S. Department of Defense under ** Contract F19628-00-C-0003. Carnegie Mellon University retains ** copyrights in all material produced under this contract. The U.S. ** Government retains a non-exclusive, royalty-free license to publish or ** reproduce these documents, or allow others to do so, for U.S. ** Government purposes only pursuant to the copyright license under the ** contract clause at 252.227.7013. ** ** @OPENSOURCE_HEADER_END@ */ /* ** Routines for buffered file io. */ #include "silk.h" RCSIDENT("$SiLK: skiobuf.c 7636 2007-06-25 17:14:20Z mthomas $"); #include "utils.h" #ifdef SK_LZO_HEADER_NAME # include SK_LZO_HEADER_NAME #endif #include "skiobuf.h" /* Options for compression types. */ typedef union _iobuf_opts_t { #ifdef HAVE_ZLIB_H /* zlib */ struct { int level; } zlib; #endif #ifdef SK_LZO_HEADER_NAME /* lzo */ struct { uint8_t *scratch; } lzo; #endif char nothing; /* Just to keep the union from being empty */ } iobuf_opts_t; typedef struct compr_sizes_t { uint32_t compr_size; uint32_t uncompr_size; } compr_sizes_t; /* An IO buffer. */ struct _sk_iobuf_t { uint8_t compr_method; /* Compression method */ iobuf_opts_t compr_opts; /* Compression options */ uint8_t *compr_buf; /* Compression buffer */ uint8_t *uncompr_buf; /* Decompression buffer */ uint32_t compr_buf_size; /* Size of compr buffer */ uint32_t uncompr_buf_size; /* Size of uncompr buffer */ uint32_t block_size; /* Block size */ uint32_t block_quantum; /* quanta size block is divided into */ uint32_t pos; /* Byte position in buffer */ uint32_t max_bytes; /* Maximim bytes allowed in buf */ void *fd; /* File descriptor */ skio_abstract_t io; /* IO information */ off_t total; /* Total read or written */ int io_errno; /* errno of error */ uint32_t error_line; /* line number of error */ unsigned fd_valid : 1; /* File descriptor valid? */ unsigned used : 1; /* Set after a read or write */ unsigned write : 1; /* Read or write */ unsigned eof : 1; /* End of file or flushed */ unsigned error : 1; /* Error state? */ unsigned interr : 1; /* Internal or external error? */ unsigned ioerr : 1; /* IO error */ }; /* Method table for different types of compression */ typedef struct _iobuf_methods_t { /* Initialization. Should set the default opts. */ int (*init_method) (iobuf_opts_t *opts); /* Deinitialization. Should free any default opts. */ int (*uninit_method) (iobuf_opts_t *opts); /* Should return the maximum compressed size given a compressed size. */ uint32_t (*compr_size_method) (uint32_t compr_size, const iobuf_opts_t *opts); /* The compression method. '*destlen' will be set to the length of the destination buffer before being called. */ int (*compr_method) (void *dest, uint32_t *destlen, const void *source, uint32_t sourcelen, const iobuf_opts_t *opts); /* The decompression method. '*destlen' will be set to the length of the destination buffer before being called. */ int (*uncompr_method) (void *dest, uint32_t *destlen, const void *source, uint32_t sourcelen, const iobuf_opts_t *opts); /* Whether this compression method requires block sizes before the compressed blocks. */ unsigned block_numbers : 1; } iobuf_methods_t; enum internal_errors { ESKIO_BADOPT = 0, ESKIO_BADCOMPMETHOD, ESKIO_BLOCKSIZE, ESKIO_COMP, ESKIO_INITFAIL, ESKIO_MALLOC, ESKIO_NOFD, ESKIO_NOREAD, ESKIO_NOWRITE, ESKIO_SHORTREAD, ESKIO_TOOBIG, ESKIO_UNCOMP, ESKIO_USED }; static char* internal_messages[] = { "Illegal compression or decompression option", "Bad compression method", "Block size is too large", "Error during compression", "Compression initialization failed", "Out of memory", "File descriptor is not set", "Attempt to read from an IO buffer writer", "Attempt to write to an IO buffer reader", "Could not read complete compressed block", "Count is too large", "Error during decompression", "Parameter set on IO buffer after buffer has been used" }; static iobuf_methods_t methods[]; static int num_methods; #define SKIOBUF_INTERNAL_ERROR(fd, err) \ do { \ (fd)->io_errno = (int)(err); \ (fd)->error = 1; \ (fd)->interr = 1; \ (fd)->error_line = __LINE__; \ return -1; \ } while (0) #define SKIOBUF_EXTERNAL_ERROR(fd) \ do { \ (fd)->io_errno = errno; \ (fd)->error = 1; \ (fd)->error_line = __LINE__; \ return -1; \ } while (0) #define SKIOBUF_IO_ERROR(fd) \ do { \ (fd)->io_errno = errno; \ (fd)->error = 1; \ (fd)->ioerr = 1; \ (fd)->fd_valid = 0; \ (fd)->error_line = __LINE__; \ return -1; \ } while (0) /* Set the buffer sizes based on the block_size and block_quantum */ static void calculate_buffer_sizes(sk_iobuf_t *fd) { iobuf_methods_t *method; method = &methods[fd->compr_method]; fd->uncompr_buf_size = fd->block_size; fd->max_bytes = fd->block_size - (fd->block_size % fd->block_quantum); if (method->compr_size_method) { fd->compr_buf_size = method->compr_size_method(fd->block_size, &fd->compr_opts); } else { fd->compr_buf_size = fd->block_size; } if (fd->compr_buf) { free(fd->compr_buf); fd->compr_buf = NULL; } if (fd->uncompr_buf) { free(fd->uncompr_buf); fd->uncompr_buf = NULL; } } /* Create an IO buffer reader */ sk_iobuf_t *skIOBufCreateReader(void) { sk_iobuf_t *fd = NULL; fd = calloc(1, sizeof(sk_iobuf_t)); if (fd == NULL) { return NULL; } fd->block_size = fd->uncompr_buf_size = SKIOBUF_DEFAULT_BLOCKSIZE; fd->block_quantum = SKIOBUF_DEFAULT_RECORDSIZE; fd->compr_method = SK_COMPMETHOD_NONE; return fd; } /* Create an IO buffer writer */ sk_iobuf_t *skIOBufCreateWriter(void) { sk_iobuf_t *fd = NULL; /* First, get a reader. */ fd = skIOBufCreateReader(); if (fd == NULL) { return NULL; } /* Next, turn it into a writer. */ fd->write = 1; fd->pos = 0; return fd; } /* Destroy an IO buffer */ void skIOBufDestroy(sk_iobuf_t *fd) { iobuf_methods_t *method; assert(fd); /* Will set error on reader, but that's okay, since we free the structure immediately afterwards. */ skIOBufFlush(fd); if (fd->compr_buf) { free(fd->compr_buf); } if (fd->uncompr_buf) { free(fd->uncompr_buf); } method = &methods[fd->compr_method]; if (method->uninit_method) { method->uninit_method(&fd->compr_opts); } if (fd->io.free_fd) { fd->io.free_fd(fd->fd); } free(fd); } /* Wrapper around simple read */ static ssize_t raw_read(void *vfd, void *dest, size_t count) { int *fd = (int *)vfd; return read(*fd, dest, count); } /* Wrapper around simple write */ static ssize_t raw_write(void *vfd, void *src, size_t count) { int *fd = (int *)vfd; return write(*fd, src, count); } static const char *raw_strerror(void *UNUSED(vfd), int io_errno) { return strerror(io_errno); } #ifdef HAVE_ZLIB_H static const char *gz_strerror(void *vfd, int io_errno) { int gzerr; gzFile fd = (gzFile)vfd; const char *errstr; errstr = gzerror(fd, &gzerr); if (gzerr == Z_ERRNO) { errstr = strerror(io_errno); } return errstr; } static int gz_flush(void *vfd) { gzFile fd = (gzFile)vfd; return gzflush(fd, Z_SYNC_FLUSH); } #endif /* Prepare for binding a file descriptor */ static int skio_bind_wrapper(sk_iobuf_t *fd, int compmethod) { off_t total; iobuf_methods_t *method; /* if we are bound to something, uninitialize it */ method = &methods[fd->compr_method]; if (method->uninit_method) { if (method->uninit_method(&fd->compr_opts)) { return -1; } } method = &methods[compmethod]; assert(fd); if (compmethod < 0 || compmethod >= num_methods) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_BADCOMPMETHOD); } #ifndef SK_LZO_HEADER_NAME if (compmethod == SK_COMPMETHOD_LZO1X) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_BADCOMPMETHOD); } #endif #ifndef HAVE_ZLIB_H if (compmethod == SK_COMPMETHOD_ZLIB) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_BADCOMPMETHOD); } #endif if (fd == NULL) { return -1; } if (fd->fd_valid) { total = skIOBufFlush(fd); if (total == -1) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_INITFAIL); } } fd->compr_method = compmethod; fd->total = 0; fd->used = 0; fd->error = 0; fd->interr = 0; fd->ioerr = 0; fd->io_errno = 0; fd->eof = 0; if (method->init_method) { if (method->init_method(&fd->compr_opts)) { return -1; } } calculate_buffer_sizes(fd); if (!fd->write) { fd->pos = fd->max_bytes; } return 0; } /* Bind an IO buffer */ int skIOBufBind(sk_iobuf_t *fd, int file, int compmethod) { skio_abstract_t io; int *fh; fh = malloc(sizeof(*fh)); if (fh == NULL) { return -1; } *fh = file; io.read = &raw_read; io.write = &raw_write; io.strerror = &raw_strerror; io.flush = NULL; io.free_fd = &free; return skIOBufBindAbstract(fd, fh, compmethod, &io); } int skIOBufBindAbstract( sk_iobuf_t *fd, void *file, int compmethod, skio_abstract_t *fd_ops) { int rv = skio_bind_wrapper(fd, compmethod); fd->io = *fd_ops; fd->fd = file; fd->fd_valid = 1; return rv; } #ifdef HAVE_ZLIB_H /* Bind an gzip IO buffer */ int skIOBufBindGzip(sk_iobuf_t *fd, gzFile file, int compmethod) { skio_abstract_t io; io.read = (skio_read_fn_t)&gzread; io.write = (skio_write_fn_t)&gzwrite; io.strerror = &gz_strerror; io.flush = &gz_flush; io.free_fd = NULL; return skIOBufBindAbstract(fd, file, compmethod, &io); } #endif /* Handle actual read and decompression of a block */ static int32_t skio_uncompr(sk_iobuf_t *fd) { uint32_t comp_block_size, uncomp_block_size, new_block_size; uint32_t num_to_read; ssize_t readlen; uint8_t *bufpos; iobuf_methods_t *method; assert(fd); method = &methods[fd->compr_method]; if ( !method->block_numbers) { /* Without block numbers, assume fd->max_bytes for * everything. */ comp_block_size = fd->max_bytes; uncomp_block_size = fd->max_bytes; new_block_size = fd->max_bytes; } else { /* Read in the compressed block sizes */ bufpos = (uint8_t*)&comp_block_size; num_to_read = sizeof(comp_block_size); while (num_to_read) { readlen = fd->io.read(fd->fd, bufpos, num_to_read); if (readlen == -1) { if (errno == EINTR) { continue; } SKIOBUF_IO_ERROR(fd); } if (readlen == 0) { /* We've reached eof. */ fd->eof = 1; if (num_to_read != sizeof(comp_block_size)) { /* partial read */ SKIOBUF_INTERNAL_ERROR(fd, ESKIO_SHORTREAD); } return 0; } fd->total += readlen; num_to_read -= readlen; bufpos += readlen; } /* If we have reached the end of the compressed stream, we have the bytes we have. */ if (comp_block_size == 0) { fd->eof = 1; return 0; } /* Read in the uncompressed block sizes */ bufpos = (uint8_t*)&uncomp_block_size; num_to_read = sizeof(uncomp_block_size); while (num_to_read) { readlen = fd->io.read(fd->fd, bufpos, num_to_read); if (readlen == -1) { if (errno == EINTR) { continue; } SKIOBUF_IO_ERROR(fd); } if (readlen == 0) { /* We've reached eof, though we weren't expecting to */ fd->eof = 1; SKIOBUF_INTERNAL_ERROR(fd, ESKIO_SHORTREAD); } fd->total += readlen; num_to_read -= readlen; bufpos += readlen; } comp_block_size = ntohl(comp_block_size); uncomp_block_size = new_block_size = ntohl(uncomp_block_size); } if (uncomp_block_size > SSIZE_MAX) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_BLOCKSIZE); } /* Reallocate buffers if necessary */ if (method->uncompr_method != NULL && (comp_block_size > fd->compr_buf_size || fd->uncompr_buf == NULL)) { if (fd->compr_buf) { free(fd->compr_buf); } fd->compr_buf = malloc(comp_block_size); if (fd->compr_buf == NULL) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_MALLOC); } fd->compr_buf_size = comp_block_size; } if (uncomp_block_size > fd->uncompr_buf_size || fd->uncompr_buf == NULL) { if (fd->uncompr_buf) { free(fd->uncompr_buf); } fd->uncompr_buf = malloc(uncomp_block_size); if (fd->uncompr_buf == NULL) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_MALLOC); } fd->uncompr_buf_size = uncomp_block_size; } /* Read in the block */ bufpos = method->uncompr_method ? fd->compr_buf : fd->uncompr_buf; num_to_read = comp_block_size; while (num_to_read) { readlen = fd->io.read(fd->fd, bufpos, num_to_read); if (readlen == -1) { if (errno == EINTR) { continue; } SKIOBUF_IO_ERROR(fd); } if (readlen == 0) { if (method->block_numbers) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_SHORTREAD); } fd->eof = 1; new_block_size = comp_block_size - num_to_read; comp_block_size = new_block_size; break; } fd->total += readlen; num_to_read -= readlen; bufpos += readlen; } /* Decompress it */ if (method->uncompr_method) { new_block_size = fd->uncompr_buf_size; if (method->uncompr_method(fd->uncompr_buf, &new_block_size, fd->compr_buf, comp_block_size, &fd->compr_opts)) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_UNCOMP); } /* Verify the block's uncompressed size */ if (new_block_size != uncomp_block_size) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_UNCOMP); } } /* Register the new data in the struct */ fd->max_bytes = new_block_size; fd->pos = 0; return fd->max_bytes; } /* Read data from an IO buffer. If 'c' is non-null, stop when the * char '*c' is encountered. */ static ssize_t _skIOBufRead(sk_iobuf_t *fd, void *buf, size_t count, int *c) { ssize_t total = 0; int found_c = 0; assert(fd); /* Take care of boundary conditions */ if (fd == NULL) { return -1; } if (fd->write) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_NOREAD); } if (!fd->fd_valid) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_NOFD); } if (count == 0) { return 0; } assert(count <= SSIZE_MAX); if (count > SSIZE_MAX) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_TOOBIG); } /* Transfer the bytes */ while (count && !found_c) { size_t left = fd->max_bytes - fd->pos; size_t num; int32_t uncompr_size; /* If we have no bytes, we must get some. */ if (left == 0) { if (fd->eof) { break; } uncompr_size = skio_uncompr(fd); if (uncompr_size == -1) { return -1; } fd->used = 1; left = fd->max_bytes; if (uncompr_size == 0) { assert(fd->eof); break; } } /* Calculate how many bytes to read from our current buffer */ num = (count < left) ? count : left; /* Copy the bytes, and update the data */ if (c != NULL) { void *after = memccpy(buf, &fd->uncompr_buf[fd->pos], *c, num); if (after != NULL) { found_c = 1; num = ((uint8_t *)after) - ((uint8_t *)buf); } } else { memcpy(buf, &fd->uncompr_buf[fd->pos], num); } fd->pos += num; total += num; count -= num; buf = (uint8_t *)buf + num; } return total; } /* Standard read function for skiobuf */ ssize_t skIOBufRead(sk_iobuf_t *fd, void *buf, size_t count) { return _skIOBufRead(fd, buf, count, NULL); } /* Read until buffer full or character in 'c' encountered */ ssize_t skIOBufReadToChar(sk_iobuf_t *fd, void *buf, size_t count, int c) { return _skIOBufRead(fd, buf, count, &c); } /* Handle actual compression and write of a block */ static int32_t skio_compr(sk_iobuf_t *fd) { uint32_t compr_size, uncompr_size; uint32_t size; ssize_t writelen; uint8_t *bufpos; iobuf_methods_t *method; uint32_t extra; uint32_t offset; assert(fd); method = &methods[fd->compr_method]; uncompr_size = fd->pos; extra = fd->pos % fd->block_quantum; /* Programmer's error if we don't have complete records */ assert(extra == 0); /* If assertions aren't turned on, at least pad the record */ if (extra != 0) { memset(&fd->uncompr_buf[fd->pos], 0, extra); uncompr_size += extra; } /* Extra bit added on for block sizes */ offset = method->block_numbers ? sizeof(compr_sizes_t) : 0; /* Compress the block */ if (method->compr_method) { /* Create the compression buffer, if necessary */ if (fd->compr_buf == NULL) { fd->compr_buf = malloc(fd->compr_buf_size + offset); if (fd->compr_buf == NULL) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_MALLOC); } } compr_size = fd->compr_buf_size; if (method->compr_method(fd->compr_buf + offset, &compr_size, fd->uncompr_buf, uncompr_size, &fd->compr_opts) != 0) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_COMP); } bufpos = fd->compr_buf; } else { compr_size = fd->pos; bufpos = fd->uncompr_buf; } size = compr_size + offset; if (method->block_numbers) { compr_sizes_t *sizes = (compr_sizes_t *)fd->compr_buf; /* Write out the block numbers */ sizes->compr_size = htonl(compr_size); sizes->uncompr_size = htonl(uncompr_size); } /* Write out compressed data */ while (size) { writelen = fd->io.write(fd->fd, bufpos, size); if (writelen == -1 || writelen == 0) { if (writelen == -1 && errno == EINTR) { continue; } SKIOBUF_IO_ERROR(fd); } fd->total += writelen; bufpos += writelen; size -= writelen; } fd->pos = 0; return (int32_t)size; } /* Write data to an IO buffer */ ssize_t skIOBufWrite(sk_iobuf_t *fd, const void *buf, size_t count) { ssize_t total = 0; assert(fd); /* Take care of boundary conditions */ if (count == 0) { return 0; } if (fd == NULL) { return -1; } if (!fd->write) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_NOWRITE); } if (!fd->fd_valid) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_NOFD); } assert(count <= SSIZE_MAX); if (count > SSIZE_MAX) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_TOOBIG); } fd->used = 1; /* If the buffer hasn't been created yet, create it. */ if (fd->uncompr_buf == NULL) { fd->uncompr_buf = malloc(fd->uncompr_buf_size); if (fd->uncompr_buf == NULL) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_MALLOC); } } /* Transfer the bytes */ while (count) { size_t left = fd->max_bytes - fd->pos; size_t num; /* If we have filled the buffer, we must write it out. */ if (left == 0) { int32_t compr_size = skio_compr(fd); if (compr_size == -1) { return -1; } left = fd->max_bytes; } /* Calculate how many bytes to write into our current buffer */ num = (count < left) ? count : left; /* Copy the bytes, and update the data */ memcpy(&fd->uncompr_buf[fd->pos], buf, num); fd->pos += num; total += num; count -= num; buf = (uint8_t *)buf + num; } return total; } /* Finish writing to an IO buffer */ off_t skIOBufFlush(sk_iobuf_t *fd) { assert(fd); if (fd == NULL) { return -1; } if (!fd->write) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_NOWRITE); } if (!fd->fd_valid) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_NOFD); } if (fd->pos) { int32_t compr_size = skio_compr(fd); if (compr_size == -1) { return -1; } } if (fd->io.flush) { fd->io.flush(fd->fd); } return fd->total; } /* Total number of bytes read/written from/to file descriptor */ off_t skIOBufTotal(sk_iobuf_t *fd) { assert(fd); if (fd == NULL) { return -1; } return fd->total; } /* Maximum total number of bytes in a compressed block */ uint32_t skIOBufUpperCompBlockSize(sk_iobuf_t *fd) { iobuf_methods_t *method; uint32_t total; assert(fd); assert(fd->write); method = &methods[fd->compr_method]; if (method->compr_size_method) { total = method->compr_size_method(fd->max_bytes, &fd->compr_opts); } else { total = fd->max_bytes; } if (method->block_numbers) { total += sizeof(compr_sizes_t); } return total; } /* Maximum total number of bytes written to file descriptor after a flush */ off_t skIOBufTotalUpperBound(sk_iobuf_t *fd) { iobuf_methods_t *method; off_t total; assert(fd); if (fd == NULL) { return -1; } if (!fd->write) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_NOWRITE); } method = &methods[fd->compr_method]; total = fd->total + fd->pos; if (method->block_numbers) { total += sizeof(compr_sizes_t); } if (method->compr_size_method) { total += (method->compr_size_method(fd->max_bytes, &fd->compr_opts) - fd->max_bytes); } return total; } /* Sets the block size */ int skIOBufSetBlockSize(sk_iobuf_t *fd, uint32_t size) { assert(fd); if (fd == NULL) { return -1; } if (fd->used) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_USED); } fd->block_size = size; calculate_buffer_sizes(fd); return 0; } /* Sets the write quantum */ int skIOBufSetRecordSize(sk_iobuf_t *fd, uint32_t size) { assert(fd); if (fd == NULL) { return -1; } if (fd->used) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_USED); } fd->block_quantum = size; calculate_buffer_sizes(fd); return 0; } /* Create an error message */ const char *skIOBufStrError(sk_iobuf_t *fd) { static char buf[256]; static char *message; message = buf; if (!fd->error) { message = "No error"; } else if (fd->interr) { snprintf(buf, sizeof(buf), "Error in " __FILE__ ":%u [%s]", fd->error_line, internal_messages[fd->io_errno]); } else if (fd->ioerr) { snprintf(buf, sizeof(buf), "IO error in " __FILE__ ":%u [%s]", fd->error_line, fd->io.strerror(fd->fd, fd->io_errno)); } else { snprintf(buf, sizeof(buf), "System error in " __FILE__ ":%u (%d)[%s]", fd->error_line, fd->io_errno, strerror(fd->io_errno)); } fd->error = 0; fd->interr = 0; fd->ioerr = 0; fd->io_errno = 0; return message; } #ifdef HAVE_ZLIB_H /* zlib methods */ int skIOBufSetZlibLevel(sk_iobuf_t *fd, int level) { assert(fd); if (fd == NULL) { return -1; } if (!(level >= Z_BEST_SPEED && level <= Z_BEST_COMPRESSION) && level != Z_NO_COMPRESSION && level != Z_DEFAULT_COMPRESSION) { SKIOBUF_INTERNAL_ERROR(fd, ESKIO_BADOPT); } fd->compr_opts.zlib.level = level; return 0; } static int zlib_init_method(iobuf_opts_t *opts) { opts->zlib.level = Z_DEFAULT_COMPRESSION; return 0; } static uint32_t zlib_compr_size_method( uint32_t compr_size, const iobuf_opts_t *UNUSED(opts)) { #ifdef HAVE_COMPRESSBOUND return compressBound(compr_size); #else return compr_size + compr_size / 1000 + 12; #endif } static int zlib_compr_method( void *dest, uint32_t *destlen, const void *source, uint32_t sourcelen, const iobuf_opts_t *opts) { uLongf dl; uLong sl; int rv; assert(sizeof(sl) >= sizeof(sourcelen)); assert(sizeof(dl) >= sizeof(*destlen)); sl = sourcelen; dl = *destlen; rv = compress2(dest, &dl, source, sl, opts->zlib.level); *destlen = dl; rv = (rv == Z_OK) ? 0 : -1; return rv; } static int zlib_uncompr_method( void *dest, uint32_t *destlen, const void *source, uint32_t sourcelen, const iobuf_opts_t *UNUSED(opts)) { uLongf dl; uLong sl; int rv; assert(sizeof(sl) >= sizeof(sourcelen)); assert(sizeof(dl) >= sizeof(*destlen)); sl = sourcelen; dl = *destlen; rv = uncompress(dest, &dl, source, sl); *destlen = dl; rv = (rv == Z_OK) ? 0 : -1; return rv; } #endif #ifdef SK_LZO_HEADER_NAME /* LZO methods */ static int lzo_init_method(iobuf_opts_t *opts) { static int initialized = 0; if (initialized) { if (lzo_init()) { return -1; } initialized = 1; } opts->lzo.scratch = (uint8_t *)calloc(1, LZO1X_1_15_MEM_COMPRESS); if (opts->lzo.scratch == NULL) { return -1; } return 0; } static int lzo_uninit_method(iobuf_opts_t *opts) { assert(opts->lzo.scratch != NULL); free(opts->lzo.scratch); opts->lzo.scratch = NULL; return 0; } static uint32_t lzo_compr_size_method( uint32_t compr_size, const iobuf_opts_t *UNUSED(opts)) { /* The following formula is in the lzo faq: http://www.oberhumer.com/opensource/lzo/lzofaq.php */ return (compr_size + (compr_size >> 4) + 64 + 3); } static int lzo_compr_method( void *dest, uint32_t *destlen, const void *source, uint32_t sourcelen, const iobuf_opts_t *opts) { lzo_uint sl, dl; int rv; assert(sizeof(sl) >= sizeof(sourcelen)); assert(sizeof(dl) >= sizeof(*destlen)); dl = *destlen; sl = sourcelen; rv = lzo1x_1_15_compress(source, sl, dest, &dl, opts->lzo.scratch); *destlen = dl; return rv; } static int lzo_uncompr_method( void *dest, uint32_t *destlen, const void *source, uint32_t sourcelen, const iobuf_opts_t *UNUSED(opts)) { lzo_uint sl, dl; int rv; assert(sizeof(sl) >= sizeof(sourcelen)); assert(sizeof(dl) >= sizeof(*destlen)); dl = *destlen; sl = sourcelen; rv = lzo1x_decompress_safe(source, sl, dest, &dl, NULL); *destlen = dl; return rv; } #endif static iobuf_methods_t methods[] = { /* NONE */ { NULL, NULL, NULL, NULL, NULL, 0 }, #ifdef HAVE_ZLIB_H /* ZLIB */ { zlib_init_method, NULL, zlib_compr_size_method, zlib_compr_method, zlib_uncompr_method, 1}, #else /* placeholder */ { NULL, NULL, NULL, NULL, NULL, 1 }, #endif #ifdef SK_LZO_HEADER_NAME /* LZO1X */ { lzo_init_method, lzo_uninit_method, lzo_compr_size_method, lzo_compr_method, lzo_uncompr_method, 1} #else /* placeholder */ { NULL, NULL, NULL, NULL, NULL, 1 } #endif }; static int num_methods = sizeof(methods) / sizeof(iobuf_methods_t); /* ** Local Variables: ** mode:c ** indent-tabs-mode:nil ** c-basic-offset:4 ** End: */