# Copyright 2002 Ben Escoto
#
# This file is part of duplicity.
#
# Duplicity is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# Duplicity is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with duplicity; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""Miscellaneous classes and methods"""
import os
import log
class MiscError(Exception):
"""Signifies a miscellaneous error..."""
pass
class FileVolumeWriter:
"""Split up an incoming fileobj into multiple volumes on disk
This class can also be used as an iterator. It returns the
filenames of the files it writes.
"""
volume_size = 50 * 1024 * 1024
blocksize = 64 * 1024
def __init__(self, infp, file_prefix):
"""FileVolumeWriter initializer
infp is a file object opened for reading. It will be closed
at end. file_prefix is the full path of the volumes that will
be written. If more than one is required, it will be appended
with .1, .2, etc.
"""
self.infp = infp
self.prefix = file_prefix
self.current_index = 1
self.finished = None # set to true when completely done
self.buffer = "" # holds data that belongs in next volume
def get_initial_buf(self):
"""Get first value of buffer, from self.buffer or infp"""
if self.buffer:
buf = self.buffer
self.buffer = ""
return buf
else: return self.infp.read(self.blocksize)
def write_volume(self, outfp):
"""Write self.volume_size bytes from self.infp to outfp
Return None if we have reached end of infp without reaching
volume size, and false otherwise.
"""
bytes_written, buf = 0, self.get_initial_buf()
while len(buf) + bytes_written <= self.volume_size:
if not buf: # reached end of input
outfp.close()
return None
if len(buf) + bytes_written > self.volume_size: break
outfp.write(buf)
bytes_written += len(buf)
buf = self.infp.read(self.blocksize)
remainder = self.volume_size - bytes_written
assert remainder < len(buf)
outfp.write(buf[:remainder])
outfp.close()
self.buffer = buf[remainder:]
return 1
def next(self):
"""Write next file, return filename"""
if self.finished: raise StopIteration
filename = "%s.%d" % (self.prefix, self.current_index)
log.Log("Starting to write %s" % filename, 5)
outfp = open(filename, "wb")
if not self.write_volume(outfp): # end of input
self.finished = 1
if self.current_index == 1: # special case first index
log.Log("One only volume required.\n"
"Renaming %s to %s" % (filename, self.prefix), 4)
os.rename(filename, self.prefix)
return self.prefix
else: self.current_index += 1
return filename
def __iter__(self): return self
class BufferedFile:
"""Buffer file open for reading, so reads will happen in fixed sizes
This is currently used to buffer a GzipFile, because that class
apparently doesn't respond well to arbitrary read sizes.
"""
def __init__(self, fileobj, blocksize = 32 * 1024):
self.fileobj = fileobj
self.buffer = ""
self.blocksize = blocksize
def read(self, length = -1):
"""Return length bytes, or all if length < 0"""
if length < 0:
while 1:
buf = self.fileobj.read(self.blocksize)
if not buf: break
self.buffer += buf
real_length = len(self.buffer)
else:
while len(self.buffer) < length:
buf = self.fileobj.read(self.blocksize)
if not buf: break
self.buffer += buf
real_length = min(length, len(self.buffer))
result = self.buffer[:real_length]
self.buffer = self.buffer[real_length:]
return result
def close(self): self.fileobj.close()
def copyfileobj(infp, outfp, byte_count = -1):
"""Copy byte_count bytes from infp to outfp, or all if byte_count < 0
Returns the number of bytes actually written (may be less than
byte_count if find eof. Does not close either fileobj.
"""
blocksize = 64 * 1024
bytes_written = 0
if byte_count < 0:
while 1:
buf = infp.read(blocksize)
if not buf: break
bytes_written += len(buf)
outfp.write(buf)
else:
while bytes_written + blocksize <= byte_count:
buf = infp.read(blocksize)
if not buf: break
bytes_written += len(buf)
outfp.write(buf)
buf = infp.read(byte_count - bytes_written)
bytes_written += len(buf)
outfp.write(buf)
return bytes_written
def copyfileobj_close(infp, outfp):
"""Copy infp to outfp, closing afterwards"""
copyfileobj(infp, outfp)
if infp.close(): raise MiscError("Error closing input file")
if outfp.close(): raise MiscError("Error closing output file")
syntax highlighted by Code2HTML, v. 0.9.1