#!/usr/bin/env python
#  -*- mode: python; indent-tabs-mode: nil; -*- coding: iso-8859-1 -*-

"""
PluckerDocs.py   $Id: PluckerDocs.py,v 1.67 2004/04/05 20:09:47 chrish Exp $

(Corresponds to disass.py  Version 0.16)


Contains a set of classes to hold information about Plucker documents.

Called as a script it is a utility to
o disassemble files generated by the plucker awk parser (or any other
  parser) or any other binary form of a plucker record,
o re-assemble these into possible a different format
o collect statistics about the disassembled documents

Call this as
  ``python <script name>'' and it will give you a list of valid options.

If you get a traceback with an assertion error, something is wrong in
the data file, i.e. some data not conforming to the specification was
found.  (Parser writers: if you disagree somewhere or find a useful
check that is missing, please tell me.)


Copyright 1999,2000 by Holger Duerer <holly@starship.python.net>

Distributable under the GNU General Public License Version 2 or newer.

"""

if __name__ == '__main__':
    ## The following section tries to get the PyPlucker directory onto the
    ## system path if called as a script and if it is not yet there:
    try: import PyPlucker
    except ImportError:
        import os, sys
        file = sys.argv[0]
        while os.path.islink (file): file = os.readlink (file)
        sys.path = [os.path.split (os.path.dirname (file))[0]] + sys.path
        try: import PyPlucker
        except ImportError:
            error("Cannot find where module PyPlucker is located!")
            sys.exit (1)

        # and forget the temp names...
        del file, os
    del PyPlucker
    ##
    ## Now PyPlucker things should generally be importable
    ##



import string
import struct
import types
import urlparse
import urllib
import PluckerDocs
try:
    import zlib
except ImportError:
    zlib = None

import StringIO
from rfc822 import Message

from UtilFns import message, error

import time
PILOT_TIME_DELTA = 2082844800L

try:
    from  Pyrite import doc_compress
    doc_compress_function = doc_compress.compress
    doc_uncompress_function = doc_compress.uncompress
except ImportError:
    # Pyrite not installed?
    # try other dirs to import the file directly
    import sys, os
    sys.path.append(".")
    sys.path.append(os.path.dirname (sys.argv[0]))
    try:
        from PyPlucker.helper import doc_compress
        doc_compress_function = doc_compress.compress
        doc_uncompress_function = doc_compress.uncompress
    except ImportError:
        doc_compress_function = None
        doc_uncompress_function = None
try:
    from Pyrite import _Doc
    doc_block_compress_function = _Doc.compress
except ImportError:
    doc_block_compress_function = doc_compress_function


## These constants are only valid for the new format!!!
DOCTYPE_HTML = 0
DOCTYPE_HTML_COMPRESSED = (1 << 8)
DOCTYPE_IMAGE = (2 << 8)
DOCTYPE_IMAGE_COMPRESSED = (3 << 8)
DOCTYPE_MAILTO = (4 << 8)
DOCTYPE_LINK_INDEX = (5 << 8)
DOCTYPE_LINKS = (6 << 8)
DOCTYPE_LINKS_COMPRESSED = (7 << 8)
DOCTYPE_BOOKMARKS = (8 << 8)
DOCTYPE_CATEGORY = (9 << 8)
DOCTYPE_METADATA = (10 << 8)
DOCTYPE_STYLE_SHEET = (11 << 8)
DOCTYPE_FONT_PAGE = (12 << 8)
DOCTYPE_TABLE = (13 << 8)
DOCTYPE_TABLE_COMPRESSED = (14 << 8)
DOCTYPE_MULTIIMAGE = (15 << 8)

## Header flags
HEADER_FLAG_CONTINUATION = 1

# DB type constants
DBTYPE_DOC = 1
DBTYPE_ZLIB = 2

## The maximum number of bytes a text document should be before being
## split into spill documents or truncated:
Max_Document_Size = 30000

## The text to add when the document gets truncated:
Document_Truncated_Text = "This document exceeded the maximum allowable size and was truncated at this point"
## The text to add as link to next spill document
Document_Next_Part_Text = "Click here for the next part"
## The text to add as link to previous spill document
Document_Previous_Part_Text = "Click here for the previous part"

## Extra size when Document_Next_Part_Text is added.
## Must be at least Header (4), Anchor (4), End_Anchor (2), plus Text.
Next_Anchor_Size = 12 + len(Document_Next_Part_Text)

_DOC_HEADER_SIZE = 8
_PARA_HEADER_SIZE = 4


def DocCompressData (data):
    """Do Doc compression of data.
    If 'in_blocks' is true, do so in in 4KB blocks"""

    if not doc_compress_function:
        raise RuntimeError("No doc compression function available!")

    res = ""
    while data:
        block = data[:4096]
        data = data[4096:]
        res = res + doc_block_compress_function (block)
    return res

def DocUncompressData (data):
    """Do Doc uncompression of data compressed in 4KB blocks"""
    if not doc_uncompress_function:
        raise RuntimeError("No doc uncompression function available!")

    return doc_uncompress_function (data)

#count = 1

def ZLibCompressData (data, username):
    #global count
    if zlib is None:
        raise RuntimeError("No ZLib support in your Python installation!")
    #sys.stderr.write('len of data for ' + str(count) + ' is ' + str(len(data)) + '\n')
    #f = open('/tmp/' + str(count) + '.z', 'w'); f.write(data); f.close(); count = count + 1
    retval = zlib.compress (data)
    #f = open('/tmp/' + str(count) + '.z', 'w'); f.write(retval); f.close(); count = count + 1
    if username:
        #sys.stderr.write('1:  retval is ' + str(retval[:min(len(retval), 50)]) + '\n')
        l = min(len(username), len(retval))
        #sys.stderr.write('l is ' + str(l) + '\n')
        retval = string.join(map(lambda x, y: chr(ord(x) ^ ord(y)), retval[:l], username[:l]), '') + retval[l:]
        #sys.stderr.write('2:  retval is ' + str(retval[:min(len(retval), 50)]) + '\n')
        #newval = string.join(map(lambda x, y: chr(ord(x) ^ ord(y)), retval[:l], username[:l]), '') + retval[l:]
        #sys.stderr.write('2:  newval is ' + str(newval[:min(len(newval), 50)]) + '\n')
        #f = open('/tmp/' + str(count) + '.z', 'w'); f.write(retval); f.close(); count = count + 1
    return retval

def ZLibUncompressData (data):
    if zlib is None:
        raise RuntimeError("No ZLib support in your Python installation!")
    return zlib.decompress (data)

CompressFunction = DocCompressData
UncompressFunction = DocUncompressData

def UseDocCompression ():
    global CompressFunction
    global UncompressFunction
    CompressFunction = DocCompressData
    UncompressFunction = DocUncompressData

def UseZLibCompression (user_id=None):
    global CompressFunction
    global UncompressFunction
    if zlib is None:
        raise RuntimeError("No ZLib support in your Python installation!")
    if user_id:
        val = struct.pack(">LLLLLLLLLL",
                          zlib.crc32(user_id * 2),
                          zlib.crc32(user_id * 3),
                          zlib.crc32(user_id * 4),
                          zlib.crc32(user_id * 5),
                          zlib.crc32(user_id * 6),
                          zlib.crc32(user_id * 7),
                          zlib.crc32(user_id * 8),
                          zlib.crc32(user_id * 9),
                          zlib.crc32(user_id * 10),
                          zlib.crc32(user_id * 11))
        sys.stderr.write("key is %d bytes:  %08x%08x%08x%08x%08x%08x%08x%08x%08x%08x\n" %
                         (len(val), zlib.crc32(user_id * 2),
                          zlib.crc32(user_id * 3),
                          zlib.crc32(user_id * 4),
                          zlib.crc32(user_id * 5),
                          zlib.crc32(user_id * 6),
                          zlib.crc32(user_id * 7),
                          zlib.crc32(user_id * 8),
                          zlib.crc32(user_id * 9),
                          zlib.crc32(user_id * 10),
                          zlib.crc32(user_id * 11)))
    else:
        val = None
    CompressFunction = lambda x, uname=val: ZLibCompressData(x, uname)
    UncompressFunction = ZLibUncompressData


##########################################################################
#
# This code is used to obtain internal identifiers for pages, and parts
# of pages (in the case of images).
#
##########################################################################

__IDCounter = 0
__IDRegistry = {}        # maps ids to PluckerDocument instances

def obtain_fresh_id():
    global __IDCounter
    __IDCounter = __IDCounter + 1
    return __IDCounter

def register_document(id, doc):
    global __IDRegistry
    __IDRegistry[id] = doc
    # sys.stderr.write("%d: %s\n" % (id, doc.get_url()))

def find_registered_document(id):
    global __IDRegistry
    return __IDRegistry.get(id)    

def display_registrations():
    global __IDRegistry
    message(0, "Plucker internal key registrations:")
    for idval, doc in __IDRegistry.items():
        message(0, "%4d:  %s" % (idval, doc))

##########################################################################
#
# A class which maintains statistics on text pages
#
##########################################################################

class PluckerTextDocumentStatistics:
    """A class to collect staticics about a Plucker text document"""

    def __init__ (self):
        self._num_paragraphs = 0
        self._num_images = 0
        self._num_databytes = 0
        self._num_compressed_databytes = 0
        self._num_paddingbytes = 0
        self._num_graphicsbytes = 0
        self._num_compressed_graphicsbytes = 0
        self._num_headerbytes = 0

    def combine_with (self, other_stat):
        self._num_paragraphs = self._num_paragraphs + other_stat._num_paragraphs
        self._num_images = self._num_images + other_stat._num_images
        self._num_databytes = self._num_databytes + other_stat._num_databytes
        self._num_compressed_databytes = self._num_compressed_databytes + other_stat._num_compressed_databytes
        self._num_paddingbytes = self._num_paddingbytes + other_stat._num_paddingbytes
        self._num_graphicsbytes = self._num_graphicsbytes + other_stat._num_graphicsbytes
        self._num_compressed_graphicsbytes = self._num_compressed_graphicsbytes + other_stat._num_compressed_graphicsbytes
        self._num_headerbytes = self._num_headerbytes + other_stat._num_headerbytes

    def get_whole_data_length (self):
        return self._num_databytes + self._num_paddingbytes + self._num_headerbytes
        
    def add_paragraphs (self, n):
        self._num_paragraphs = self._num_paragraphs + n
        
    def add_images (self, n):
        self._num_images = self._num_images + n
        
    def add_databytes (self, n):
        self._num_databytes = self._num_databytes + n
        
    def add_compressed_databytes (self, n):
        self._num_compressed_databytes = self._num_compressed_databytes + n
        
    def add_paddingbytes (self, n):
        self._num_paddingbytes = self._num_paddingbytes + n
        
    def add_graphicsbytes (self, n):
        self._num_graphicsbytes = self._num_graphicsbytes + n

    def add_compressed_graphicsbytes (self, n):
        self._num_compressed_graphicsbytes = self._num_compressed_graphicsbytes + n

    def add_headerbytes (self, n):
        self._num_headerbytes = self._num_headerbytes + n
        
    def pretty_print (self, prefix_string=""):
        full_data_length = self.get_whole_data_length ()
        if full_data_length:
            data_percent = "%d" % (100 * self._num_databytes / full_data_length)
            compress_percent = "%d" % (100 * self._num_compressed_databytes / full_data_length)
            padding_percent = "%d" % (100 * self._num_paddingbytes / full_data_length)
            header_percent = "%d" % (100 * self._num_headerbytes / full_data_length)
        else:
            data_percent = "??"
            compress_percent = "??"
            padding_percent = "??"
            header_percent = "??"
        if full_data_length:
            message(0, "%s%d bytes form %d paragraphs" % (prefix_string, full_data_length, self._num_paragraphs))
            message(0, "%s%d bytes (=%s%%) of document data, %d bytes (= %s%%) of padding, %d bytes (= %s%%) for headers" % \
                    (prefix_string, self._num_databytes, data_percent,
                     self._num_paddingbytes, padding_percent,
                     self._num_headerbytes, header_percent))
            if self._num_compressed_databytes and self._num_databytes:
                compression_ratio = "%4.1f" % (100.0 * self._num_compressed_databytes / self._num_databytes)
                message(0, "%sWith Doc compression of data: %d bytes of (compressed) data makes %s percent of original size" % \
                        (prefix_string, self._num_compressed_databytes, compression_ratio))
        if self._num_images:
            message(0, "%s%d bytes in %d images" % (prefix_string, self._num_graphicsbytes, self._num_images))
            if self._num_compressed_graphicsbytes and self._num_graphicsbytes:
                compression_ratio = "%4.1f" % (100.0 * self._num_compressed_graphicsbytes / self._num_graphicsbytes)
                message(0, "%sWith Doc compression of image data: %d bytes of (compressed) data makes %s percent of original size" % \
                        (prefix_string, self._num_compressed_graphicsbytes, compression_ratio))


class PluckerDocument:
    """A base class for all types of Plucker documents"""
    def __init__ (self, url):
        self._url = url
        self._id_tag = None
        self._tables = []
        self._bookmarks = []
        self._bookmark_ids = {}

    def add_bookmark(self, title, url):
        self._bookmarks.append((title, url))

    def get_bookmark_ids(self):
        return self._bookmark_ids

    def add_table (self, table):
        self._tables.append(table)

    def get_tables(self):
        return self._tables

    def get_url (self):
        return self._url

    def get_urls (self):
        # return a list of all urls, that are associated with this document
        # the first entry should be the same as the result of get_url()
        return [self._url]

    def get_documents(self):
        """Returns a list containing all known subdocuments.  First doc in
        list is always 'self'."""
        return [self]

    def is_text_document (self):
        return 0
            
    def is_image_document (self):
        return 0
            
    def is_mailto_document (self):
        return 0
            
    def is_special_document (self):
        return 0

    def is_table_document (self):
        return 0
            
    def is_multiimage_document (self):
        return 0
        
    def register_doc (self, idtag):
        register_document(idtag, self)

    def get_size_hint (self):

        """Returns a 'gale-force' number between 1 and 10 to indicate the 'complexity'
        of the document, sort of a cross between size and complexity.  Should correlate
        on a log scale with the time it takes to render the parsed document into a Palm
        binary record"""

        return 0

    def __repr__(self):
        return "<%s '%s' at %s>" % (self.__class__.__name__, self._url, hash(self))


CMD_TEXT = 1
CMD_IMAGE = 2
CMD_ANCHOR_START = 3
CMD_ANCHOR_END = 4
CMD_ITALICS_START = 5
CMD_ITALICS_END = 6
CMD_NEWLINE = 7
CMD_SET_MARGIN = 8
CMD_SET_ALIGNMENT = 9
CMD_HR = 10
CMD_SET_STYLE = 11
CMD_ULINE_START = 12
CMD_ULINE_END = 13
CMD_STRIKE_START = 14
CMD_STRIKE_END = 15
CMD_SET_FORECOLOR = 16 
CMD_UNICODE_CHAR = 17
CMD_TABLE = 18


CMD_NAMES = ("TEXT", "IMAGE", "ANCHOR_START", "ANCHOR_END",
             "ITALICS_START", "ITALICS_END", "NEWLINE", "SET_MARGIN",
             "SET_ALIGNMENT", "HR", "SET_STYLE", "ULINE_START", "ULINE_END",
             "STRIKE_START", "STRIKE_END", "SET_FORECOLOR", "UNICODE_CHAR",
             "TABLE")

# An int to ascii conversion, used by forecolor if unpacking a database
def itoa (n, base = 2):
    if type (n) != types.IntType:
        raise TypeError('First arg should be an integer')
    if (type (base) != types.IntType) or not (2<=base<=36):
        raise TypeError('Second arg should be an integer between 2 and 36')
    output = []
    pos_n = abs (n)
    while pos_n:
        lowest_digit = pos_n % base
        if lowest_digit>=10:
            output.append( chr( ord('a')+lowest_digit-10 ) )
        else:
            output.append( str(lowest_digit) )
        pos_n = (pos_n - lowest_digit) / base
    output.reverse ()
    if n < 0:
        output.insert (0, '-')
    return string.join (output, '')


def _cmd_name(index):
    if (index <= len(CMD_NAMES)):
        return CMD_NAMES[index-1]
    else:
        return str(index)


class PluckerTextParagraph:
    """A class to contain information about one paragraph of a Plucker
    Text document."""

    
    
    def __init__ (self, extra_space=0):
        self._data = ""
        self._items = []
        self._extra_space = extra_space
        self._names = []
        self._document_refs = []
        self._image_refs = []
        self._estimated_length = _PARA_HEADER_SIZE
        self._in_anchor = 0
        self._refs_cleared = 0


    def set_extra_spacing (self, extra_space):
        extra_space = int (extra_space)
        assert 0 <= extra_space <= 7, \
               "Extra space for a paragraph needs to be between 0 and 7 but is %s" % repr (extra_space)
        self._extra_space = extra_space


    def get_extra_spacing (self):
        return self._extra_space


    def get_estimated_length (self):
        return self._estimated_length


    def get_names (self):
        """Return the list of names of this paragraph (i.e. named anchors
        in this paragraph)"""
        return self._names


    def get_external_references (self):
        assert(not self._refs_cleared)
        return (self._document_refs, self._image_refs)

    
    def clear_external_references (self):
        # free up ref to dicts holding external refs
        self._document_refs = None
        self._image_refs = None
        self._refs_cleared = 1

    
    def padding_needed (self):
        """Calculate how much passing is needed for this paragraph.
        (Old format of plucker docs)"""
        padding = 4 - (len (self._data) % 4)
        if padding == 4:
            padding = 0
        return padding


    def _add (self, something):
        """Privete function: Add some entry to internal list of
        parsed things"""
        message(4, "Adding element %s: %s", _cmd_name(something[0]), something[1])
        self._items.append (something)


    def add_name (self, name):
        self._names.append (name)

        
    def add_text (self, text):
        self._add ((CMD_TEXT, text))
        self._estimated_length = self._estimated_length + len (text)


    def add_unicode_char (self, char, text_equivalent):
        self._add ((CMD_UNICODE_CHAR, (char, text_equivalent)))
        self._estimated_length = self._estimated_length + 3 + len(text_equivalent) + (((char > 0xFFFF) and 4) or 2)

    def add_table (self, dict_of_items):
        self._add ((CMD_TABLE, dict_of_items))
        self._estimated_length = self._estimated_length + 4


    def add_anchor_start (self, dict_of_items):
        self._add ((CMD_ANCHOR_START, dict_of_items))
        if dict_of_items.has_key ('href'):
            self._document_refs.append ((dict_of_items['href'], dict_of_items))
        # the overhead might just be 4, but is generally 6.  Err on the side
        # of lavishness by using 6.
        self._estimated_length = self._estimated_length + 6


    def add_anchor_end (self):
        self._add ((CMD_ANCHOR_END, None))
        self._estimated_length = self._estimated_length + 2


    def add_italics_start (self):
        self._add ((CMD_ITALICS_START, None))
        self._estimated_length = self._estimated_length + 2


    def add_italics_end (self):
        self._add ((CMD_ITALICS_END, None))
        self._estimated_length = self._estimated_length + 2


    def add_underline_start (self):
        self._add ((CMD_ULINE_START, None))
        self._estimated_length = self._estimated_length + 2


    def add_underline_end (self):
        self._add ((CMD_ULINE_END, None))
        self._estimated_length = self._estimated_length + 2


    def add_strike_start (self):
        self._add ((CMD_STRIKE_START, None))
        self._estimated_length = self._estimated_length + 2


    def add_strike_end (self):
        self._add ((CMD_STRIKE_END, None))
        self._estimated_length = self._estimated_length + 2
        
    def add_image_reference (self, dict_of_items):
        self._add ((CMD_IMAGE, dict_of_items))
        if dict_of_items.has_key ('src'):
            self._image_refs.append ((dict_of_items['src'], dict_of_items))
        # either 4 or 6 bytes, depending on whether there's a "big" version
        self._estimated_length = self._estimated_length + 6
       
    def add_newline (self):
        self._add ((CMD_NEWLINE, None))
        self._estimated_length = self._estimated_length + 2

    def add_set_margin (self, left, right):
        self._add ((CMD_SET_MARGIN, (left, right)))
        self._estimated_length = self._estimated_length + 4

    def add_set_alignment (self, value):
        assert 0 <= value and value <= 3, "Alignment must be >=0 and <=3 but is %s" % repr(value)
        self._add ((CMD_SET_ALIGNMENT, value))
        self._estimated_length = self._estimated_length + 3

    def add_set_forecolor (self, value):
        self._add ((CMD_SET_FORECOLOR, value))
        self._estimated_length = self._estimated_length + 5
          
    def add_hr (self, height=0, width=0, width_percent=0):
        self._add ((CMD_HR, (height, width, width_percent)))
        self._estimated_length = self._estimated_length + 5

    def add_style_change (self, new_stilenum):
        self._add ((CMD_SET_STYLE, new_stilenum))
        self._estimated_length = self._estimated_length + 3

    def cleanup (self):
        """Clean up the paragraph.  Should only be called when nothing
        more is to be changed in this paragraph"""

        def is_formatting_code (item):
            type = item[0]
            return (type == CMD_SET_STYLE or
                    type == CMD_SET_FORECOLOR or
                    type == CMD_SET_ALIGNMENT or
                    type == CMD_SET_MARGIN or
                    type == CMD_SET_STYLE or
                    type == CMD_ITALICS_END or
                    type == CMD_ITALICS_START or
                    type == CMD_NEWLINE or
                    type == CMD_ULINE_END or
                    type == CMD_ULINE_START or
                    type == CMD_STRIKE_END or
                    type == CMD_STRIKE_START)

        # first, remove formatting codes at end of paragraph
        while (self._items and is_formatting_code(self._items[-1])):
            message(4, "Elided %s at end of paragraph", self._items[-1][0])
            del self._items[-1]

        idx = 0
        items = self._items
        while 1:
            idx = max (0, idx) # make legal in case we set it too far back
            if idx >= len (items):
                # done!
                break
            this = items[idx][0]
            if idx+1 >= len (items):
                next = None
            else:
                next = items[idx+1][0]
            if this == CMD_SET_STYLE:
                if next is None:
                    # at end
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_ANCHOR_END:
                    items[idx], items[idx+1] = items[idx+1], items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_SET_STYLE:
                    del items[idx]
                    continue

            elif this == CMD_SET_ALIGNMENT:
                if next is None:
                    # at end
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_SET_ALIGNMENT:
                    del items[idx]
                    continue

            elif this == CMD_SET_MARGIN:
                if next is None:
                    # at end
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_SET_MARGIN:
                    del items[idx]
                    continue

            elif this == CMD_ITALICS_START:
                if next is None:
                    # at end
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_ITALICS_END:
                    del items[idx+1]
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_ITALICS_START:
                    del items[idx]
                    continue
                elif next == CMD_SET_STYLE:
                    items[idx], items[idx+1] = items[idx+1], items[idx]
                    idx = idx - 1
                    continue
            elif this == CMD_ITALICS_END:
                if next is None:
                    # at end
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_ITALICS_START:
                    del items[idx+1]
                    del items[idx]
                    idx = idx - 1
                    continue

            elif this == CMD_ULINE_START:
                if next is None:
                    # at end
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_ULINE_END:
                    del items[idx+1]
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_ULINE_START:
                    del items[idx]
                    continue
                elif next == CMD_SET_STYLE:
                    items[idx], items[idx+1] = items[idx+1], items[idx]
                    idx = idx - 1
                    continue
            elif this == CMD_ULINE_END:
                if next is None:
                    # at end
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_ULINE_START:
                    del items[idx+1]
                    del items[idx]
                    idx = idx - 1
                    continue
                    
            elif this == CMD_SET_FORECOLOR:
                if next is None:
                    # at end
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_SET_FORECOLOR:
                    del items[idx]
                    continue
                elif next == CMD_SET_STYLE:
                    items[idx], items[idx+1] = items[idx+1], items[idx] #Robert: how many here?
                    idx = idx - 1
                    continue
                    
            elif this == CMD_STRIKE_START:
                if next is None:
                    # at end
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_STRIKE_END:
                    del items[idx+1]
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_STRIKE_START:
                    del items[idx]
                    continue
                elif next == CMD_SET_STYLE:
                    items[idx], items[idx+1] = items[idx+1], items[idx]
                    idx = idx - 1
                    continue
            elif this == CMD_STRIKE_END:
                if next is None:
                    # at end
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_STRIKE_START:
                    del items[idx+1]
                    del items[idx]
                    idx = idx - 1
                    continue
                    
            elif this == CMD_ANCHOR_START:
                if next == CMD_ANCHOR_END:
                    del items[idx+1]
                    del items[idx]
                    idx = idx - 1
                    continue
                elif next == CMD_SET_STYLE or next == CMD_ITALICS_START:
                    items[idx], items[idx+1] = items[idx+1], items[idx]
                    idx = idx - 1
                    continue
            elif this == CMD_ANCHOR_END:
                pass
            elif this == CMD_IMAGE:
                pass
            elif this == CMD_TABLE:
                pass
            elif this == CMD_TEXT:
                if idx == 0:
                    text = string.lstrip (items[idx][1])
                    items[idx] = (CMD_TEXT, text)
                text = items[idx][1]
                if not text:
                    del items[idx]
                    continue
            elif this == CMD_UNICODE_CHAR:
                pass

            # now proceed
            idx = idx + 1
        # end of loop
        self._items = items

        

    def resolve_ids (self, resolver):
        """Resolve document ids for anchor starts and image references.
        This calls resolver_function for each such item with the associated
        dictionary and expects an integer as return as the id to use"""
        # sys.stderr.write("resolving IDs for para " + str(self) + "\n")
        for (tag, value) in self._items:

            if tag == CMD_ANCHOR_START:

                if value.has_key('_plucker_id_tag'):
                    doc = find_registered_document(value['_plucker_id_tag'])
                    if doc:
                        oldurl, tag = urllib.splittag(value['href'])
                        if tag:
                            newurl = doc.get_url() + '#' + tag
                        else:
                            newurl = doc.get_url()
                    else:
                        newurl = value['href']
                    pidval = doc and resolver.get_or_add(newurl)
                    message(3, "  Resolved _plucker_id_tag %d to %s  (%s)\n" % (value['_plucker_id_tag'], pidval, value['href']))
                else:
                    pidval = None
                    message(3, "  *** No _plucker_id_tag for %s\n" % value['href'])
                value['recordnumber'] = pidval or resolver.get_or_add(value['href'])
                message(3, "    -> %s to docid %s\n" % (value['href'], value['recordnumber']))

            elif tag == CMD_IMAGE:

                if value.has_key('_plucker_id_tag_inlineimage'):
                    doc = find_registered_document(value['_plucker_id_tag_inlineimage'])
                    value['recordnumber'] = doc and resolver.get_or_add(doc)
                elif value.has_key('src'):
                    value['recordnumber'] = resolver.get_or_add(value['src'])
                if value.has_key('_plucker_id_tag_outoflineimage'):
                    doc = find_registered_document(value['_plucker_id_tag_outoflineimage'])
                    if doc:
                        value['big_id'] = resolver.get_or_add(doc)
                #sys.stderr.write("Resolved %s to %s\n" % (value, value['recordnumber']))

            elif tag == CMD_TABLE:

                if value.has_key('_plucker_id_tag'):
                    doc = find_registered_document(value['_plucker_id_tag'])
                value['recordnumber'] = doc and resolver.get_or_add(value['href'])

    def old_resolve_ids (self, resolver_function):
        """Resolve document ids for anchor starts and image references.
        This calls resolver_function for each such item with the associated
        dictionary and expects an integer as return as the id to use"""
        for (tag, value) in self._items:
            if tag == CMD_ANCHOR_START:
                value['recordnumber'] = resolver_function (value, 0)
            elif tag == CMD_IMAGE:
                value['recordnumber'] = resolver_function (value, 1)
                big_url = value['src']
                fake_dict = {'src': big_url + "_BIG"}
                big_id = resolver_function (fake_dict, 1)
                if big_id == None:
                    big_id = 0
                value['big_id'] = big_id


    def _dump_record_body (self, allow_fragments, allow_newstuff):
        """(Re-)Assemble the binary representation of just the body of this paragraph.
        (returns tupel (data, padding)"""

        # continous appending t oa string is inefficient (uses O(n)
        # time), so we collect in all in an array and concat all in
        # the end
        data = [] 
        
        for (tag, value) in self._items:
            if tag == CMD_TEXT:
                data.append (value)
            elif tag == CMD_ANCHOR_START:
                # sys.stderr.write("href=%s, id=%s\n" % (value.get('href'), value.get('recordnumber')))
                assert self._in_anchor == 0, "Nested anchors"
                assert value.has_key ('recordnumber'), "Anchor start information lacks document id: " + str(value) + ", self=" + str(self)
                id = value['recordnumber']
                if type (id) == types.TupleType:
                    assert len (id) == 2, "Recordnumber must be None, record-ID, or (record-ID, paragraph #) tuple, but is %s" % repr (id)
                    if allow_fragments:
                        data.append (struct.pack (">BBHH", 0, 0x0c, id[0], id[1]))
                    else:
                        data.append (struct.pack (">BBH", 0, 0x0a, id[0]))
                    self._in_anchor = 1
                elif type (id) == types.IntType:
                    data.append (struct.pack (">BBH", 0, 0x0a, id))
                    self._in_anchor = 1
                elif id is None:
                    # allowed for unfound links
                    pass
                else:
                    raise ValueError("%s:  Recordnumber must be None, record-ID or (record-ID,"
                                     " paragraph #) tuple, but is %s" % (self, repr (id)))
            elif tag == CMD_ANCHOR_END:
                if self._in_anchor:
                    data.append (struct.pack (">BB", 0, 0x08))
                    self._in_anchor = 0
            elif tag == CMD_SET_STYLE:
                data.append (struct.pack (">BBB", 0, 0x11, value))
            elif tag == CMD_IMAGE:
                assert value.has_key ('recordnumber'), "Image reference information lacks document id"
                id = value['recordnumber']
                if id is None:
                    if value.has_key ('alt'):
                        data.append (value['alt'])
                    else:
                        message(3, "No recordnumber for image %s", value)
                        data.append ('[img]')
                else:
                    big_id = value.get('big_id')
                    if big_id:
                        data.append (struct.pack (">BBHH", 0, 0x5c, big_id, id,))
                    else:
                        data.append (struct.pack (">BBH", 0, 0x1a, id))
            elif tag == CMD_SET_MARGIN:
                if allow_newstuff:
                    data.append (struct.pack (">BBBB", 0, 0x22, value[0], value[1]))
                else:
                    # ignored :-(
                    pass
            elif tag == CMD_SET_ALIGNMENT:
                if allow_newstuff:
                    data.append (struct.pack (">BBB", 0, 0x29, value))
                else:
                    # ignored :-(
                    pass
            elif tag == CMD_HR:
                if allow_newstuff:
                    (height, width, perc_width) = value
                    data.append (struct.pack (">BBBBB", 0, 0x33, height, width, perc_width))
                else:
                    # ignored :-(
                    pass
            elif tag == CMD_NEWLINE:
                if not allow_newstuff:
                    raise RuntimeError("Text paragraph contains a newline but undump may not use it")
                data.append (struct.pack (">BB", 0, 0x38))
            elif tag == CMD_ITALICS_START:
                if allow_newstuff:
                    data.append (struct.pack (">BB", 0, 0x40))
                else:
                    # ignored :-(
                    pass
            elif tag == CMD_ITALICS_END:
                if allow_newstuff:
                    data.append (struct.pack (">BB", 0, 0x48))
                else:
                    # ignored :-(
                    pass
            elif tag == CMD_ULINE_START:
                if allow_newstuff:
                    data.append (struct.pack (">BB", 0, 0x60))
                else:
                    # ignored :-(
                    pass
            elif tag == CMD_ULINE_END:
                if allow_newstuff:
                    data.append (struct.pack (">BB", 0, 0x68))
                else:
                    # ignored :-(
                    pass
            elif tag == CMD_STRIKE_START:
                if allow_newstuff:
                    data.append (struct.pack (">BB", 0, 0x70))
                else:
                    # ignored :-(
                    pass
            elif tag == CMD_STRIKE_END:
                if allow_newstuff:
                    data.append (struct.pack (">BB", 0, 0x78))
                else:
                    # ignored :-(
                    pass
            elif tag == CMD_SET_FORECOLOR:
                if allow_newstuff:
                    rgb = value
                    hexr = string.atoi (rgb[0:2], 16) # extract first 2 characters, then convert to hex
                    hexg = string.atoi (rgb[2:4], 16) # extract middle 2 characters, then convert to hex
                    hexb = string.atoi (rgb[4:6], 16) # extract last 2 characters,, then convert to hex
                    data.append (struct.pack (">BBBBB", 0, 0x53, hexr, hexg, hexb)) 
                else:
                    # ignored :-(   
                    pass
            elif tag == CMD_UNICODE_CHAR:
                if allow_newstuff:
                    if value[0] > 0xFFFF:
                        data.append(struct.pack (">BBBL", 0, 0x85, len(value[1]), value[0]))
                    else:
                        data.append(struct.pack (">BBBH", 0, 0x83, len(value[1]), value[0]))
                    data.append(value[1])
                else:
                    pass
            elif tag == CMD_TABLE:
                if allow_newstuff:
                    id = value['recordnumber']
                    data.append (struct.pack (">BBH", 0, 0x92, id))
                else:
                    pass
                    
        data = string.join (data, "")

        padding = 4 - (len (data) % 4)
        if padding == 4:
            padding = 0
        return (data, "X"*padding)


    def dump_record (self):
        """(Re-)Assemble the binary representation of this paragraph.
        Returns a tupel of (header, body) data"""

        (data, padding) = self._dump_record_body (allow_fragments=1, allow_newstuff=1)
        
        space = self._extra_space & 0x07
        header = struct.pack (">HH", len(data), space)

        return (header, data)

        
    def undump_record (self, text, verbose=0):
        """Dissassemble just one paragraph"""
        self._data = text
        while 1:
            res = string.split(text, "\0", 1)
            if res[0]:
                # some text before a function or before the end
                message("  Text %s" % repr(res[0]))
                self.add_text (res[0])
            if len(res) == 1:
                # just text and no function was found, i.e. we are now done
                break
            else:
                # function found
                rest_text = res[1]
                assert len (rest_text)>= 1, "No function data found after function marker"
                function_code = rest_text[0]
                if function_code == "\012":
                    # anchor
                    (id,) = struct.unpack(">H", rest_text[1:3])
                    text = rest_text[3:]
                    message("  Anchor start for document #%d" % id)
                    self.add_anchor_start ({'recordnumber': id})
                elif function_code == "\014":
                    # anchor with fragment part
                    (id, fragmentid) = struct.unpack(">HH", rest_text[1:5])
                    text = rest_text[5:]
                    if verbose:
                        print "  Anchor start for document #(%d, %d)" % (id, fragmentid)
                    self.add_anchor_start ({'recordnumber': (id, fragmentid)})
                elif function_code == "\010":
                    # anchor end
                    text = rest_text[1:]
                    if verbose:
                        print "  End anchor"
                    self.add_anchor_end ()
                elif function_code == "\021":
                    # set style
                    (id,) = struct.unpack(">B", rest_text[1:2])
                    text = rest_text[2:]
                    assert id>=0 and id<=7, "Illegal style code %d found" % id
                    if verbose:
                        if id==0:
                            idcode = "normal"
                        elif id==7:
                            idcode = "bold"
                        else:
                            idcode = "header %d" % id
                        print "  Style code %d (%s)" % (id, idcode)
                    self.add_style_change (id)
                elif function_code == "\032":
                    # image
                    (id,) = struct.unpack(">H", rest_text[1:3])
                    text = rest_text[3:]
                    if verbose:
                        print "  Reference to image #%d" % id
                    self.add_image_reference ({'recordnumber': id})
                elif function_code == "\042":
                    # set margin
                    (left, right) = struct.unpack(">BB", rest_text[1:3])
                    text = rest_text[3:]
                    self.add_set_margin (left, right)
                    if verbose:
                        print "  Set Margins  %d, %d" % (left, right)
                elif function_code == "\051":
                    # alignment
                    (code,) = struct.unpack(">B", rest_text[1:2])
                    text = rest_text[2:]
                    self.add_set_alignment (code)
                    if verbose:
                        if code == 0:
                            al = "left"
                        elif code == 1:
                            al = "right"
                        elif code == 2:
                            al = "center"
                        else:
                            al = "???"
                        print "  Alignment %s" % al
                elif function_code == "\063":
                    # alignment
                    (height, width, perc_width) = struct.unpack(">BBB", rest_text[1:4])
                    text = rest_text[4:]
                    self.add_hr (height, width, perc_width)
                    if verbose:
                        print "  Horizontal rule: height: %d, width: %d, %%-width: %d" % (height, width, perc_width)
                elif function_code == "\070":
                    # newline
                    self.add_newline ()
                    text = rest_text[1:]
                    if verbose:
                        print "  NewLine"
                elif function_code == "\100":
                    # italics start
                    self.add_italics_start ()
                    text = rest_text[1:]
                    if verbose:
                        print "  Italics start"
                elif function_code == "\110":
                    # italics end
                    self.add_italics_end ()
                    text = rest_text[1:]
                    if verbose:
                        print "  Italics end"
                elif function_code == "\140":
                    # underline start
                    self.add_underline_start ()
                    text = rest_text[1:]
                    if verbose:
                        print "  Underline start"
                elif function_code == "\150":
                    # underline end
                    self.add_underline_end ()
                    text = rest_text[1:]
                    if verbose:
                        print "  Underline end"
                elif function_code == "\160":
                    # strikethrough start
                    self.add_strike_start ()
                    text = rest_text[1:]
                    if verbose:
                        print "  Strikethrough start"
                elif function_code == "\170":
                    # strikethrough end
                    self.add_strike_end ()
                    text = rest_text[1:]
                    if verbose:
                        print "  Strikethrough end"
                elif function_code == "\123":
                    # set forecolor
                    (r, g, b) = struct.unpack(">BBB", rest_text[1:4])
                    text = rest_text[4:]
                    # add_set_forecolor function expects a string of the 3 hex values
                    # so convert them to strings with custom itoa function then concatenate
                    rgb = itoa (r, 16) + itoa (g, 16) + itoa (b, 16)
                    self.add_set_forecolor (rgb)
                    if verbose:
                        print "  ForeColor start: red: %d, green: %d, blue: %d" % (r, g, b)
                else:
                    raise AssertionError("Unknown function code %s (%d) found" % (repr(function_code), ord(function_code)))



class PluckerTextDocument (PluckerDocument):
    """A class to contain information about one Plucker Text document."""
    
    global seamless_fragments
    global link_fragments

    def __init__ (self, url):
        PluckerDocument.__init__(self, url)

        # a sequence of subdocuments chained off this one
        self._documents = [self]

        # a sequence of paragraphs contained in this document
        self._paragraphs = []

        # _current_document points to the last document in _documents, and _current_document_length
        # is a running estimate of the length of _current_document
        self._current_document = self
        self._current_document_length = 0

        self._stats = PluckerTextDocumentStatistics ()
        self._charset = None

        self._header_flags = 0
        

    ########################
    ## overridden methods...
    ########################

    def get_urls (self):
        return [self.get_url()] + map(lambda x: x.get_url(), self._documents[1:])


    def is_text_document (self):
        return 1


    def get_size_hint (self):
        import math
        return math.log(self._current_document_length * len(self._paragraphs) * len(self._documents))


    def get_documents (self):
        """Returns a list of sub-documents, including the main one"""
        return self._documents


    ########################
    ## new methods...
    ########################
    
    def set_continuation (self):
        self._header_flags = self._header_flags | HEADER_FLAG_CONTINUATION

    def set_charset (self, charset):
        self._charset = charset
        for doc in self._documents[1:]:
            doc.set_charset(charset)

    def get_charset (self):
        # if the document URL is via HTTP or HTTPS, and there is no charset,
        # we apply either the user-specified default charset, or if there is
        # none, we use ISO-8859-1, as specified in the HTTP spec.
        return self._charset

    def _get_part_url (self, part_no):
        """Return the url for the part 'part_no' of this document"""
        if part_no == 0:
            return self.get_url ()
        else:
            return "plucker:/~parts~/" + \
                   urllib.quote(self.get_url (), "") + \
                   ("/%d" % part_no)


    def resolve_ids (self, resolver):
        """Resolve references to external documents"""
        # resolve references in paragraphs internally
        #sys.stderr.write("resolving IDs for doc " + str(self) + ", " + self.get_url() + "\n")
        map (lambda x, r=resolver: r.get_or_add(x), self._documents)
        # First, do other documents chained from this one, if any
        map (lambda x, r=resolver: x.resolve_ids(r), self._documents[1:])
        # now, do paragraphs in self
        map (lambda p, r=resolver: p.resolve_ids(r), self._paragraphs)

        # now, do the bookmarks
        for (title, url) in self._bookmarks:
            if not self._bookmark_ids.has_key(title):
                pidval = resolver.get_or_add(url)
                if type(pidval) == types.TupleType and len(pidval) == 2:
                    self._bookmark_ids[title] = pidval
                elif type(pidval) == types.IntType:
                    self._bookmark_ids[title] = (pidval, 0)





    def _build_document (self, id, para_count, headers, bodies):
        """Build a (new format) dump of a document from headers and bodies."""

        assert _DOC_HEADER_SIZE==8

        headers = string.join (headers, "")
        bodies = string.join (bodies, "")

        compressed_bodies = CompressFunction (bodies)
        if len (compressed_bodies) < len (bodies):
            shipped_bodies = compressed_bodies
            content_type = DOCTYPE_HTML_COMPRESSED
        else:
            shipped_bodies = bodies
            content_type = DOCTYPE_HTML

        # sys.stderr.write("uid = " + str(id) + ", para_count = " + str(para_count) + ", len(bodies) = " + str(len(bodies)) + ", len(shipped_bodies) = " + str(len(shipped_bodies)) + ", content_type = " + str(content_type) + "\n")

        header = struct.pack (">HHHBB",
                              id,                 # uid
                              para_count,         # number of paragraphs
                              len (bodies),       # size
                              content_type >> 8,  # content type
                              self._header_flags) # flags

        return header + headers + shipped_bodies
        

    def dump_record (self, id):
        """(Re-)Assemble the binary representation of this text document"""

        # sys.stderr.write("dump record:  %3d %s  %s\n" % (id, self, self.get_url()))
        headers = []
        bodies = []
        size_sum = 0
        para_count = 0
        for para in self._paragraphs:
            para.cleanup ()
            (header, body) = para.dump_record()
            para_count = para_count + 1
            assert len (header) == _PARA_HEADER_SIZE, "Length of header (%d) is not %d" % (len (header), _PARA_HEADER_SIZE)
            if size_sum + len (header) + len (body) > Max_Document_Size + Next_Anchor_Size:
                error("Truncating %s\n  size_sum = %d, len(header) = %d, len(body) = %d, total = %d\n",
                      self.get_url(), size_sum, len(header), len(body), size_sum+len(header)+len(body))
                para = PluckerTextParagraph (5)
                para.add_text (Document_Truncated_Text)
                (header, body) = para.dump_record()
                headers.append (header)
                bodies.append (body)
                break
                    
            headers.append (header)
            bodies.append (body)
            size_sum = size_sum + len (header) + len (body)

        return self._build_document (id, para_count, headers, bodies)

    
    def dump_record_with_splits (self, resolver):
        """(Re-)Assemble the binary representation of this text document"""

        # make sure all the subdocs have ids
        self.resolve_ids(resolver)
        results = []
        for document in self._documents:
            id = resolver.get_or_add(document)
            url = document.get_url()
            dump = document.dump_record(id)
            results.append((url, id, dump))
        # sys.stderr.write("results are " + str(results) + "\n")
        return results

    
    def undump_record (self, data, verbose=0):
        """Given the contents of a cache file (as e.g. generated by the awk parser),
        disassemble the contents and show it to the user"""

        assert len (data) > _DOC_HEADER_SIZE, \
               "Length of text document should be >%d but is %d" % (_DOC_HEADER_SIZE, len (data))
        header_data = data[:_DOC_HEADER_SIZE]

        if _DOC_HEADER_SIZE == 8:
            (uid, paragraphs, data_size, content_type, _self._header_flags) = struct.unpack (">HHHBB", header_data)
            content_type = content_type << 8
        if content_type == DOCTYPE_HTML:
            typetext = "uncompressed text"
        elif content_type == DOCTYPE_HTML_COMPRESSED:
            typetext = "compressed text"
        else:
            typetext = "?? illegal content type for text document"
        if verbose:
            print "Text Document header:\n" \
                  "\tnumber of paragraphs: %d" \
                  "\tbody size: %d" \
                  "\ttype: %d (%s)" % \
                  (paragraphs, data_size, content_type, typetext)
        ## Do sanity check on header values
        assert content_type==DOCTYPE_HTML or content_type==DOCTYPE_HTML_COMPRESSED, \
               "Content type of text document is %d which is not a legal value" % content_type

        self._stats.add_headerbytes (_DOC_HEADER_SIZE)

        header_size = paragraphs * _PARA_HEADER_SIZE
        rest_data = data[_DOC_HEADER_SIZE:]
        headers = rest_data[:header_size]
        if content_type == DOCTYPE_HTML_COMPRESSED:
            # compressed HTML
            compressed_body = rest_data[header_size:]
            body = UncompressFunction (compressed_body)
        else:
            # uncompressed HTML
            body = rest_data[header_size:]
            if doc_compress_function:
                compressed_body = CompressFunction (body)
            else:
                compressed_body = ""
        assert data_size == len (body), \
               "Text document's header says data size is %d but it is %d" % (data_size, len (body))
        
        offset = 0
        for i in range (paragraphs):
            para_header = headers[:_PARA_HEADER_SIZE]
            headers = headers[_PARA_HEADER_SIZE:]
            (length, attr) = struct.unpack (">HH", para_header)

            ## check all the header values for sanity:
            # only lower 3 bits are used in attribute:
            assert 0 <= attr <= 15, \
                   "Text paragraph's attribute is %d -- must be between 0 and 15" % attr
            
            contents = body[offset:offset+length]
            offset = offset + length

            if not attr:
                attr_text = "none"
            else:
                space = attr & 3
                attr_text = "%d units of space above " % space
                if attr & 4:
                    attr_text = attr_text + ", extra spacing"
            if verbose:
                print "Paragraph: length %d\n\t   attributes: %d (%s)\n" \
                      "\t   offset: %d (calculated)" % \
                      (length, attr, attr_text, offset)

            paragraph = PluckerTextParagraph (extra_space=((attr&4) == 4))
            paragraph.undump_record (contents, verbose=verbose)

            self._documents[0].append (paragraph)

            self._stats.add_paragraphs (1)
            self._stats.add_headerbytes (_PARA_HEADER_SIZE)
            self._stats.add_databytes (len (contents))

        self._stats.add_compressed_databytes (len (compressed_body))

        # All header data should be used up:
        assert len (headers) == 0, \
               "After parsing all text document paragraph's headers, '%s' header info remains" % \
               repr (headers)
        # At the end, all data should be used up:
        assert offset == len (body), \
               "Adding all text document's paragraph's lengths (%d) is not the length of the data (%d)" % \
               (offset, len (body))


    def add_paragraph (self, par, force=0):
        """Add a new paragraph to this document.
        It gets appended.  This should be used to incrementally
        contruct a document"""
        #sys.stderr.write("add_paragraph:  %8d, %8d, %8d, %3d\n" % (hash(self), hash(par), self._current_document_length, len(self._documents)))
        par_length = par.get_estimated_length ()
        if not force and (par_length + self._current_document_length > Max_Document_Size):
            # a split into a new sub document is needed
            this_url = self._get_part_url (len (self._documents) - 1)
            next_url = self._get_part_url (len (self._documents))
            if PluckerTextDocument.link_fragments:
                tmp_para = PluckerTextParagraph (5)
                tmp_para.add_anchor_start ({'href': next_url})
                tmp_para.add_text (Document_Next_Part_Text)
                tmp_para.add_anchor_end ()
                #sys.stderr.write("new temp para, id is %d, est size is %d\n" % (hash(tmp_para), tmp_para.get_estimated_length()))
                self._current_document.add_paragraph(tmp_para, 1)
            if PluckerTextDocument.seamless_fragments:
                self._current_document.set_continuation()
            # start new document:
            self._current_document = PluckerTextDocument(next_url)
            self._documents.append(self._current_document)
            if PluckerTextDocument.link_fragments:
                tmp_para = PluckerTextParagraph (0)
                tmp_para.add_anchor_start ({'href': this_url})
                tmp_para.add_text (Document_Previous_Part_Text)
                tmp_para.add_anchor_end ()
                self._current_document_length = tmp_para.get_estimated_length ()
                self._current_document.add_paragraph (tmp_para, 1)
            else:
                self._current_document_length = 0
        if (self._current_document == self):
            self._paragraphs.append(par)
        else:
            self._current_document.add_paragraph (par, 1)
        self._current_document_length = self._current_document_length + par_length


    def get_name_map (self):
        """ Returns the map that states what paragraph number a named anchor
        is located in.
        Should be useful to build a resolver"""

        result = {}

        # do names local to this document's direct paragraphs
        url = self.get_url()
        for par_id in range (len(self._paragraphs)):
            for name in self._paragraphs[par_id].get_names():
                result[name] = (url, par_id)

        # do other documents chained off this one, and return results
        #   In this lambda, x is the "result", the accumulating hash table, and y is a sub-doc.
        #   The or clause is because {}.update() doesn't return a value, so we explicitly return x.
        return reduce(lambda x, y: x.update(y.get_name_map()) or x, self._documents[1:], result)
    

    def get_external_references (self):
        hrefs = []
        images = []
        for para in self._paragraphs:
            (h, i) = para.get_external_references()
            hrefs = hrefs + h
            images = images + i
        for doc in self._documents[1:]:
            subhrefs, subimages = doc.get_external_references()
            hrefs = hrefs + subhrefs
            images = images + subimages
        return (hrefs, images)


    def clear_external_references (self):
        for para in self._paragraphs:
            para.clear_external_references()
        for doc in self._documents[1:]:
            doc.clear_external_references()


    def get_stats (self):
        return self._stats



class PluckerImageDocument (PluckerDocument):
    """A class to contain information about one Plucker Image document (PalmOS image format)."""

    def __init__ (self, url, config=None):
        PluckerDocument.__init__ (self, url)
        self._data = ""
        self._stats = PluckerTextDocumentStatistics ()
        self._config = config
        self._size_data = None
        self._related_images = []

    def is_image_document (self):
        return 1

    def get_size_hint (self):
        import math
        return math.log(len(self._data))

    def get_related_images (self):
        return self._related_images

    def add_related_image (self, url, attributes):
        self._related_images.append((url, attributes,))

    def set_data(self, data):
        """Set the image data directly.
        DATA must be a complete image in PalmOS image format.
        Useful for building a new document"""
        self._data=data


    def print_summary (self, prefix):
        """Give some information about this bitmap."""
        assert len (self._data) > 16, "Image data is not a bitmap!"
        bitmap_header = self._data[:16]
        bitmap_data = self._data[16:]
        (width, height, row_bytes, flags, pixel_size, version, \
         next_depth_offset, reserved1, reserved2) = \
                struct.unpack (">HHHHBBHHH", bitmap_header)
        flags_text="%d = 0x%x" % (flags, flags)
        if flags:
            flags_text = flags_text + " ("
            if flags & 1:
                flags_text = flags_text = "compressed  "
            if flags & 2:
                flags_text = flags_text = "has_color_table"
            flags_text = flags_text + ")"
            
        print "%sBitmap: %d x %d" % (prefix, width, height)
        print "%s        flags: %s" % (prefix, flags_text)
        print "%s        version %d; %d bits per pixel,  %d bytes per row" % \
              (prefix, version, pixel_size, row_bytes)
        print "%s        next depth offset: %d" % (prefix, next_depth_offset)
        print "%s        reserved values: %d, %d" % (prefix, reserved1, reserved2)
        if not (flags & 1):
            should_length = height * row_bytes
            is_length = len (bitmap_data)
            if is_length == should_length:
                print "%s        %d bytes of uncompresses image data" % (prefix, is_length)
            else:
                print "%s        %d bytes of uncompresses image data expected but has %d bytes!!!" % \
                      (prefix, should_length, is_length)
                

        

    def dump_record (self, id):
        """(Re-)Assemble the binary representation of this image document"""
        assert _DOC_HEADER_SIZE==8

        if len (self._data) > self._config.get_int ('image_compression_limit', 0):
            compressed_data = CompressFunction (self._data)
            if len (compressed_data) < len (self._data):
                data = compressed_data
                type = DOCTYPE_IMAGE_COMPRESSED
            else:
                data = self._data
                type = DOCTYPE_IMAGE
        else:
            data = self._data
            type = DOCTYPE_IMAGE

        header = struct.pack (">HHHH",
                              id,                # uid
                              0,                # header size
                              len (self._data),        # size
                              type)                # content type

        return header + data



    def undump_record (self, data, verbose=0):
        """Given the contents of an image cache file (as
        e.g. generated by the awk parser), dissasseble the contents
        and show it to the user"""

        assert len (data) > _DOC_HEADER_SIZE
        header_data = data[:_DOC_HEADER_SIZE]
        (uid, paragraphs, data_size, content_type) = struct.unpack (">HHHH", header_data)
        if verbose:
            print "Documente header:\n" \
                  "\tdoc id: %d\n" \
                  "\tnumber of paragraphs: %d\n" \
                  "\tdata size: %d" \
                  "\ttype: %d" % \
                  (uid, paragraphs, data_size, content_type)
        
        if content_type == DOCTYPE_IMAGE_COMPRESSED:
            compressed_data = data[_DOC_HEADER_SIZE:]
            rest_data = UncompressFunction (compressed_data)
        else:
            rest_data = data[_DOC_HEADER_SIZE:]
            compressed_data = rest_data

        ## Do sanity check on header values
        assert paragraphs == 0, \
               "Number of paragraphs for image documents must be %d but is %d" % (0, header_size)
        assert data_size == len (rest_data),\
               "Image document's header says length is %d but really it is %d" % \
               (data_size, len (rest_data))
        assert content_type==DOCTYPE_IMAGE or content_type == DOCTYPE_IMAGE_COMPRESSED, \
               "Content type must be %d or %d but is %d " % (DOCTYPE_IMAGE, DOCTYPE_IMAGE_COMPRESSED, content_type)

        # self._stats.add_headerbytes (_DOC_HEADER_SIZE)

        self._data = rest_data
        self._stats.add_graphicsbytes (len (rest_data))
        self._stats.add_compressed_graphicsbytes (len (compressed_data))
        self._stats.add_images (1)

        if verbose:
            self.print_summary (prefix="\t")


    def get_stats (self):
        return self._stats



class PluckerMultiImageDocument (PluckerDocument):
    """A class to contain information about the separate pieces of an image each a Plucker Image document (PalmOS image format)."""

    def __init__ (self, url, config=None):
        PluckerDocument.__init__ (self, url)
        self._data = ""
        self._stats = PluckerTextDocumentStatistics ()
        self._config = config
        self._size_data = None
        self._piece_images = []
        self._piece_ids = []
        self._columns = 0
        self._rows = 0
        self._related_images = []

    def is_multiimage_document (self):
        return 1

    def get_size_hint (self):
        return 0

    def get_related_images (self):
        return self._related_images

    def add_related_image (self, url, attributes):
        self._related_images.append((url, attributes,))

    def add_piece_image (self, doc, id = 0):
        self._piece_images.append((doc, id))

    def get_pieces (self):
        return self._piece_images

    def set_size (self, columns, rows):
        self._columns = columns
        self._rows = rows

    def resolve_ids (self, resolver):
        for (piece_doc, piece_id) in self._piece_images:
            pid = resolver.get_or_add(piece_doc.get_url())
            self._piece_ids.append(pid)

    def dump_record (self, id):
        data = []

        type = DOCTYPE_MULTIIMAGE

        data.append( struct.pack (">HH", self._columns, self._rows))

        for piece_id in self._piece_ids:
            data.append( struct.pack (">H", piece_id))

        data = string.join(data, "")

        header = struct.pack (">HHHH",
                              id,               # uid
                              0,                # number of paragraphs
                              len(data),        # size
                              type)             # content type

        return header + data



class PluckerMailtoDocument (PluckerDocument):
    """A class to contain information about one Plucker Mailto document."""

    def __init__ (self, url):
        PluckerDocument.__init__ (self, url)
        self._data = ""
        self._stats = PluckerTextDocumentStatistics ()


    def is_mailto_document (self):
        return 1

    def get_size_hint (self):
        return len(self._url)

    def parse (self, url):
        to_offset = cc_offset = subject_offset = body_offset = 0
        total = 8
        result = []

        data = str (url)
        data = string.replace (data, "\r", "")
        data = string.replace (data, "\n", "")
        data = string.replace (data, "?", "\n")
        data = string.replace (data, "&amp", "\n")
        data = string.replace (data, "&", "\n")
        data = string.replace (data, "=", ":")
        data = data + "\n"

        file = StringIO.StringIO (data)

        m = Message(file)
        to = m.getrawheader("mailto")
        if (len (to) < 3):
            to = m.getrawheader("to")
        cc = m.getrawheader("cc")
        subject = m.getrawheader("subject")
        body = m.getrawheader("body")
        file.close()

        if to is not None:
            to_offset = total
            total = total + len (to)
        if cc is not None:
            cc_offset = total
            total = total + len (cc)
        if subject is not None:
            subject_offset = total
            total = total + len (subject)
        if body is not None:
            body_offset = total
            total = total + len (body)


        result.append (struct.pack (">HHHH",
                        to_offset, cc_offset, subject_offset, body_offset))
        if to is not None:
            result.append (to)
        if cc is not None:
            result.append (cc)
        if subject is not None:
            result.append (subject)
        if body is not None:
            result.append (body)

        return result, total


    def dump_record (self, id):
        """(Re-)Assemble the binary representation of this mailto document"""
        (data, length) = self.parse (self._url)
        type = DOCTYPE_MAILTO

        data = string.join (data, "")
        data = string.replace(data, "\n", "\000")

        header = struct.pack (">HHHH",
                              id,               # uid
                              0,                # number of paragraphs
                              len(data),        # size
                              type)             # content type

        return header + data


    def undump_record (self, data, verbose=0):
        """Given the contents of an mailto cache file (as
        e.g. generated by the awk parser), dissasseble the contents
        and show it to the user"""

        assert len (data) > _DOC_HEADER_SIZE
        header_data = data[:_DOC_HEADER_SIZE]
        (uid, paragraphs, data_size, content_type) = struct.unpack (">HHHH", header_data)
        if verbose:
            print "Documente header:\n" \
                  "\tdoc id: %d\n" \
                  "\tNumber of paragraphs: %d\n" \
                  "\tdata size: %d" \
                  "\ttype: %d" % \
                  (uid, paragraphs, data_size, content_type)
        
        rest_data = data[_DOC_HEADER_SIZE:]

        ## Do sanity check on header values
        assert paragraphs == 0, \
               "Number of paragraphs for mailto documents must be %d but is %d" % (0, header_size)
        assert data_size == len (rest_data),\
               "Mailto document's header says length is %d but really it is %d" % \
               (data_size, len (rest_data))
        off_data = rest_data[:8]
        (to_offset, cc_offset, subject_offset, body_offset) = struct.unpack (">HHHH", off_data)
        print "Mailto offsets: \n" \
            "\tto      %d\n" \
            "\tcc      %d\n" \
            "\tsubject %d\n" \
            "\tbody    %d" % \
            (to_offset, cc_offset, subject_offset, body_offset)
        print "Data: \n" \
            "\tto      %s\n" \
            "\tcc      %s\n" \
            "\tsubject %s\n" \
            "\tbody    %s\n" % \
            (rest_data[to_offset:cc_offset], rest_data[cc_offset:subject_offset], \
             rest_data[subject_offset:body_offset], rest_data[body_offset:])



class Row:
    def __init__(self):
        self.cols = []

class TableCell:
    def __init__(self, align = 0, attr = None):
        cols = 1
        rows = 1
        if attr is not None:
            if attr.has_key('colspan'):
                cols = int(attr['colspan'])
            if attr.has_key('rowspan'):
                rows = int(attr['rowspan'])
        self.attr = attr
        self.align = align
        self.cols = cols
        self.rows = rows
        self.image_ref = 0
        self.text = ""
        self.parts = []


class PluckerTableDocument (PluckerDocument):
    """A class to contain information about a table"""

    def __init__(self, url, border = 0, attr = None, depth = 1, border_color = 0, link_color = 0):
        PluckerDocument.__init__ (self, url)
        self.border = border
        self.rows = []
        self.attributes = attr
        self.depth = depth
        if self.depth < 1:
            self.depth = 1
        self.border_color = border_color
        self.link_color = link_color
        self.totcols = 0

    def is_table_document (self):
        return 1

    def add_row(self):
        self.rows.append(Row())

    def add_cell(self, align = 0, attr = None):
        if not len(self.rows):
            self.rows.append(Row())
        self.rows[-1].cols.append(TableCell(align = align, attr = attr))

    def add_cell_text(self, text = ""):
        if len(text):
            self.rows[-1].cols[-1].parts.append((CMD_TEXT, text))

    def add_cell_image(self, image = 0, attr = None):
        self.rows[-1].cols[-1].parts.append((CMD_IMAGE, attr))

    def add_link(self, attr = None):
        self.rows[-1].cols[-1].parts.append((CMD_ANCHOR_START, attr))

    def end_link(self):
        self.rows[-1].cols[-1].parts.append((CMD_ANCHOR_END, 0))

    def add_table(self, table = 0, attr = None):
        if not len(self.rows):
            self.add_cell()
        self.rows[-1].cols[-1].parts.append((CMD_TABLE, attr))

    def set_align (self, align = 0):
        self.rows[-1].cols[-1].align = align

    def get_attrs (self):
        return self.attributes

    def resolve_ids (self, resolver):
        for j in range(0, len(self.rows)):
            for k in range(0, len(self.rows[j].cols)):
                acell = self.rows[j].cols[k]
                parts = []
                in_anchor = 0
                for (cmd, data) in acell.parts:
                    if cmd == CMD_TEXT and not data.isspace():
                        """ In case we were <PRE> """
                        data = string.split(data, "\n")
                        data = string.join(data, "\000\070")
                        parts.append(data)
                    elif cmd == CMD_ANCHOR_START:
                        if data.has_key('href'):
                            doc = find_registered_document(data['_plucker_id_tag'])
                            if doc is not None:
                                link_ref = resolver.get_or_add(doc)
                            else:
                                link_ref = resolver.get_or_add(data['href'])
                            if type (link_ref) == types.TupleType:
                                parts.append (struct.pack (">BBHH", 0, 0x0c, link_ref[0], link_ref[1]))
                                in_anchor = 1
                            elif type (link_ref) == types.IntType:
                                parts.append(struct.pack (">BBH", 0, 0x0A, link_ref))
                                in_anchor = 1
                    elif cmd == CMD_ANCHOR_END:
                        if in_anchor == 1:
                            parts.append(struct.pack (">BB", 0, 0x08))
                        in_anchor = 0
                    elif cmd == CMD_TABLE:
                        if data.has_key('href'):
                            doc = find_registered_document(data['_plucker_id_tag'])
                            if doc is not None:
                                link_ref = resolver.get_or_add(doc)
                            else:
                                link_ref = resolver.get_or_add(data['href'])
                            if link_ref is None:
                                link_ref = 0
                            parts.append(struct.pack (">BBH", 0, 0x92, link_ref))
                    elif cmd == CMD_IMAGE:
                        if data.has_key('_plucker_id_tag_inlineimage'):
                            doc = find_registered_document(data['_plucker_id_tag_inlineimage'])
                            link_ref = doc and resolver.get_or_add(doc)
                        elif acell.attr.has_key('src'):
                            link_ref = resolver.get_or_add(data['src'])
                        if link_ref is None:
                            if data.has_key ('alt'):
                                parts.append (data['alt'])
                            else:
                                parts.append ('[img]')
                        else:
                            if data.has_key('_plucker_id_tag_outoflineimage'):
                                doc = find_registered_document(data['_plucker_id_tag_outoflineimage'])
                                big_ref = doc and resolver.get_or_add(doc)
                            if big_ref is not None:
                                parts.append(struct.pack (">BBHH", 0, 0x5C, big_ref, link_ref))
                            else:
                                parts.append(struct.pack (">BBH", 0, 0x1A, link_ref))

                if len(parts):
                    acell.text = string.join(parts, "")


    def split_line (self, text, cols):
        """Split line into pieces with maxlen of 25 * cols plus next space"""
        """Or 20 * cols if the characters are BOLD """
        maxlen = 25
        clines = []
        ltext = string.split (text, '\000\070')
        while len(ltext):
            while len(ltext[0]):
                line = ""
                count = 0
                offset = 0
                while offset < len(ltext[0]) and count < (maxlen * cols):
                    if ord(ltext[0][offset]):
                        offset = offset + 1
                        count = count + 1
                    else:
                        if ord(ltext[0][offset + 1]) == 0x11:
                            """ BOLD maxlen is 20 """
                            if ord(ltext[0][offset + 2]) == 7:
                                maxlen = 20
                            else:
                                maxlen = 25
                        offset = offset + 2 + (ord(ltext[0][offset + 1]) & 7)
                while offset < len(ltext[0]) and ord(ltext[0][offset]) > 32:
                    offset = offset + 1
                if offset < len(ltext[0]) and not ord(ltext[0][offset]):
                    #trailing function
                    offset = offset + 2 + (ord(ltext[0][offset + 1]) & 7)
                line = ltext[0][:offset]
                ltext[0] = ltext[0][offset:]
                clines.append(line)
            del ltext[0]
        clines = string.join(clines, '\000\070')
        return clines


    def calc_table (self):
        self.totcols = 0
        for j in range(0, len(self.rows)):
            rowcols = 0
            for k in range(0, len(self.rows[j].cols)):
                rowcols = rowcols + self.rows[j].cols[k].cols
            if rowcols > self.totcols:
                self.totcols = rowcols
        for j in range(0, len(self.rows)):
            while len(self.rows[j].cols) < self.totcols:
                self.rows[j].cols.append(TableCell(0, None))

        for j in range(0, len(self.rows)):
            for k in range(0, len(self.rows[j].cols)):
                acell = self.rows[j].cols[k]
                if len(acell.text):
                    acell.text = self.split_line(acell.text, acell.cols)
                if acell.image_ref is None:
                    acell.image_ref = 0

    def dump_record (self, id):

        data = []
        self.calc_table ()

        for j in range(0, len(self.rows)):
            data.append (struct.pack (">BB", 0, 0x90))
            for k in range(0, len(self.rows[j].cols)):
                cell = self.rows[j].cols[k]
                if len (cell.text) or cell.image_ref:
                    data.append (struct.pack (">BBBHBBH", 0, 0x97,
                                               cell.align,     # align
                                               cell.image_ref, # image ref
                                               cell.cols,      # colspan
                                               cell.rows,      # rowspan
                                               len(cell.text)))
                    data.append(cell.text)

        data.append (struct.pack (">B", 0))
        data = string.join (data, "")

        table_header = struct.pack (">HHHBBLL",
                              len(data),
                              self.totcols,
                              len(self.rows),
                              self.depth,
                              self.border,
                              self.border_color,
                              self.link_color)

        table_header = string.join (table_header, "")
        data = table_header + data

        compressed_data = CompressFunction (data)
        if len (compressed_data) < len (data):
            ship_data = compressed_data
            type = DOCTYPE_TABLE_COMPRESSED
        else:
            ship_data = data
            type = DOCTYPE_TABLE

        header = struct.pack (">HHHH",
                              id,               # uid
                              0,                # number of paragraphs
                              len(data),        # size
                              type)             # content type

        return header + ship_data

    def dump(self):
        totcols = 0
        for j in range(0, len(self.rows)):
            rowcols = 0
            for k in range(0, len(self.rows[j].cols)):
                rowcols = rowcols + self.rows[j].cols[k].cols
            if rowcols > totcols:
                totcols = rowcols
            for k in range(0, len(self.rows[j].cols)):
                print "Cell A = %d I = %d L = %d %s" % (
                                            self.rows[j].cols[k].align,
                                            self.rows[j].cols[k].image_ref,
                                            len(self.rows[j].cols[k].text),
                                            self.rows[j].cols[k].text)

        print "Table border = %d c = %d r = %d" % (self.border,
                                                   totcols,
                                                   len(self.rows))


class PluckerSpecialDocument (PluckerDocument):
    """A class to contain information about other types of  Plucker documents."""

    def __init__ (self, url):
        PluckerDocument.__init__ (self, url)


    def is_special_document (self):
        return 1


class PluckerIndexDocument (PluckerSpecialDocument):
    """A class to contain information about the DB (record 0)"""
    def __init__ (self, url, config, has_metadata, has_bookmarks):
        PluckerSpecialDocument.__init__(self, url)
        self._config = config
        self._has_metadata = has_metadata
        self._has_bookmarks = has_bookmarks

    RSVD_REC_NAME_HOME = 0
    RSVD_REC_NAME_EXT_BOOKMARKS = 1
    RSVD_REC_NAME_URLS = 2
    RSVD_REC_NAME_CATEGORIES = 3
    RSVD_REC_NAME_METADATA = 4

    def dump_record (self, record_id = 1):
        """(Re-)Assemble the binary representation of this document"""

        if self._config.get_bool ('zlib_compression', 0):
            type = DBTYPE_ZLIB
        else:
            type = DBTYPE_DOC

        # we always have a home document with record id 2, so start with that
        reserved = struct.pack(">HH", self.RSVD_REC_NAME_HOME, 2)

        if not self._config.get_bool ('no_urlinfo', 0):
            # OK, the URL info is in record 3
            reserved = reserved + struct.pack(">HH", self.RSVD_REC_NAME_URLS, 3)

        if self._config.get_string('category'):
            # OK, the category info is in record 4
            reserved = reserved + struct.pack(">HH", self.RSVD_REC_NAME_CATEGORIES, 4)

        if self._has_metadata:
            # That goes in record 5
            reserved = reserved + struct.pack(">HH", self.RSVD_REC_NAME_METADATA, 5)

        if self._has_bookmarks:
            # That goes in record 6
            reserved = reserved + struct.pack(">HH", self.RSVD_REC_NAME_EXT_BOOKMARKS, 6)

        index = struct.pack (">HHH",
                              record_id,        # uid
                              type,                # compression type
                              len(reserved)/4)        # number of reserved records

        return index + reserved


class PluckerCategoryDocument (PluckerSpecialDocument):
    """A class to contain information about the default category"""
    def __init__ (self, url, config):
        PluckerSpecialDocument.__init__(self, url)
        self._config = config

    def dump_record (self, record_id = 4):
        """(Re-)Assemble the binary representation of this document"""

        category_list = self._config.get_string('category')

        categories = string.replace (category_list, ";", "\0")

        header = struct.pack (">HHHH", 
                              record_id,            # uid
                              0,                    # number of paragraphs
                              len(categories) + 1,  # size
                              DOCTYPE_CATEGORY)     # content type

        return header + categories + '\0'



class PluckerMetadataDocument (PluckerSpecialDocument):
    """A class to contain information about extra metadata like charsets"""
    def __init__ (self, url, info):
        PluckerSpecialDocument.__init__(self, url)
        self._info = info

    TYPECODE_CHARSET = 1
    TYPECODE_EXCEPTIONAL_CHARSETS = 2
    TYPECODE_OWNER_ID = 3
    TYPECODE_AUTHOR = 4
    TYPECODE_TITLE = 5
    TYPECODE_PUBLICATION_DATE = 6

    def dump_record (self, record_id = 5):
        """(Re-)Assemble the binary representation of this document"""

        subrecords = []
        count = 0
        for key in self._info.keys():

            if key == 'CharSet':
                subrecords.append(struct.pack(">HHH",
                                              self.TYPECODE_CHARSET,
                                              1,
                                              self._info[key]))
                count = count + 1

            elif key == 'ExceptionalCharSets':
                subrecords.append(struct.pack('>HH', self.TYPECODE_EXCEPTIONAL_CHARSETS,
                                              2 * len(self._info[key])))
                subrecords.append(string.join(map(lambda v: struct.pack('>HH', v[0], v[1]), self._info[key]), ''))
                count = count + 1
                                  
            elif key == 'OwnerID':
                val = zlib.crc32(self._info[key])
                message(3, 'CRC-32 of owner-id "%s" is 0x%x\n', self._info[key], val)
                subrecords.append(struct.pack('>HHL', self.TYPECODE_OWNER_ID, 2, val))
                count = count + 1

            elif key == 'Author':
                vlen = len(self._info[key])
                nwords = (vlen + 1) / 2
                dataval = self._info[key] + '\0' * (vlen % 2)
                assert(len(dataval) == (nwords * 2), "Invalid length calculation in marshalling of PluckerMetadataDocument")
                subrecords.append(struct.pack('>HH', self.TYPECODE_AUTHOR, nwords))
                subrecords.append(dataval)
                count = count + 1

            elif key == 'Title':
                vlen = len(self._info[key])
                nwords = (vlen + 1) / 2
                dataval = self._info[key] + '\0' * (vlen % 2)
                assert(len(dataval) == (nwords * 2), "Invalid length calculation in marshalling of PluckerMetadataDocument")
                subrecords.append(struct.pack('>HH', self.TYPECODE_TITLE, nwords))
                subrecords.append(dataval)
                count = count + 1

            else:
                raise ValueError("Unknown metadata key " + key)

        # if we do a metadata record at all, put the publication date/time in
        val = long(time.time()) + PILOT_TIME_DELTA
        subrecords.append(struct.pack('>HHL', self.TYPECODE_PUBLICATION_DATE, 2, val))
        count = count + 1

        header = struct.pack (">HHHHH", 
                              record_id,            # uid
                              0,                    # number of paragraphs
                              reduce(lambda x, y: x + len(y), subrecords, 2),        # size in bytes
                              DOCTYPE_METADATA,     # record type
                              count)                    # number of subrecords                

        return string.join([header] + subrecords, '')


class PluckerLinkIndexDocument (PluckerSpecialDocument):
    """A class to contain information about the link documents"""
    def __init__ (self, url, docs, resolver):
        PluckerSpecialDocument.__init__(self, url)
        self._docs = docs
        self._resolver = resolver

    def dump_record (self, id = 3):
        """(Re-)Assemble the binary representation of this document"""

        data = [struct.pack (">HHHH",
                             id,                        # uid
                             0,                                # number of paragraphs
                             len(self._docs) * 4,        # size
                             DOCTYPE_LINK_INDEX)]        # content type

        data = data + map(lambda y, r=self._resolver: struct.pack(">HH",
                                                                  y.get_bounds()[1],
                                                                  r.get_or_add(y)),
                          self._docs)
        return string.join(data, '')


class PluckerLinksDocument (PluckerSpecialDocument):
    """A class to contain information about the links"""
    def __init__ (self, url, links_vector, start):
        PluckerSpecialDocument.__init__(self, url)
        self._min_id = start
        self._max_id = min(start + 200, len(links_vector))-1
        self._urls = string.join(links_vector[start:self._max_id+1], '\0') + '\0'

    def get_bounds(self):
        return (self._min_id, self._max_id)

    def dump_record (self, id):
        """(Re-)Assemble the binary representation of this document"""

        compressed_data = CompressFunction (self._urls)
        if len (compressed_data) > len (self._urls):
            data = self._urls
            type = DOCTYPE_LINKS
        else:
            data = compressed_data
            type = DOCTYPE_LINKS_COMPRESSED
        header = struct.pack (">HHHH",
                              id,               # uid
                              0,                # number of paragraphs
                              len(self._urls),  # size
                              type)             # content type
        result = header + data
        return result


class PluckerBookmarkDocument (PluckerSpecialDocument):
    """A class to contain information about the links"""
    def __init__ (self, url, bookmarks):
        PluckerSpecialDocument.__init__(self, url)
        self._bookmarks = bookmarks

    def dump_record (self, id = 6):
        """(Re-)Assemble the binary representation of this document"""

        names = []
        the_keys = self._bookmarks.keys()
        the_keys.sort()

        for key in the_keys:
            names.append(key[:20])

        names = string.join(names, '\0') + '\0'
        data = struct.pack (">HH", len(self._bookmarks), _DOC_HEADER_SIZE + 4 + len(names))

        data = data + names
        for key in the_keys:
            (doc, para) = self._bookmarks[key]
            data = data + struct.pack (">HH", doc, para)

        header = struct.pack (">HHHH",
                               id,                    # uid
                               0,                     # number of paragraphs
                               len(data),             # size
                               DOCTYPE_BOOKMARKS)     # content type

        return header + data


def Undump_PluckerDocument (url, data, verbose=0):
    """Given 'data' try to find out what sort of document that is and
    unparse it as such.  Returns a PluckerDocument."""

    pluckerdoc = None
    
    if len (data) > _DOC_HEADER_SIZE:
        header_data = data[:_DOC_HEADER_SIZE]
        (uid, paragraphs, size, content_type) = struct.unpack (">HHHH", header_data)
        # We assign some dummy values to pass the test below
        vert_offset=0
        first_visible=_DOC_HEADER_SIZE
        first_paraY=0
        last_visible=0
        last_paraY=0 
        height=0
        if content_type==DOCTYPE_HTML or content_type==DOCTYPE_HTML_COMPRESSED:
            # aha, a text document
            pluckerdoc = PluckerTextDocument (url)
            pluckerdoc.undump_record (data, verbose)
        elif content_type==DOCTYPE_IMAGE or content_type==DOCTYPE_IMAGE_COMPRESSED:
            # aha, an image document
            pluckerdoc = PluckerImageDocument (url)
            pluckerdoc.undump_record (data, verbose)
        elif content_type==DOCTYPE_MAILTO:
            # aha, a mailto document
            pluckerdoc = PluckerMailtoDocument (url)
            pluckerdoc.undump_record (data, verbose)

    if not pluckerdoc:
        # nothing worked, i.e. unknown data
        raise ValueError("Unknown Plucker data »%s«" % repr(data))
    return pluckerdoc





        
if __name__ == '__main__':
    # This gets executed if this file is called as a script
    import PyPlucker
    import getopt
    import os
    import sys
    
    def usage(reason=None):
        if reason is not None:
            print reason
        print "Usage: %s [-h] [-v] [-s] [-S] [-d] [-z] <filename> ..." % sys.argv[0]
        print "  Parses the plucker cache file(s) and verifies the contents"
        print "   -h : display usage information and exit"
        print "   -v : output version information and exit"
        print "   -s : show statics of each document"
        print "   -S : show summary statistics of all documents"
        print "   -d : show disassembly information of each document"
        print "   -z : use ZLib to uncompress the documents"

        if reason is not None:
            sys.exit (1)
        else:
            sys.exit (0)

    # the option dictionary will be used to hold flags if that option is set
    # it gets initialized to all false
    option = {}
    for letter in string.lowercase + string.uppercase:
        option[letter] = None
        
    (optlist, args) = getopt.getopt(sys.argv[1:], "hvsSdz")
    for (k,v) in optlist:
        # k is of the form e.g. '-s', so we pick the second char
        if v == "":
            v = 1
        option[k[1]] = v

    _DOC_HEADER_SIZE = 8
    _PARA_HEADER_SIZE = 4

    if option['h']:
        usage ()

    if option['v']:
        print "$Revision: 1.67 $"
        sys.exit(0)

    if not args:
        usage ("Please specify cache file(s) to parse")

    if option['z']:
        UseZLibCompression ()

    all_stats = PluckerTextDocumentStatistics ()
    num_files = 0
    for filename in args:
        if option['d'] or option['s']:
            print "\nProcessing %s..." % filename
        try:
            file = open(filename, "rb")
            text = string.join (file.readlines (), "")
            file.close()
        except IOError, text:
            print "Error: %s" % text
            text = ""

        if text:
            # non-empty file
            try:
                plucker = Undump_PluckerDocument ("file:/"+filename,
                                                  text,
                                                  verbose=option['d'])

                if not option['d'] and option['s'] and plucker.is_special_document ():
                        print "Not a document file"
                if plucker.is_text_document () or plucker.is_image_document ():
                    stats = plucker.get_stats ()
                    num_files = num_files + 1
                    all_stats.combine_with (stats)
                    if option['s']:
                        stats.pretty_print (prefix_string = "")
                    
            except AssertionError, text:
                print "!!! Parsing %s failed with an assertion error: %s" % (filename, text)
                # usually text is empty, so we write the traceback
                import traceback
                traceback.print_exc ()
            except ValueError, text:
                print "!!! Parsing %s failed. %s" % (filename, text)
                import traceback
                traceback.print_exc ()
                
    if option['S']:
        print "\n\n%d files processed:" % num_files
        all_stats.pretty_print (prefix_string="")