#!/usr/bin/env python """ $URL: svn+ssh://svn.mems-exchange.org/repos/trunk/qp/bin/qpcensus.py $ $Id: qpcensus.py 29694 2007-03-29 14:07:26Z dbinger $ """ from cPickle import loads from datetime import datetime from durus.connection import Connection, ROOT_OID from durus.file_storage import FileStorage from durus.serialize import unpack_record, split_oids from qp.lib.site import Site from qp.lib.spec import get_spec_problems, get_specs, get_spec_doc from qp.lib.spec import persistent_vars, Specified, Mixin import sys class ObjectLocator: def __init__(self, connection, reference_index): self.connection = connection self.reference_index = reference_index def where_is(self, obj): root = self.connection.get_root() def locate(obj, seen): if len(seen) > 50: return '...' if obj is root: return 'root' oid = obj._p_oid for referring_oid in self.reference_index.get(oid, []): if referring_oid not in seen: seen.add(referring_oid) break else: return repr(obj) # give up referring = self.connection.get(referring_oid) ref = locate(referring, seen) for key, value in persistent_vars(referring).iteritems(): if obj is value: return "%s.%s" % (ref, key) try: for k, v in value.iteritems(): if obj is v: return "%s[%r]" % (ref, k) except AttributeError: pass try: for k, v in enumerate(value): if obj is v: return "%s[%r]" % (ref, k) except TypeError: pass return '%s ... ' % ref return locate(obj, set()) def format_specs(klass): if klass.__bases__: bases = '(%s):' % ', '.join( [b.__name__ for b in klass.__bases__]) else: bases = ":" s = "class %s%s\n " % (klass.__name__, bases) s += get_spec_doc(klass).replace('\n', '\n ') or 'pass' s += "\n\n" return s def check_durus(file, schema, klasses, show_all, show_max, show_path, show_shadows): storage = FileStorage(file, readonly=True) connection = Connection(storage, cache_size=100) if schema: schema_output = open(schema, 'w') problem_record = {} specs = {} counts = {} size_min = {} size_max = {} maximal = {} size_sum = {} refs = {} instance_count = 0 start = datetime.now() for oid, record in storage.gen_oid_record(start_oid=ROOT_OID): obj = connection.cache.get(oid) record_oid, data, refdata = unpack_record(record) if show_path: for refoid in split_oids(refdata): refs.setdefault(refoid, []).append(oid) if obj is None: klass = loads(data) obj = connection.cache.get_instance(oid, klass, connection) if obj._p_is_ghost(): state = connection.reader.get_state(data, load=True) obj.__setstate__(state) obj._p_set_status_saved() size = len(record) klass = obj.__class__ if klass not in counts: counts[klass] = 1 size_min[klass] = size size_max[klass] = size size_sum[klass] = size maximal[klass] = obj specs[klass] = get_specs(obj) if schema: schema_output.write(format_specs(klass)) else: counts[klass] += 1 size_min[klass] = min(size, size_min[klass]) if size > size_max[klass]: maximal[klass] = obj size_max[klass] = size size_sum[klass] += size if show_all or klass not in problem_record: problems = get_spec_problems(obj, specs=specs[klass]) if problems: problem_record.setdefault(klass, []).append((obj, problems)) obj._p_set_status_ghost() instance_count += 1 if instance_count % 10000 == 0: sys.stdout.write('.') sys.stdout.flush() connection.abort() end = datetime.now() print print file print end seconds = (end - start).seconds if seconds > 60: minutes, seconds = divmod(seconds, 60) fmt = "%sm, %ss" % (minutes, seconds) else: fmt = "%ss" % seconds print "Scanned %s instances in in %s." % (instance_count, fmt) klasses = sorted(counts, key=lambda x: x.__name__) if show_path: where_is = ObjectLocator(connection, refs).where_is else: def where_is(obj): return obj._p_format_oid() for klass in klasses: if klass in problem_record: for obj, problem in problem_record[klass]: print print "Problem with instance of %s:" % (klass.__name__) print ' ', where_is(obj) for line in problem: if len(line) > 200: line = line[:200] + ' ...' print ' ', line print print "%7s %-35s %7s %7s %7s %12s" % ( "Count", "Class", "Min", "Max", "Ave", "Sum") for klass in klasses: print "%7s %-35s %7s %7s %7s %12s" % ( counts[klass], klass.__name__, size_min[klass], size_max[klass], size_sum[klass] / counts[klass], size_sum[klass]) if show_max: print print 'Maximal Size Instances:' for klass in klasses: print klass.__name__ print ' ', where_is(maximal[klass]) if show_shadows: for klass in get_all_classes(): print_shadows(klass) def get_all_classes(): all_classes = set() todo = [object] while todo: klass = todo.pop() all_classes.add(klass) for subclass in type.__subclasses__(klass): if subclass not in all_classes: todo.append(subclass) return all_classes def print_shadows(klass): """ Report attributes of klass whose value depends on base class order. We try to avoid having any of these. """ warnings = [] # There are a few names where shadows seem to pose no problem. skip_names = ['__slots__', '__dict__', '__weakref__'] for name in dir(klass): if name not in skip_names and name not in vars(klass): first_parent = None for parent in klass.__mro__[1:]: if name in vars(parent): if first_parent is None: first_parent = parent elif not issubclass(first_parent, parent): warnings.append((name, first_parent, parent)) # The shadow of Mixin.__metaclass__ by Specified.__metaclass__ is normal. skip = ('__metaclass__', Specified, Mixin) if skip in warnings: warnings.remove(skip) if warnings: print print 'In', klass for name, first, other in warnings: print ' %s from %s shadows the one from %s.' % ( name, first.__name__, other.__name__) print def check_durus_main(): from optparse import OptionParser parser = OptionParser() parser.set_description( "Searches for spec anomolies in a durus database. " "Specific site names may be given as extra arguments. ") parser.add_option( '--file', dest="file", default=None, help="If this is given, check this file only.") parser.add_option( '--schema', dest="schema", default=None, help="If given, a schema will be written to this file.") parser.add_option( '--class', dest="classes", default=[], action="append", help="If given, limit study to this class.") parser.add_option( '--all', dest="all", action='store_true', help="If given, all errors are shown not just the first in each class.") parser.add_option( '--showmax', dest="show_max", action='store_true', help=("If given, an instance of maximal size is identified for" " each class.")) parser.add_option( '--shadows', dest="show_shadows", action='store_true', help=("If given, also report known class attributes whose value " "depends on the base class order.")) parser.add_option( '--showpath', dest="show_path", action='store_true', help="If given, work harder to show a path to instances.") (options, args) = parser.parse_args() if options.file: check_durus(options.file, options.schema, options.classes, options.all, options.show_max, options.show_path, options.show_shadows) elif args: for name in args: print "%s:" % name site = Site(name) check_durus(site.get_durus_file(), options.schema, options.classes, options.all, options.show_max, options.show_path, options.show_shadows) if __name__ == '__main__': check_durus_main()