#!/usr/bin/env python # fngrab.py # # Copyright Rob Cakebread 2001 # Released under the GNU Public License (See freenet:KSK@gpl.txt) # Thank you LMN741N for StreamingRequest and threading code! # # See README for help or run fngrab.py with no options for usage # # Python system stuff import os, socket, sys, string, time, threading # Custom stuff: from FreenetGrab import cmdline from FreenetGrab import xmlrpclib #import cmdline #from xmlrpclib import * TRANSFERRING=0 DONE=1 FAILED=2 __version__ = '0.0.4' usage = '''Usage: fngrab.py [options] key Options: -D, --debug Print exception info, misc debugging info -d, --dir directory Speficy directory where download files are saved -f, --file filename Download all keys in file -g, --range n,n Specify range: 4,15 using incremental index -h, --htl n Hops to Live. Default: 20 -i, --increment n Number to increase htl on retry (-r, --retry) -l, --threshold n Total nbr of fails before quitting (-x) default: 3 -n, --number n Number of simultaneous requests (not implemented) -o, --output filename Write to file with filename -o filename -p, --port n XML-RPC port. Default: 6690 -r, --retry n Number of attempts to retry on fail -s, --server server Server address. Default: 127.0.0.1 -t, --test Test only, good for verifying lists of keys with -f -v, --verbose Verbose output, shows filesize in chunks as received. -V, --version Prints version, copyright info, then exits. -x, --index name Downloads range (-g) of keys from in-Freenet index -z, --past n Get index from n days in the past Program dumps to stdout if -o, --output is not speficied. The -t test option aborts the download after beginning. See README for examples''' # Print usage message if no options, no key on command line nbr_args = len(sys.argv) if nbr_args == 1: print usage sys.exit(0) # There are two cases where you would only have 1 argument passed: # 1. -V, --version # 2. A single key with no options if nbr_args == 2: if sys.argv[1][0] == '-': if cmdline.receivedOption('-V --version'): print '' print ('fngrab.py version %s Copyright Rob Cakebread 2001' % (__version__)) print '' print 'Released under the GNU Public License - See freenet:KSK@gpl.txt' print '' else: # Thay passed a single agrument with no key print usage sys.exit(0) # Grab arguments passed on commandline key = sys.argv[nbr_args -1] server = cmdline.getStringValueOf('-s --server', '127.0.0.1') port = cmdline.getIntegerValueOf('-p --port', 6690) htl = cmdline.getIntegerValueOf('-h --htl', 20) filename = cmdline.getStringValueOf('-o --output', '') fromFile = cmdline.getStringValueOf('-f --fromfile', '') retry = cmdline.getIntegerValueOf('-r --retry', 0) increment = cmdline.getIntegerValueOf('-i --increment', 10) threshold = cmdline.getIntegerValueOf('-l --threshold', 3) index = cmdline.getStringValueOf('-x --index', '') range = cmdline.getStringValueOf('-g --range', '') dir = cmdline.getStringValueOf('-d --dir', '') past = cmdline.getIntegerValueOf('-z --past', 0) simul = cmdline.getIntegerValueOf('-n --number', 1) if dir: if os.path.exists(dir): pass else: print ('Directory %s does not exist.' % dir) sys.exit(0) if cmdline.receivedOption('-g --range'): incremental = 1 else: incremental = 0 if cmdline.receivedOption('-d --debug'): debug = 1 else: debug = 0 if cmdline.receivedOption('-v --verbose'): verbose = 1 else: verbose = 0 if cmdline.receivedOption('-t --test'): test = 1 else: test = 0 if filename: # delete list of verified keys if it exists if test: if os.path.exists(filename): os.unlink(filename) class getKey(threading.Thread): def __init__(self, server, port, key, filename, htl, test): threading.Thread.__init__(self) self.server=server self.port=port self.key=key self.filename=filename self.htl=htl self.test=test self.index=index if filename: if self.test: self.file = open(self.filename, "ab") else: if dir: self.file = open(('%s/%s' % (dir, self.filename)), "wb") else: self.file = open(self.filename, "wb") if verbose: print ('Requesting %s with %s hops to live...' % (self.key, self.htl)) def run(self): client = Server('http://%s:%s/' % (self.server, self.port)) try: id = client.streaming.getPort() except: print 'Error- Is XML-RPC enabled in your Freenet config file and is Freenet running?' print ('server:%s port:%s\n' % (self.server, self.port)) sys.exit(0) s=socket.socket(socket.AF_INET,socket.SOCK_STREAM) s.connect((self.server,id)) client.streaming.startRequest(id, self.key, self.htl) status=client.streaming.getStatus(id) ttlFileSize = client.streaming.getFileSize(id) if verbose: if ttlFileSize != -1: print ('File size %i (%s)' % (ttlFileSize, self.filename)) filesize = 0 testDone = 0 sizeCounter = 0 try: while 1: if status==FAILED: print 'FAILED' break chunk=s.recv(1024) if not chunk: break filesize = filesize + len(chunk) if testDone: break if verbose: if simul == 1: #If more than one download, don't print filesize for each one in chunks sizeCounter = sizeCounter + 1 if sizeCounter >= 10: #Print size every 10k print filesize sizeCounter = 0 if self.filename: if test: if filesize > 10000: self.file.write('GOOD: %s\n' % key) print ('GOOD: %s\n' % key) self.file.close() testDone = 1 else: self.file.write(chunk) else: print chunk print ('DONE: %s Size: %s' % (self.key, ttlFileSize)) return status # New to Streaming except Fault, x: status=client.streaming.getStatus(id) if status==DONE: if test: self.file.write('GOOD: %s\n' % self.key) self.file.close() if filename: print 'Freenet Grab error #1' #print ('DONE: %s Size: %s' % (self.key, ttlFileSize)) self.file.close() return status else: if debug: print x # info for exceptions if test: self.file.write('FAIL: %s\n' % self.key) print ('FAIL: %s' % self.key) self.file.close() else: print ('FAIL: %s' % self.key) return status # This is where the action begins. There is # one section for getting keys from a file, # one section for indexes and one section # for a single key on the command line. if fromFile: # get list of keys from file infile = open(fromFile, 'r') keys = infile.readlines() n = 0 threadlist = [] nbrThreads = threading.activeCount() while 1: try: key = keys[n][:-1] if test: print ('Trying: %s' % key) else: f = string.replace(key, '\\', '/') f = string.replace(key, '@', '/') if string.find(f, '/') != -1: dirname, filename = os.path.split(f) else: filename = key s = getKey(server, port, key, filename, htl, test) nbrThreads = threading.activeCount() while nbrThreads > simul - 1: time.sleep(1) nbrThreads = threading.activeCount() s.start() threadlist.append(s) if retry: if s == 2: # Failed htl = htl + increment print ('Retrying with htl of %s' % htl) s = getKey(server, port, key, filename, htl, test) s.start() threadlist.append(s) retry = retry - 1 n = n + 1 except: nbrThreads = threading.activeCount() if nbrThreads == 1: print 'Done with all keys.' sys.exit(0) elif index: # Get keys from in-Freenet key index now = int(time.time()) # Seconds since Epoch midnight = now - (now % 86400) # GMT Midnight failures = 0 if incremental: pass else: if past: midnight = midnight - (past * 86400) index = ('%s-%s' % (index, midnight)) if range: fromTo = string.split(range, ',') start = string.atoi(fromTo[0]) end = string.atoi(fromTo[1]) else: start = 0 end = 0 # Keep going till number of failures == threshold inc = start nbrThreads = 1 while 1: key = ('%s-%i' % (index, inc)) filename = 'tmpkey' s = getKey(server, port, key, filename, htl, test) s.start() inc = inc + 1 f = open(filename, 'r') l = f.readline() if l == '': # Fail failures = failures + 1 if failures > threshold: sys.exit(0) if end != 0: if inc == end: sys.exit(0) if string.find(l, '