''' urlOpenerSettings.py Copyright 2006 Andres Riancho This file is part of w3af, w3af.sourceforge.net . w3af is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. w3af is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with w3af; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ''' # Some misc imports import core.controllers.outputManager as om import core.data.kb.config as cf from core.controllers.w3afException import w3afException # Handler imports import core.data.url.handlers.localCache as localCache from core.data.url.handlers.keepalive import HTTPHandler as kAHTTP from core.data.url.handlers.keepalive import HTTPSHandler as kAHTTPS import core.data.url.handlers.MultipartPostHandler as MultipartPostHandler import core.data.url.handlers.certHTTPSHandler as certHTTPSHandler import core.data.url.handlers.logHandler as logHandler import core.data.url.handlers.idHandler as idHandler import core.data.url.handlers.mangleHandler as mangleHandler from core.controllers.configurable import configurable from core.controllers.misc.parseOptions import parseOptions class urlOpenerSettings( configurable ): ''' This is a urllib2 configuration manager. @author: Andres Riancho ( andres.riancho@gmail.com ) ''' import urllib2 as _ulib import socket as _socket import urlparse as _uparse from time import sleep as _sleep from random import random as _random from robotparser import RobotFileParser as _rparser import cookielib as _cookielib def __init__(self): # Set the openers to None self._basicAuthHandler = None self._proxyHandler = None self._cookieHandler = None self._httpsHandler = None self._mangleHandler = None self._cookieHandler = None #self._keepAliveHandler = None self._kAHTTP = kAHTTP() self._kAHTTPS = kAHTTPS() # Openers self._nonCacheOpener = None self._cacheOpener = None # User configured variables self._timeout = 10 self._socket.setdefaulttimeout(self._timeout) self._headersFile = '' self._cookieJarFile = '' self._userAgent = 'w3af.sourceforge.net' cf.cf.save('User-Agent', self._userAgent) self._proxyAddress = '' self._proxyPort = '8080' self._basicAuthPass = '' self._basicAuthUser = '' self._basicAuthDomain = '' self.HeaderList = [('User-Agent',self._userAgent)] self._sslCertFile = '' self._sslKeyFile = '' self._ignoreSessCookies = False self._maxFileSize = 400000 self._maxRetrys = 2 self.needUpdate = False # By default, dont mangle any request/responses self._manglePlugins = [] def setHeadersFile(self, HeadersFile ): ''' Sets the special headers to use, this headers are specified in a file by the user. The file can have multiple lines, each line should have the following structure : - HEADER:VALUE_OF_HEADER @parameter HeadersFile: The filename where the special headers are specified. @return: No value is returned. ''' om.out.debug( 'Called SetHeaders') if HeadersFile != '': try: f=open(HeadersFile, 'r') except: raise w3afException('Unable to open headers file: ' + HeadersFile ) for line in f: HeaderName = line.split(':')[0] HeaderValue = ':'.join( line.split(':')[1:] ) HeaderValue = HeaderValue.strip() self.HeaderList.append( (HeaderName,HeaderValue) ) om.out.debug( 'Added the following header: '+ HeaderName+ ': '+ HeaderValue) self._headersFile = HeadersFile def getHeadersFile( self ): return self._headersFile def setCookieJarFile(self, CookieJarFile ): om.out.debug( 'Called SetCookie') if CookieJarFile != '': cj = self._cookielib.MozillaCookieJar() try: cj.load( CookieJarFile ) except Exception, e: raise w3afException( 'Error while loading cookiejar file.' ) self._cookieHandler = self._ulib.HTTPCookieProcessor(cj) self._cookieJarFile = CookieJarFile def getCookieJarFile( self ): return self._cookieJarFile def getSSLKeyFile( self ): ''' @return: A string with the SSL key path and filename. ''' return self._sslKeyFile def setSSLKeyFile( self, keyFile ): ''' @parameter keyFile: A string with the SSL key path and filename. @return: None ''' self._sslKeyFile = keyFile def getSSLCertFile( self ): ''' @return: A string with the SSL cert path and filename. ''' return self._sslCertFile def setSSLCertFile( self, file ): ''' @parameter file: A string with the SSL cert path and filename. @return: None ''' self._sslCertFile = file def setTimeout( self, timeout ): om.out.debug( 'Called SetTimeout(' + str(timeout) + ')' ) if timeout > 60 or timeout < 1: raise w3afException('The timeout parameter should be between 1 and 60 seconds.') else: self._timeout = timeout # Set the default timeout # I dont need to use timeoutsocket.py , it has been added to python sockets self._socket.setdefaulttimeout(self._timeout) def getTimeout( self ): return self._timeout def setUserAgent( self, useragent ): om.out.debug( 'Called SetUserAgent') self.HeaderList = [ i for i in self.HeaderList if i[0]!='User-Agent'] self.HeaderList.append( ('User-Agent',useragent) ) self._userAgent = useragent cf.cf.save('User-Agent', useragent) def getUserAgent( self ): return self._userAgent def setProxy( self, b, ip , port): om.out.debug( 'Called SetProxy(' + b + ',' + ip + ',' + str(port) + ')') if port > 65535 or port < 1: raise w3afException('Invalid port number.') scheme, domain, path, x1, x2, x3 = self._uparse.urlparse( b ) self._proxy = { scheme : scheme + "://" + ip + ":" + str(port) } self._proxyHandler = self._ulib.ProxyHandler( self._proxy ) self._proxy = scheme + "://" + ip + ":" + str(port) def getProxy( self ): return self._proxy def setBasicAuth( self, url, username, password ): om.out.debug( 'Called SetBasicAuth') if not hasattr( self, '_password_mgr' ): # create a password manager self._password_mgr = self._ulib.HTTPPasswordMgrWithDefaultRealm() # add the username and password scheme, domain, path, x1, x2, x3 = self._uparse.urlparse( url ) self._password_mgr.add_password(None, domain, username, password) self._basicAuthHandler = self._ulib.HTTPBasicAuthHandler(self._password_mgr) # Only for w3af, no usage in urllib2 self._basicAuthStr = scheme + '://' + username + ':' + password + '@' + domain + '/' self.needUpdate = True def getBasicAuth( self ): return self._basicAuthStr def buildOpeners(self): om.out.debug( 'Called buildOpeners') ba = self._basicAuthHandler pro = self._proxyHandler if self._cookieHandler == None and not self._ignoreSessCookies: cj = self._cookielib.MozillaCookieJar() self._cookieHandler = self._ulib.HTTPCookieProcessor(cj) self._nonCacheOpener = self._ulib.build_opener( ba, pro, self._cookieHandler, \ MultipartPostHandler.MultipartPostHandler, \ self._kAHTTP, self._kAHTTPS, logHandler.logHandler,\ idHandler.idHandler, mangleHandler.mangleHandler(self._manglePlugins) ) # Prevent the urllib from putting his user-agent header self._nonCacheOpener.addheaders = [ ('Accept', '*/*') ] self._cacheOpener = self._ulib.build_opener( ba, pro, self._cookieHandler, \ localCache.CacheHandler(".urllib2cache"), \ MultipartPostHandler.MultipartPostHandler, self._kAHTTP, self._kAHTTPS,\ logHandler.logHandler,\ idHandler.idHandler, mangleHandler.mangleHandler(self._manglePlugins) ) # Prevent the urllib from putting his user-agent header self._cacheOpener.addheaders = [ ('Accept', '*/*') ] # Use this if you want to "bypass" the cache opener # debugging purposes only #self._cacheOpener = self._nonCacheOpener def getCustomUrlopen(self): return self._nonCacheOpener def getCachedUrlopen(self): return self._cacheOpener def getCfg( self ): ''' This is a faster and simpler way to call all getters. ''' result = {} result['timeout'] = self.getTimeout() result['basicAuth'] = self.getBasicAuth() result['cookie'] = self.getCookie() result['headers'] = self.getHeadersFile() result['proxy'] = self.getProxy() result['userAgent'] = self.getUserAgent() result['ignoreSessionCookies'] = self.ignoreSessionCookies return result def setManglePlugins( self, mp ): ''' Configure the mangle plugins to be used. @parameter mp: A list of mangle plugin instances. ''' self._manglePlugins = mp def getManglePlugins( self ): return self._manglePlugins def getMaxFileSize( self ): return self._maxFileSize def setMaxFileSize( self, fsize ): self._maxFileSize = 400000 def setMaxRetrys( self, retryN ): self._maxRetrys = retryN def getMaxRetrys( self ): return self._maxRetrys def getOptionsXML(self): ''' This method returns a XML containing the Options that the plugin has. Using this XML the framework will build a window, a menu, or some other input method to retrieve the info from the user. The XML has to validate against the xml schema file located at : w3af/core/ui/userInterface.dtd @return: XML with the plugin options. ''' return '\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ' def setOptions( self, OptionMap ): ''' This method sets all the options that are configured using the user interface generated by the framework using the result of getOptionsXML(). @parameter OptionMap: A dictionary with the options for the plugin. @return: No value is returned. ''' f00, OptionMap = parseOptions( 'url-settings', OptionMap ) self.setTimeout( OptionMap['timeout'] ) self.setHeadersFile( OptionMap['headersFile'] ) self.setBasicAuth( OptionMap['basicAuthUser'], OptionMap['basicAuthPass'], OptionMap['basicAuthDomain'] ) self.setCookieJarFile( OptionMap['cookieJarFile'] ) self.setProxy( 'http://localhost/', OptionMap['proxyAddress'], OptionMap['proxyPort'] ) self.setUserAgent( OptionMap['userAgent'] ) self.setMaxFileSize( OptionMap['maxFileSize'] ) self._ignoreSessCookies = OptionMap['ignoreSessCookies'] self.setMaxRetrys( OptionMap['maxRetrys'] )