""" httplib_async.py Asynchronous HTTP client. """ __copyright__ = "Copyright (c) 2002-2005 Free Software Foundation, Inc." __license__ = """ GNU General Public License This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. """ import asyncore import socket import mimetools try: from cStringIO import StringIO except: from StringIO import StringIO import sys from error import log class HTTPConnection_async(asyncore.dispatcher_with_send): # asynchronous http client def __init__(self, host, ip, port, consumer, sock=None, map=None, proxy=None): asyncore.dispatcher_with_send.__init__(self) if ip is None: ip = host self.ip = ip self.host = host self.port = port self.consumer = consumer self.status = None self.http_version = "HTTP/1.1" self.proxy = None self.proxy_port = None if proxy: self.proxy = proxy[0] self.proxy_port = proxy[1] def set_proxy(self, proxy): self.proxy = proxy[0] self.proxy_port = proxy[1] def request(self, method, path, headers={}, data=None): self.header = None self.data = "" send_data = [] self.method = method or "GET" self.path = path or "/" if self.path: if self.path[0] != "/": self.path = "/" + self.path else: self.path = "/" if self.proxy and self.proxy_port: self.path = "http://%s%s" % (self.host, path) self._initialize_chunked() send_data.append("%s %s %s\r\n" % (self.method, self.path, self.http_version)) host_found = 0 cl_found = 0 for h, v in headers.items(): h = h.lower() if h == 'host': host_found = 1 if h == 'content-length': cl_found = 1 send_data.append("%s: %s\r\n" % (h, v)) if not host_found: send_data.append("Host: %s\r\n" % self.host) if data and not cl_found: send_data.append("Content-length: %d\r\n" % len(data)) send_data.append("\r\n") if data: send_data.append(data) self.send_data = "".join(send_data) if self.connected: self.send(self.send_data) def execute(self): self.create_socket(socket.AF_INET, socket.SOCK_STREAM) if self.proxy and self.proxy_port: self.connect((self.proxy, self.proxy_port)) else: self.connect((self.ip, self.port)) def handle_connect(self): # connection succeeded self.send(self.send_data) def handle_expt(self): # This is used for OOB data. log("OOB data received") def handle_error(self): exctype, value = sys.exc_info()[:2] self.close() self.consumer.http_failed(value) def _initialize_chunked(self, chunked=0): self.chunked = chunked self.chunk_left = None self.chunk_data = [] self.chunks_done = 0 def handle_read(self): data = self.recv(4096) self.data += data if not self.header: # check if we've seen a full header header = self.data.split("\r\n\r\n", 1) if len(header) <= 1: # i guess not everyone can read the spec - congrats go to # thinkgeek header = self.data.split("\n\n", 1) if len(header) <= 1: return header, data = header # parse header fp = StringIO(header) line = fp.readline() try: self.status = line.split(" ", 2) self.status[1] = int(self.status[1]) except ValueError: self.consumer.http_failed(BadStatusException(line)) self.header = mimetools.Message(fp) self.data = data if self.status[1] in (301, 302): self.consumer.http_redirect(self.header.getheader('location'),self.status[1] == 301) self.close() return elif self.header.getheader('transfer-encoding', '').lower() == 'chunked': self._initialize_chunked(1) else: self.consumer.http_header(self.status, self.header) if not self.connected: return # channel was closed by consumer if self.chunked: # we are reading chunked data, read it all and the possible # extra headers after that while not self.chunks_done and len(self.data) > 0: if self.chunk_left is None or self.chunk_left < 0: # one chunk over, read the length of the next one if self.data[:2] == '\r\n': self.data = self.data[2:] cdata = self.data.split("\r\n", 1) if len(cdata) <= 1: return line, self.data = cdata i = line.find(';') if i >= 0: line = line[:i] self.chunk_left = int(line, 16) if self.chunk_left == 0: # no more chunks, put back CRLF to help things later self.data = "\r\n" + self.data self.chunks_done = 1 break self.chunk_data.append(self.data[:self.chunk_left]) l = len(self.data) self.data = self.data[self.chunk_left:] self.chunk_left -= l if self.chunks_done: cdata = self.data.split("\r\n\r\n", 1) if len(cdata) <= 1: return headerlines = cdata[0][2:] fp = StringIO(headerlines) header = mimetools.Message(fp) chunk_data = "".join(self.chunk_data) for h, v in header.items(): self.header[h] = v if not self.header.has_key('content-length'): self.header['content-length'] = str(len(chunk_data)) del self.header['transfer-encoding'] self.consumer.http_header(self.status, self.header) self.consumer.feed(chunk_data) return elif data: self.consumer.feed(data) def handle_close(self): self.close() self.consumer.http_close() self.consumer = None class HTTPException(Exception): pass class BadStatusException(HTTPException): def __init__(self, line): HTTPException.__init__(self) self.args = line self.line = line