diff options
| -rw-r--r-- | Lib/urllib2.py | 1063 |
1 files changed, 1063 insertions, 0 deletions
diff --git a/Lib/urllib2.py b/Lib/urllib2.py new file mode 100644 index 0000000..40a6715 --- /dev/null +++ b/Lib/urllib2.py @@ -0,0 +1,1063 @@ +"""An extensible library for opening URLs using a variety protocols + +The simplest way to use this module is to call the urlopen function, +which accepts a string containing a URL or a Request object (described +below). It opens the URL and returns the results as file-like +object; the returned object has some extra methods described below. + +The OpenerDirectory manages a collection of Handler objects that do +all the actual work. Each Handler implements a particular protocol or +option. The OpenerDirector is a composite object that invokes the +Handlers needed to open the requested URL. For example, the +HTTPHandler performs HTTP GET and POST requests and deals with +non-error returns. The HTTPRedirectHandler automatically deals with +HTTP 301 & 302 redirect errors, and the HTTPDigestAuthHandler deals +with digest authentication. + +urlopen(url, data=None) -- basic usage is that same as original +urllib. pass the url and optionally data to post to an HTTP URL, and +get a file-like object back. One difference is that you can also pass +a Request instance instead of URL. Raises a URLError (subclass of +IOError); for HTTP errors, raises an HTTPError, which can also be +treated as a valid response. + +build_opener -- function that creates a new OpenerDirector instance. +will install the default handlers. accepts one or more Handlers as +arguments, either instances or Handler classes that it will +instantiate. if one of the argument is a subclass of the default +handler, the argument will be installed instead of the default. + +install_opener -- installs a new opener as the default opener. + +objects of interest: +OpenerDirector -- + +Request -- an object that encapsulates the state of a request. the +state can be a simple as the URL. it can also include extra HTTP +headers, e.g. a User-Agent. + +BaseHandler -- + +exceptions: +URLError-- a subclass of IOError, individual protocols have their own +specific subclass + +HTTPError-- also a valid HTTP response, so you can treat an HTTP error +as an exceptional event or valid response + +internals: +BaseHandler and parent +_call_chain conventions + +Example usage: + +import urllib2 + +# set up authentication info +authinfo = urllib2.HTTPBasicAuthHandler() +authinfo.add_password('realm', 'host', 'username', 'password') + +# build a new opener that adds authentication and caching FTP handlers +opener = urllib2.build_opener(authinfo, urllib2.CacheFTPHandler) + +# install it +urllib2.install_opener(opener) + +f = urllib2.urlopen('http://www.python.org/') + + +""" + +# XXX issues: +# If an authentication error handler that tries to perform + # authentication for some reason but fails, how should the error be + # signalled? The client needs to know the HTTP error code. But if + # the handler knows that the problem was, e.g., that it didn't know + # that hash algo that requested in the challenge, it would be good to + # pass that information along to the client, too. + +# XXX to do: +# name! +# documentation (getting there) +# complex proxies +# abstract factory for opener +# ftp errors aren't handled cleanly +# gopher can return a socket.error +# check digest against correct (i.e. non-apache) implementation + +import string +import socket +import UserDict +import httplib +import re +import base64 +import types +import urlparse +import os +import md5 +import mimetypes +import mimetools +import ftplib +import sys +import time +import gopherlib + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +try: + import sha +except ImportError: + # need 1.5.2 final + sha = None + +# not sure how many of these need to be gotten rid of +from urllib import unwrap, unquote, splittype, splithost, \ + addinfourl, splitport, splitgophertype, splitquery, \ + splitattr, ftpwrapper, noheaders + +# support for proxies via environment variables +from urllib import getproxies + +# support for FileHandler +from urllib import localhost, thishost, url2pathname, pathname2url + +# support for GopherHandler +from urllib import splitgophertype, splitquery + +__version__ = "2.0a1" + +_opener = None +def urlopen(url, data=None): + global _opener + if _opener is None: + _opener = build_opener() + return _opener.open(url, data) + +def install_opener(opener): + global _opener + _opener = opener + +# do these error classes make sense? +# make sure all of the IOError stuff is overriden. we just want to be + # subtypes. + +class URLError(IOError): + # URLError is a sub-type of IOError, but it doesn't share any of + # the implementation. need to override __init__ and __str__ + def __init__(self, reason): + self.reason = reason + + def __str__(self): + return '<urlopen error %s>' % self.reason + +class HTTPError(URLError, addinfourl): + """Raised when HTTP error occurs, but also acts like non-error return""" + + def __init__(self, url, code, msg, hdrs, fp): + addinfourl.__init__(self, fp, hdrs, url) + self.code = code + self.msg = msg + self.hdrs = hdrs + self.fp = fp + # XXX + self.filename = url + + def __str__(self): + return 'HTTP Error %s: %s' % (self.code, self.msg) + + def __del__(self): + # XXX is this safe? what if user catches exception, then + # extracts fp and discards exception? + self.fp.close() + +class GopherError(URLError): + pass + +class Request: + def __init__(self, url, data=None, headers={}): + # unwrap('<URL:type://host/path>') --> 'type://host/path' + self.__original = unwrap(url) + self.type = None + # self.__r_type is what's left after doing the splittype + self.host = None + self.port = None + self.data = data + self.headers = {} + self.headers.update(headers) + + def __getattr__(self, attr): + # XXX this is a fallback mechanism to guard against these + # methods getting called in a non-standard order. this may be + # too complicated and/or unnecessary. + # XXX should the __r_XXX attributes be public? + if attr[:12] == '_Request__r_': + name = attr[12:] + if hasattr(Request, 'get_' + name): + getattr(self, 'get_' + name)() + return getattr(self, attr) + raise AttributeError, attr + + def add_data(self, data): + self.data = data + + def has_data(self): + return self.data is not None + + def get_data(self): + return self.data + + def get_full_url(self): + return self.__original + + def get_type(self): + if self.type is None: + self.type, self.__r_type = splittype(self.__original) + return self.type + + def get_host(self): + if self.host is None: + self.host, self.__r_host = splithost(self.__r_type) + if self.host: + self.host = unquote(self.host) + return self.host + + def get_selector(self): + return self.__r_host + + def set_proxy(self, proxy): + self.__proxy = proxy + # XXX this code is based on urllib, but it doesn't seem + # correct. specifically, if the proxy has a port number then + # splittype will return the hostname as the type and the port + # will be include with everything else + self.type, self.__r_type = splittype(self.__proxy) + self.host, XXX = splithost(self.__r_type) + self.host = unquote(self.host) + self.__r_host = self.__original + + def add_header(self, key, val): + # useful for something like authentication + self.headers[key] = val + +class OpenerDirector: + def __init__(self): + server_version = "Python-urllib/%s" % __version__ + self.addheaders = [('User-agent', server_version)] + # manage the individual handlers + self.handlers = [] + self.handle_open = {} + self.handle_error = {} + + def add_handler(self, handler): + added = 0 + for meth in get_methods(handler): + if meth[-5:] == '_open': + protocol = meth[:-5] + if self.handle_open.has_key(protocol): + self.handle_open[protocol].append(handler) + else: + self.handle_open[protocol] = [handler] + added = 1 + continue + i = string.find(meth, '_') + j = string.find(meth[i+1:], '_') + i + 1 + if j != -1 and meth[i+1:j] == 'error': + proto = meth[:i] + kind = meth[j+1:] + try: + kind = string.atoi(kind) + except ValueError: + pass + dict = self.handle_error.get(proto, {}) + if dict.has_key(kind): + dict[kind].append(handler) + else: + dict[kind] = [handler] + self.handle_error[proto] = dict + added = 1 + continue + if added: + self.handlers.append(handler) + handler.add_parent(self) + + def __del__(self): + self.close() + + def close(self): + for handler in self.handlers: + handler.close() + self.handlers = [] + + def _call_chain(self, chain, kind, meth_name, *args): + # XXX raise an exception if no one else should try to handle + # this url. return None if you can't but someone else could. + handlers = chain.get(kind, ()) + for handler in handlers: + func = getattr(handler, meth_name) + result = apply(func, args) + if result is not None: + return result + + def open(self, fullurl, data=None): + # accept a URL or a Request object + if type(fullurl) == types.StringType: + req = Request(fullurl, data) + else: + req = fullurl + if data is not None: + req.add_data(data) + assert isinstance(req, Request) # really only care about interface + + result = self._call_chain(self.handle_open, 'default', + 'default_open', req) + if result: + return result + + type_ = req.get_type() + result = self._call_chain(self.handle_open, type_, type_ + \ + '_open', req) + if result: + return result + + return self._call_chain(self.handle_open, 'unknown', + 'unknown_open', req) + + def error(self, proto, *args): + if proto == 'http': + # XXX http protocol is special cased + dict = self.handle_error[proto] + proto = args[2] # YUCK! + meth_name = 'http_error_%d' % proto + http_err = 1 + orig_args = args + else: + dict = self.handle_error + meth_name = proto + '_error' + http_err = 0 + args = (dict, proto, meth_name) + args + result = apply(self._call_chain, args) + if result: + return result + + if http_err: + args = (dict, 'default', 'http_error_default') + orig_args + return apply(self._call_chain, args) + +def is_callable(obj): + # not quite like builtin callable (which I didn't know existed), + # not entirely sure it needs to be different + if type(obj) in (types.BuiltinFunctionType, + types.BuiltinMethodType, types.LambdaType, + types.MethodType): + return 1 + if type(obj) == types.InstanceType: + return hasattr(obj, '__call__') + return 0 + +def get_methods(inst): + methods = {} + classes = [] + classes.append(inst.__class__) + while classes: + klass = classes[0] + del classes[0] + classes = classes + list(klass.__bases__) + for name in dir(klass): + attr = getattr(klass, name) + if type(attr) == types.UnboundMethodType: + methods[name] = 1 + for name in dir(inst): + if is_callable(getattr(inst, name)): + methods[name] = 1 + return methods.keys() + +# XXX probably also want an abstract factory that knows things like + # the fact that a ProxyHandler needs to get inserted first. +# would also know when it makes sense to skip a superclass in favor of + # a subclass and when it might make sense to include both + +def build_opener(*handlers): + """Create an opener object from a list of handlers. + + The opener will use several default handlers, including support + for HTTP and FTP. If there is a ProxyHandler, it must be at the + front of the list of handlers. (Yuck.) + + If any of the handlers passed as arguments are subclasses of the + default handlers, the default handlers will not be used. + """ + + opener = OpenerDirector() + default_classes = [ProxyHandler, UnknownHandler, HTTPHandler, + HTTPDefaultErrorHandler, HTTPRedirectHandler, + FTPHandler, FileHandler] + skip = [] + for klass in default_classes: + for check in handlers: + if type(check) == types.ClassType: + if issubclass(check, klass): + skip.append(klass) + elif type(check) == types.InstanceType: + if isinstance(check, klass): + skip.append(klass) + for klass in skip: + default_classes.remove(klass) + + for klass in default_classes: + opener.add_handler(klass()) + + for h in handlers: + if type(h) == types.ClassType: + h = h() + opener.add_handler(h) + return opener + +class BaseHandler: + def add_parent(self, parent): + self.parent = parent + def close(self): + self.parent = None + +class HTTPDefaultErrorHandler(BaseHandler): + def http_error_default(self, req, fp, code, msg, hdrs): + raise HTTPError(req.get_full_url(), code, msg, hdrs, fp) + +class HTTPRedirectHandler(BaseHandler): + # Implementation note: To avoid the server sending us into an + # infinite loop, the request object needs to track what URLs we + # have already seen. Do this by adding a handler-specific + # attribute to the Request object. + def http_error_302(self, req, fp, code, msg, headers): + if headers.has_key('location'): + newurl = headers['location'] + elif headers.has_key('uri'): + newurl = headers['uri'] + else: + return + nil = fp.read() + fp.close() + + # XXX Probably want to forget about the state of the current + # request, although that might interact poorly with other + # handlers that also use handler-specific request attributes + new = Request(newurl, req.get_data()) + new.error_302_dict = {} + if hasattr(req, 'error_302_dict'): + if req.error_302_dict.has_key(newurl): + raise HTTPError(req.get_full_url(), code, + self.inf_msg + msg, headers) + new.error_302_dict.update(req.error_302_dict) + new.error_302_dict[newurl] = newurl + return self.parent.open(new) + + http_error_301 = http_error_302 + + inf_msg = "The HTTP server returned a redirect error that would" \ + "lead to an inifinte loop.\n" \ + "The last 302 error message was:\n" + +class ProxyHandler(BaseHandler): + def __init__(self, proxies=None): + if proxies is None: + proxies = getproxies() + assert hasattr(proxies, 'has_key'), "proxies must be a mapping" + self.proxies = proxies + for type, url in proxies.items(): + setattr(self, '%s_open' % type, + lambda r, proxy=url, type=type, meth=self.proxy_open: \ + meth(r, proxy, type)) + + def proxy_open(self, req, proxy, type): + orig_type = req.get_type() + req.set_proxy(proxy) + if orig_type == type: + # let other handlers take care of it + # XXX this only makes sense if the proxy is before the + # other handlers + return None + else: + # need to start over, because the other handlers don't + # grok the proxy's URL type + return self.parent.open(req) + +# feature suggested by Duncan Booth +# XXX custom is not a good name +class CustomProxy: + # either pass a function to the constructor or override handle + def __init__(self, proto, func=None, proxy_addr=None): + self.proto = proto + self.func = func + self.addr = proxy_addr + + def handle(self, req): + if self.func and self.func(req): + return 1 + + def get_proxy(self): + return self.addr + +class CustomProxyHandler(BaseHandler): + def __init__(self, *proxies): + self.proxies = {} + + def proxy_open(self, req): + proto = req.get_type() + try: + proxies = self.proxies[proto] + except KeyError: + return None + for p in proxies: + if p.handle(req): + req.set_proxy(p.get_proxy()) + return self.parent.open(req) + return None + + def do_proxy(self, p, req): + p + return self.parent.open(req) + + def add_proxy(self, cpo): + if self.proxies.has_key(cpo.proto): + self.proxies[cpo.proto].append(cpo) + else: + self.proxies[cpo.proto] = [cpo] + +class HTTPPasswordMgr: + def __init__(self): + self.passwd = {} + + def add_password(self, realm, uri, user, passwd): + # uri could be a single URI or a sequence + if type(uri) == types.StringType: + uri = [uri] + uri = tuple(map(self.reduce_uri, uri)) + if not self.passwd.has_key(realm): + self.passwd[realm] = {} + self.passwd[realm][uri] = (user, passwd) + + def find_user_password(self, realm, authuri): + domains = self.passwd.get(realm, {}) + authuri = self.reduce_uri(authuri) + for uris, authinfo in domains.items(): + for uri in uris: + if self.is_suburi(uri, authuri): + return authinfo + return None, None + + def reduce_uri(self, uri): + """Accept netloc or URI and extract only the netloc and path""" + parts = urlparse.urlparse(uri) + if parts[1]: + return parts[1], parts[2] or '/' + else: + return parts[2], '/' + + def is_suburi(self, base, test): + """Check if test is below base in a URI tree + + Both args must be URIs in reduced form. + """ + if base == test: + return 1 + if base[0] != test[0]: + return 0 + common = os.path.commonprefix((base[1], test[1])) + if len(common) == len(base[1]): + return 1 + return 0 + + +class HTTPBasicAuthHandler(BaseHandler): + rx = re.compile('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"') + + # XXX there can actually be multiple auth-schemes in a + # www-authenticate header. should probably be a lot more careful + # in parsing them to extract multiple alternatives + + def __init__(self): + self.passwd = HTTPPasswordMgr() + self.add_password = self.passwd.add_password + self.__current_realm = None + # if __current_realm is not None, then the server must have + # refused our name/password and is asking for authorization + # again. must be careful to set it to None on successful + # return. + + def http_error_401(self, req, fp, code, msg, headers): + # XXX could be mult. headers + authreq = headers.get('www-authenticate', None) + if authreq: + mo = HTTPBasicAuthHandler.rx.match(authreq) + if mo: + scheme, realm = mo.groups() + if string.lower(scheme) == 'basic': + return self.retry_http_basic_auth(req, realm) + + def retry_http_basic_auth(self, req, realm): + if self.__current_realm is None: + self.__current_realm = realm + else: + self.__current_realm = realm + return None + # XXX host isn't really the correct URI? + host = req.get_host() + user,pw = self.passwd.find_user_password(realm, host) + if pw: + raw = "%s:%s" % (user, pw) + auth = string.strip(base64.encodestring(raw)) + req.add_header('Authorization', 'Basic %s' % auth) + resp = self.parent.open(req) + self.__current_realm = None + return resp + else: + self.__current_realm = None + return None + +class HTTPDigestAuthHandler(BaseHandler): + """An authentication protocol defined by RFC 2069 + + Digest authentication improves on basic authentication because it + does not transmit passwords in the clear. + """ + + def __init__(self): + self.passwd = HTTPPasswordMgr() + self.add_password = self.passwd.add_password + self.__current_realm = None + + def http_error_401(self, req, fp, code, msg, headers): + # XXX could be mult. headers + authreq = headers.get('www-authenticate', None) + if authreq: + kind = string.split(authreq)[0] + if kind == 'Digest': + return self.retry_http_digest_auth(req, authreq) + + def retry_http_digest_auth(self, req, auth): + token, challenge = string.split(auth, ' ', 1) + chal = parse_keqv_list(parse_http_list(challenge)) + auth = self.get_authorization(req, chal) + if auth: + req.add_header('Authorization', 'Digest %s' % auth) + resp = self.parent.open(req) + self.__current_realm = None + return resp + + def get_authorization(self, req, chal): + try: + realm = chal['realm'] + nonce = chal['nonce'] + algorithm = chal.get('algorithm', 'MD5') + # mod_digest doesn't send an opaque, even though it isn't + # supposed to be optional + opaque = chal.get('opaque', None) + except KeyError: + return None + + if self.__current_realm is None: + self.__current_realm = realm + else: + self.__current_realm = realm + return None + + H, KD = self.get_algorithm_impls(algorithm) + if H is None: + return None + + user, pw = self.passwd.find_user_password(realm, + req.get_full_url()) + if user is None: + return None + + # XXX not implemented yet + if req.has_data(): + entdig = self.get_entity_digest(req.get_data(), chal) + else: + entdig = None + + A1 = "%s:%s:%s" % (user, realm, pw) + A2 = "%s:%s" % (req.has_data() and 'POST' or 'GET', + # XXX selector: what about proxies and full urls + req.get_selector()) + respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) + # XXX should the partial digests be encoded too? + + base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \ + 'response="%s"' % (user, realm, nonce, req.get_selector(), + respdig) + if opaque: + base = base + ', opaque="%s"' % opaque + if entdig: + base = base + ', digest="%s"' % entdig + if algorithm != 'MD5': + base = base + ', algorithm="%s"' % algorithm + return base + + def get_algorithm_impls(self, algorithm): + # lambdas assume digest modules are imported at the top level + if algorithm == 'MD5': + H = lambda x, e=encode_digest:e(md5.new(x).digest()) + elif algorithm == 'SHA': + H = lambda x, e=encode_digest:e(sha.new(x).digest()) + # XXX MD5-sess + KD = lambda s, d, H=H: H("%s:%s" % (s, d)) + return H, KD + + def get_entity_digest(self, data, chal): + # XXX not implemented yet + return None + +def encode_digest(digest): + hexrep = [] + for c in digest: + n = (ord(c) >> 4) & 0xf + hexrep.append(hex(n)[-1]) + n = ord(c) & 0xf + hexrep.append(hex(n)[-1]) + return string.join(hexrep, '') + + +class HTTPHandler(BaseHandler): + def http_open(self, req): + # XXX devise a new mechanism to specify user/password + host = req.get_host() + if not host: + raise URLError('no host given') + + h = httplib.HTTP(host) # will parse host:port +## h.set_debuglevel(1) + if req.has_data(): + data = req.get_data() + h.putrequest('POST', req.get_selector()) + h.putheader('Content-type', 'application/x-www-form-urlencoded') + h.putheader('Content-length', '%d' % len(data)) + else: + h.putrequest('GET', req.get_selector()) + # XXX proxies would have different host here + h.putheader('Host', host) + for args in self.parent.addheaders: + apply(h.putheader, args) + for k, v in req.headers.items(): + h.putheader(k, v) + h.endheaders() + if req.has_data(): + h.send(data + '\r\n') + + code, msg, hdrs = h.getreply() + fp = h.getfile() + if code == 200: + return addinfourl(fp, hdrs, req.get_full_url()) + else: + # want to make sure the socket is closed, even if error + # handling doesn't return immediately. the socket won't + # actually be closed until fp is also closed. + if h.sock: + h.sock.close() + h.sock = None + return self.parent.error('http', req, fp, code, msg, hdrs) + +class UnknownHandler(BaseHandler): + def unknown_open(self, req): + type = req.get_type() + raise URLError('unknown url type: %s' % type) + +def parse_keqv_list(l): + """Parse list of key=value strings where keys are not duplicated.""" + parsed = {} + for elt in l: + k, v = string.split(elt, '=', 1) + if v[0] == '"' and v[-1] == '"': + v = v[1:-1] + parsed[k] = v + return parsed + +def parse_http_list(s): + """Parse lists as described by RFC 2068 Section 2. + + In particular, parse comman-separated lists where the elements of + the list may include quoted-strings. A quoted-string could + contain a comma. + """ + # XXX this function could probably use more testing + + list = [] + end = len(s) + i = 0 + inquote = 0 + start = 0 + while i < end: + cur = s[i:] + c = string.find(cur, ',') + q = string.find(cur, '"') + if c == -1: + list.append(s[start:]) + break + if q == -1: + if inquote: + raise ValueError, "unbalanced quotes" + else: + list.append(s[start:i+c]) + i = i + c + 1 + continue + if inquote: + if q < c: + list.append(s[start:i+c]) + i = i + c + 1 + start = i + inquote = 0 + else: + i = i + q + else: + if c < q: + list.append(s[start:i+c]) + i = i + c + 1 + start = i + else: + inquote = 1 + i = i + q + 1 + return map(string.strip, list) + +class FileHandler(BaseHandler): + # Use local file or FTP depending on form of URL + def file_open(self, req): + url = req.get_selector() + if url[:2] == '//' and url[2:3] != '/': + req.type = 'ftp' + return self.parent.open(req) + else: + return self.open_local_file(req) + + # names for the localhost + names = None + def get_names(self): + if FileHandler.names is None: + FileHandler.names = (socket.gethostbyname('localhost'), + socket.gethostbyname(socket.gethostname())) + return FileHandler.names + + # not entirely sure what the rules are here + def open_local_file(self, req): + mtype = mimetypes.guess_type(req.get_selector())[0] + headers = mimetools.Message(StringIO('Content-Type: %s\n' \ + % (mtype or 'text/plain'))) + host = req.get_host() + file = req.get_selector() + if host: + host, port = splitport(host) + if not host or \ + (not port and socket.gethostbyname(host) in self.get_names()): + return addinfourl(open(url2pathname(file), 'rb'), + headers, 'file:'+file) + raise URLError('file not on local host') + +class FTPHandler(BaseHandler): + def ftp_open(self, req): + host = req.get_host() + if not host: + raise IOError, ('ftp error', 'no host given') + # XXX handle custom username & password + host = socket.gethostbyname(host) + host, port = splitport(host) + if port is None: + port = ftplib.FTP_PORT + path, attrs = splitattr(req.get_selector()) + path = unquote(path) + dirs = string.splitfields(path, '/') + dirs, file = dirs[:-1], dirs[-1] + if dirs and not dirs[0]: + dirs = dirs[1:] + user = passwd = '' # XXX + try: + fw = self.connect_ftp(user, passwd, host, port, dirs) + type = file and 'I' or 'D' + for attr in attrs: + attr, value = splitattr(attr) + if string.lower(attr) == 'type' and \ + value in ('a', 'A', 'i', 'I', 'd', 'D'): + type = string.upper(value) + fp, retrlen = fw.retrfile(file, type) + if retrlen is not None and retrlen >= 0: + sf = StringIO('Content-Length: %d\n' % retrlen) + headers = mimetools.Message(sf) + else: + headers = noheaders() + return addinfourl(fp, headers, req.get_full_url()) + except ftplib.all_errors, msg: + raise IOError, ('ftp error', msg), sys.exc_info()[2] + + def connect_ftp(self, user, passwd, host, port, dirs): + fw = ftpwrapper(user, passwd, host, port, dirs) +## fw.ftp.set_debuglevel(1) + return fw + +class CacheFTPHandler(FTPHandler): + # XXX would be nice to have pluggable cache strategies + # XXX this stuff is definitely not thread safe + def __init__(self): + self.cache = {} + self.timeout = {} + self.soonest = 0 + self.delay = 60 + self.max_conns = 16 + + def setTimeout(self, t): + self.delay = t + + def setMaxConns(self, m): + self.max_conns = m + + def connect_ftp(self, user, passwd, host, port, dirs): + key = user, passwd, host, port + if self.cache.has_key(key): + self.timeout[key] = time.time() + self.delay + else: + self.cache[key] = ftpwrapper(user, passwd, host, port, dirs) + self.timeout[key] = time.time() + self.delay + self.check_cache() + return self.cache[key] + + def check_cache(self): + # first check for old ones + t = time.time() + if self.soonest <= t: + for k, v in self.timeout.items(): + if v < t: + self.cache[k].close() + del self.cache[k] + del self.timeout[k] + self.soonest = min(self.timeout.values()) + + # then check the size + if len(self.cache) == self.max_conns: + for k, v in self.timeout.items(): + if v == self.soonest: + del self.cache[k] + del self.timeout[k] + break + self.soonest = min(self.timeout.values()) + +class GopherHandler(BaseHandler): + def gopher_open(self, req): + host = req.get_host() + if not host: + raise GopherError('no host given') + host = unquote(host) + selector = req.get_selector() + type, selector = splitgophertype(selector) + selector, query = splitquery(selector) + selector = unquote(selector) + if query: + query = unquote(query) + fp = gopherlib.send_query(selector, query, host) + else: + fp = gopherlib.send_selector(selector, host) + return addinfourl(fp, noheaders(), req.get_full_url()) + +#bleck! don't use this yet +class OpenerFactory: + + default_handlers = [UnknownHandler, HTTPHandler, + HTTPDefaultErrorHandler, HTTPRedirectHandler, + FTPHandler, FileHandler] + proxy_handlers = [ProxyHandler] + handlers = [] + replacement_handlers = [] + + def add_proxy_handler(self, ph): + self.proxy_handlers = self.proxy_handlers + [ph] + + def add_handler(self, h): + self.handlers = self.handlers + [h] + + def replace_handler(self, h): + pass + + def build_opener(self): + opener = OpenerDirectory() + for ph in self.proxy_handlers: + if type(ph) == types.ClassType: + ph = ph() + opener.add_handler(ph) + +if __name__ == "__main__": + # XXX some of the test code depends on machine configurations that + # are internal to CNRI. Need to set up a public server with the + # right authentication configuration for test purposes. + if socket.gethostname() == 'bitdiddle': + localhost = 'bitdiddle.cnri.reston.va.us' + elif socket.gethostname() == 'walden': + localhost = 'localhost' + else: + localhost = None + urls = [ + # Thanks to Fred for finding these! + 'gopher://gopher.lib.ncsu.edu/11/library/stacks/Alex', + 'gopher://gopher.vt.edu:10010/10/33', + + 'file:/etc/passwd', + 'file://nonsensename/etc/passwd', + 'ftp://www.python.org/pub/tmp/httplib.py', + 'ftp://www.python.org/pub/tmp/imageop.c', + 'ftp://www.python.org/pub/tmp/blat', + 'http://www.espn.com/', # redirect + 'http://www.python.org/Spanish/Inquistion/', + ('http://grail.cnri.reston.va.us/cgi-bin/faqw.py', + 'query=pythonistas&querytype=simple&casefold=yes&req=search'), + 'http://www.python.org/', + 'ftp://prep.ai.mit.edu/welcome.msg', + 'ftp://www.python.org/pub/tmp/figure.prn', + 'ftp://www.python.org/pub/tmp/interp.pl', + 'http://checkproxy.cnri.reston.va.us/test/test.html', + ] + + if localhost is not None: + urls = urls + [ + 'file://%s/etc/passwd' % localhost, + 'http://%s/simple/' % localhost, + 'http://%s/digest/' % localhost, + 'http://%s/not/found.h' % localhost, + ] + + bauth = HTTPBasicAuthHandler() + bauth.add_password('basic_test_realm', localhost, 'jhylton', + 'password') + dauth = HTTPDigestAuthHandler() + dauth.add_password('digest_test_realm', localhost, 'jhylton', + 'password') + + + cfh = CacheFTPHandler() + cfh.setTimeout(1) + + # XXX try out some custom proxy objects too! + def at_cnri(req): + host = req.get_host() + print host + if host[-18:] == '.cnri.reston.va.us': + return 1 + p = CustomProxy('http', at_cnri, 'proxy.cnri.reston.va.us') + ph = CustomProxyHandler(p) + + install_opener(build_opener(dauth, bauth, cfh, GopherHandler, ph)) + + for url in urls: + if type(url) == types.TupleType: + url, req = url + else: + req = None + print url + try: + f = urlopen(url, req) + except IOError, err: + print "IOError:", err + except socket.error, err: + print "socket.error:", err + else: + buf = f.read() + f.close() + print "read %d bytes" % len(buf) + print + time.sleep(0.1) |