diff options
| -rw-r--r-- | .travis.yml | 6 | ||||
| -rw-r--r-- | eventlet/green/http/__init__.py | 2 | ||||
| -rw-r--r-- | eventlet/green/http/client.py | 9 | ||||
| -rw-r--r-- | eventlet/green/http/cookiejar.py | 13 | ||||
| -rw-r--r-- | eventlet/green/http/cookies.py | 7 | ||||
| -rw-r--r-- | eventlet/green/http/server.py | 17 | ||||
| -rw-r--r-- | eventlet/green/selectors.py | 11 | ||||
| -rw-r--r-- | eventlet/green/subprocess.py | 12 | ||||
| -rw-r--r-- | eventlet/green/urllib.py | 38 | ||||
| -rw-r--r-- | eventlet/green/urllib/__init__.py | 40 | ||||
| -rw-r--r-- | eventlet/green/urllib/error.py | 4 | ||||
| -rw-r--r-- | eventlet/green/urllib/parse.py | 3 | ||||
| -rw-r--r-- | eventlet/green/urllib/request.py | 44 | ||||
| -rw-r--r-- | eventlet/green/urllib/response.py | 3 | ||||
| -rw-r--r-- | eventlet/greenio/__init__.py | 8 | ||||
| -rw-r--r-- | eventlet/greenio/base.py (renamed from eventlet/greenio.py) | 238 | ||||
| -rw-r--r-- | eventlet/greenio/py2.py | 222 | ||||
| -rw-r--r-- | eventlet/greenio/py3.py | 191 | ||||
| -rw-r--r-- | eventlet/wsgi.py | 30 | ||||
| -rw-r--r-- | tests/greenio_test.py | 12 | ||||
| -rw-r--r-- | tests/patcher_test.py | 14 | ||||
| -rw-r--r-- | tests/test__greenness.py | 15 | ||||
| -rw-r--r-- | tests/wsgi_test.py | 32 | ||||
| -rw-r--r-- | tests/wsgi_test_conntimeout.py | 4 |
24 files changed, 655 insertions, 320 deletions
diff --git a/.travis.yml b/.travis.yml index f106ff2..e3ea872 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,12 +20,6 @@ env: matrix: fast_finish: true allow_failures: - - env: TOX_ENV=py33-selects - - env: TOX_ENV=py33-poll - - env: TOX_ENV=py33-epolls - - env: TOX_ENV=py34-selects - - env: TOX_ENV=py34-poll - - env: TOX_ENV=py34-epolls - env: TOX_ENV=pypy-selects - env: TOX_ENV=pypy-poll - env: TOX_ENV=pypy-epolls diff --git a/eventlet/green/http/__init__.py b/eventlet/green/http/__init__.py new file mode 100644 index 0000000..c9e2a23 --- /dev/null +++ b/eventlet/green/http/__init__.py @@ -0,0 +1,2 @@ +from eventlet.support import six +assert six.PY3, 'This is a Python 3 module' diff --git a/eventlet/green/http/client.py b/eventlet/green/http/client.py new file mode 100644 index 0000000..480a252 --- /dev/null +++ b/eventlet/green/http/client.py @@ -0,0 +1,9 @@ +from eventlet import patcher +from eventlet.green import os, socket +from eventlet.green.urllib import parse as urllib_parse + +patcher.inject('http.client', globals(), + ('os', os), ('socket', socket), ('urllib.parse', urllib_parse)) + +del patcher +del urllib_parse diff --git a/eventlet/green/http/cookiejar.py b/eventlet/green/http/cookiejar.py new file mode 100644 index 0000000..5e511d2 --- /dev/null +++ b/eventlet/green/http/cookiejar.py @@ -0,0 +1,13 @@ +from eventlet.green import threading, time +from eventlet.green.http import client +from eventlet.green.urllib import parse as urllib_parse, request as urllib_request +from eventlet import patcher + +patcher.inject('http.cookiejar', globals(), + ('http.client', client), ('threading', threading), + ('urllib.parse', urllib_parse), ('urllib.request', urllib_request), + ('time', time)) + +del urllib_request +del urllib_parse +del patcher diff --git a/eventlet/green/http/cookies.py b/eventlet/green/http/cookies.py new file mode 100644 index 0000000..e139069 --- /dev/null +++ b/eventlet/green/http/cookies.py @@ -0,0 +1,7 @@ +from eventlet import patcher +from eventlet.green import time + +patcher.inject('http.cookies', globals()) +_getdate = patcher.patch_function(_getdate, ('time', time)) + +del patcher diff --git a/eventlet/green/http/server.py b/eventlet/green/http/server.py new file mode 100644 index 0000000..35c3ab2 --- /dev/null +++ b/eventlet/green/http/server.py @@ -0,0 +1,17 @@ +from eventlet import patcher +from eventlet.green import os, time, select, socket, SocketServer, subprocess +from eventlet.green.http import client +from eventlet.green.urllib import parse as urllib_parse + +patcher.inject('http.server', globals(), + ('http.client', client), ('os', os), ('select', select), + ('socket', socket), ('socketserver', SocketServer), ('time', time), + ('urllib.parse', urllib_parse)) + + +CGIHTTPRequestHandler.run_cgi = patcher.patch_function( + CGIHTTPRequestHandler.run_cgi, ('subprocess', subprocess)) + +del urllib_parse +del client +del patcher diff --git a/eventlet/green/selectors.py b/eventlet/green/selectors.py new file mode 100644 index 0000000..26427ec --- /dev/null +++ b/eventlet/green/selectors.py @@ -0,0 +1,11 @@ +import sys + +from eventlet import patcher +from eventlet.green import select + +patcher.inject('selectors', globals(), ('select', select)) + +del patcher + +if sys.platform != 'win32': + SelectSelector._select = staticmethod(select.select) diff --git a/eventlet/green/subprocess.py b/eventlet/green/subprocess.py index 1d7c49a..7ce38cf 100644 --- a/eventlet/green/subprocess.py +++ b/eventlet/green/subprocess.py @@ -1,15 +1,21 @@ import errno -import time +import sys from types import FunctionType import eventlet from eventlet import greenio from eventlet import patcher -from eventlet.green import select +from eventlet.green import select, threading, time from eventlet.support import six -patcher.inject('subprocess', globals(), ('select', select)) +to_patch = [('select', select), ('threading', threading), ('time', time)] + +if sys.version_info > (3, 4): + from eventlet.green import selectors + to_patch.append(('selectors', selectors)) + +patcher.inject('subprocess', globals(), *to_patch) subprocess_orig = __import__("subprocess") diff --git a/eventlet/green/urllib.py b/eventlet/green/urllib.py deleted file mode 100644 index f5c8f13..0000000 --- a/eventlet/green/urllib.py +++ /dev/null @@ -1,38 +0,0 @@ -from eventlet import patcher -from eventlet.green import socket -from eventlet.green import time -from eventlet.green import httplib -from eventlet.green import ftplib - -to_patch = [('socket', socket), ('httplib', httplib), - ('time', time), ('ftplib', ftplib)] -try: - from eventlet.green import ssl - to_patch.append(('ssl', ssl)) -except ImportError: - pass - -patcher.inject('urllib', globals(), *to_patch) -try: - URLopener -except NameError: - patcher.inject('urllib.request', globals(), *to_patch) - - -# patch a bunch of things that have imports inside the -# function body; this is lame and hacky but I don't feel -# too bad because urllib is a hacky pile of junk that no -# one should be using anyhow -URLopener.open_http = patcher.patch_function(URLopener.open_http, ('httplib', httplib)) -if hasattr(URLopener, 'open_https'): - URLopener.open_https = patcher.patch_function(URLopener.open_https, ('httplib', httplib)) - -URLopener.open_ftp = patcher.patch_function(URLopener.open_ftp, ('ftplib', ftplib)) -ftpwrapper.init = patcher.patch_function(ftpwrapper.init, ('ftplib', ftplib)) -ftpwrapper.retrfile = patcher.patch_function(ftpwrapper.retrfile, ('ftplib', ftplib)) - -del patcher - -# Run test program when run as a script -if __name__ == '__main__': - main() diff --git a/eventlet/green/urllib/__init__.py b/eventlet/green/urllib/__init__.py new file mode 100644 index 0000000..7cb4ea6 --- /dev/null +++ b/eventlet/green/urllib/__init__.py @@ -0,0 +1,40 @@ +from eventlet import patcher +from eventlet.green import socket +from eventlet.green import time +from eventlet.green import httplib +from eventlet.green import ftplib +from eventlet.support import six + +if six.PY2: + to_patch = [('socket', socket), ('httplib', httplib), + ('time', time), ('ftplib', ftplib)] + try: + from eventlet.green import ssl + to_patch.append(('ssl', ssl)) + except ImportError: + pass + + patcher.inject('urllib', globals(), *to_patch) + try: + URLopener + except NameError: + patcher.inject('urllib.request', globals(), *to_patch) + + + # patch a bunch of things that have imports inside the + # function body; this is lame and hacky but I don't feel + # too bad because urllib is a hacky pile of junk that no + # one should be using anyhow + URLopener.open_http = patcher.patch_function(URLopener.open_http, ('httplib', httplib)) + if hasattr(URLopener, 'open_https'): + URLopener.open_https = patcher.patch_function(URLopener.open_https, ('httplib', httplib)) + + URLopener.open_ftp = patcher.patch_function(URLopener.open_ftp, ('ftplib', ftplib)) + ftpwrapper.init = patcher.patch_function(ftpwrapper.init, ('ftplib', ftplib)) + ftpwrapper.retrfile = patcher.patch_function(ftpwrapper.retrfile, ('ftplib', ftplib)) + + del patcher + + # Run test program when run as a script + if __name__ == '__main__': + main() diff --git a/eventlet/green/urllib/error.py b/eventlet/green/urllib/error.py new file mode 100644 index 0000000..6913813 --- /dev/null +++ b/eventlet/green/urllib/error.py @@ -0,0 +1,4 @@ +from eventlet import patcher +from eventlet.green.urllib import response +patcher.inject('urllib.error', globals(), ('urllib.response', response)) +del patcher diff --git a/eventlet/green/urllib/parse.py b/eventlet/green/urllib/parse.py new file mode 100644 index 0000000..f3a8924 --- /dev/null +++ b/eventlet/green/urllib/parse.py @@ -0,0 +1,3 @@ +from eventlet import patcher +patcher.inject('urllib.parse', globals()) +del patcher diff --git a/eventlet/green/urllib/request.py b/eventlet/green/urllib/request.py new file mode 100644 index 0000000..8160bb9 --- /dev/null +++ b/eventlet/green/urllib/request.py @@ -0,0 +1,44 @@ +from eventlet import patcher +from eventlet.green import ftplib, os, socket, time +from eventlet.green.http import client as http_client +from eventlet.green.urllib import error, parse, response + +# TODO should we also have green email version? +# import email + + +to_patch = [ + ('http.client', http_client), + ('os', os), + ('socket', socket), + ('time', time), + ('urllib.error', error), + ('urllib.parse', parse), + ('urllib.response', response), +] + +try: + from eventlet.green import ssl +except ImportError: + pass +else: + to_patch.append(('ssl', ssl)) + +patcher.inject('urllib.request', globals(), *to_patch) +del to_patch + +to_patch_in_functions = [('ftplib', ftplib)] +del ftplib + +FTPHandler.ftp_open = patcher.patch_function(FTPHandler.ftp_open, *to_patch_in_functions) +URLopener.open_ftp = patcher.patch_function(URLopener.open_ftp, *to_patch_in_functions) + +ftperrors = patcher.patch_function(ftperrors, *to_patch_in_functions) + +ftpwrapper.init = patcher.patch_function(ftpwrapper.init, *to_patch_in_functions) +ftpwrapper.retrfile = patcher.patch_function(ftpwrapper.retrfile, *to_patch_in_functions) + +del error +del parse +del response +del to_patch_in_functions diff --git a/eventlet/green/urllib/response.py b/eventlet/green/urllib/response.py new file mode 100644 index 0000000..f9aaba5 --- /dev/null +++ b/eventlet/green/urllib/response.py @@ -0,0 +1,3 @@ +from eventlet import patcher +patcher.inject('urllib.response', globals()) +del patcher diff --git a/eventlet/greenio/__init__.py b/eventlet/greenio/__init__.py new file mode 100644 index 0000000..72cd33e --- /dev/null +++ b/eventlet/greenio/__init__.py @@ -0,0 +1,8 @@ +from eventlet.support import six + +from eventlet.greenio.base import * # noqa + +if six.PY2: + from eventlet.greenio.py2 import * # noqa +else: + from eventlet.greenio.py3 import * # noqa diff --git a/eventlet/greenio.py b/eventlet/greenio/base.py index 46e3909..8da51ca 100644 --- a/eventlet/greenio.py +++ b/eventlet/greenio/base.py @@ -7,9 +7,13 @@ import time import warnings from eventlet.support import get_errno, six -from eventlet.hubs import trampoline, notify_close, notify_opened, IOClosed +from eventlet.hubs import trampoline, notify_opened, IOClosed -__all__ = ['GreenSocket', 'GreenPipe', 'shutdown_safe'] +__all__ = [ + 'GreenSocket', '_GLOBAL_DEFAULT_TIMEOUT', 'set_nonblocking', + 'SOCKET_CLOSED', 'CONNECT_ERR', 'CONNECT_SUCCESS', + 'shutdown_safe', 'SSL', +] BUFFER_SIZE = 4096 CONNECT_ERR = set((errno.EINPROGRESS, errno.EALREADY, errno.EWOULDBLOCK)) @@ -17,11 +21,8 @@ CONNECT_SUCCESS = set((0, errno.EISCONN)) if sys.platform[:3] == "win": CONNECT_ERR.add(errno.WSAEINVAL) # Bug 67 -if six.PY3: - from io import IOBase as file - _fileobject = socket.SocketIO -elif six.PY2: - _fileobject = socket._fileobject +if six.PY2: + _python2_fileobject = socket._fileobject def socket_connect(descriptor, address): @@ -293,7 +294,7 @@ class GreenSocket(object): else: def makefile(self, *args, **kwargs): dupped = self.dup() - res = _fileobject(dupped, *args, **kwargs) + res = _python2_fileobject(dupped, *args, **kwargs) if hasattr(dupped, "_drop"): dupped._drop() return res @@ -418,119 +419,11 @@ class GreenSocket(object): getattr(self.fd, '_sock', self.fd)._drop() -class _SocketDuckForFd(object): - """Class implementing all socket method used by _fileobject - in cooperative manner using low level os I/O calls. - """ - _refcount = 0 - - def __init__(self, fileno): - self._fileno = fileno - notify_opened(fileno) - self._closed = False - - def _trampoline(self, fd, read=False, write=False, timeout=None, timeout_exc=None): - if self._closed: - # Don't trampoline if we're already closed. - raise IOClosed() - try: - return trampoline(fd, read=read, write=write, timeout=timeout, - timeout_exc=timeout_exc, - mark_as_closed=self._mark_as_closed) - except IOClosed: - # Our fileno has been obsoleted. Defang ourselves to - # prevent spurious closes. - self._mark_as_closed() - raise - - def _mark_as_closed(self): - self._closed = True - - @property - def _sock(self): - return self - - def fileno(self): - return self._fileno - - def recv(self, buflen): - while True: - try: - data = os.read(self._fileno, buflen) - return data - except OSError as e: - if get_errno(e) not in SOCKET_BLOCKING: - raise IOError(*e.args) - self._trampoline(self, read=True) - - def recv_into(self, buf, nbytes=0, flags=0): - if nbytes == 0: - nbytes = len(buf) - data = self.recv(nbytes) - buf[:nbytes] = data - return len(data) - - def send(self, data): - while True: - try: - return os.write(self._fileno, data) - except OSError as e: - if get_errno(e) not in SOCKET_BLOCKING: - raise IOError(*e.args) - else: - trampoline(self, write=True) - - def sendall(self, data): - len_data = len(data) - os_write = os.write - fileno = self._fileno - try: - total_sent = os_write(fileno, data) - except OSError as e: - if get_errno(e) != errno.EAGAIN: - raise IOError(*e.args) - total_sent = 0 - while total_sent < len_data: - self._trampoline(self, write=True) - try: - total_sent += os_write(fileno, data[total_sent:]) - except OSError as e: - if get_errno(e) != errno. EAGAIN: - raise IOError(*e.args) - - def __del__(self): - self._close() - - def _close(self): - notify_close(self._fileno) - self._mark_as_closed() - try: - os.close(self._fileno) - except: - # os.close may fail if __init__ didn't complete - # (i.e file dscriptor passed to popen was invalid - pass - - def __repr__(self): - return "%s:%d" % (self.__class__.__name__, self._fileno) - - def _reuse(self): - self._refcount += 1 - - def _drop(self): - self._refcount -= 1 - if self._refcount == 0: - self._close() - # Python3 - _decref_socketios = _drop - - -def _operationOnClosedFile(*args, **kwargs): +def _operation_on_closed_file(*args, **kwargs): raise ValueError("I/O operation on closed file") -class GreenPipe(_fileobject): - """ +greenpipe_doc = """ GreenPipe is a cooperative replacement for file class. It will cooperate on pipes. It will block on regular file. Differneces from file class: @@ -542,115 +435,6 @@ class GreenPipe(_fileobject): - file argument can be descriptor, file name or file object. """ - def __init__(self, f, mode='r', bufsize=-1): - if not isinstance(f, six.string_types + (int, file)): - raise TypeError('f(ile) should be int, str, unicode or file, not %r' % f) - - if isinstance(f, six.string_types): - f = open(f, mode, 0) - - if isinstance(f, int): - fileno = f - self._name = "<fd:%d>" % fileno - else: - fileno = os.dup(f.fileno()) - self._name = f.name - if not ( - f.mode == mode or - six.PY3 and f.mode == 'rb+' and mode == 'wb+' - ): - raise ValueError('file.mode %r does not match mode parameter %r' % (f.mode, mode)) - self._name = f.name - f.close() - - if six.PY3: - mode = mode.rstrip('+') - - super(GreenPipe, self).__init__(_SocketDuckForFd(fileno), mode) - set_nonblocking(self) - self.softspace = 0 - - if six.PY3: - def write(self, data): - while data: - sent = _fileobject.write(self, data) - data = data[sent:] - - @property - def name(self): - return self._name - - def __repr__(self): - return "<%s %s %r, mode %r at 0x%x>" % ( - self.closed and 'closed' or 'open', - self.__class__.__name__, - self.name, - self.mode, - (id(self) < 0) and (sys.maxint + id(self)) or id(self)) - - def close(self): - super(GreenPipe, self).close() - for method in [ - 'fileno', 'flush', 'isatty', 'next', 'read', 'readinto', - 'readline', 'readlines', 'seek', 'tell', 'truncate', - 'write', 'xreadlines', '__iter__', '__next__', 'writelines']: - setattr(self, method, _operationOnClosedFile) - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def _get_readahead_len(self): - return len(self._rbuf.getvalue()) - - def _clear_readahead_buf(self): - len = self._get_readahead_len() - if len > 0: - self.read(len) - - def tell(self): - self.flush() - try: - return os.lseek(self.fileno(), 0, 1) - self._get_readahead_len() - except OSError as e: - raise IOError(*e.args) - - def seek(self, offset, whence=0): - self.flush() - if whence == 1 and offset == 0: # tell synonym - return self.tell() - if whence == 1: # adjust offset by what is read ahead - offset -= self._get_readahead_len() - try: - rv = os.lseek(self.fileno(), offset, whence) - except OSError as e: - raise IOError(*e.args) - else: - self._clear_readahead_buf() - return rv - - if getattr(file, "truncate", None): # not all OSes implement truncate - def truncate(self, size=-1): - self.flush() - if size == -1: - size = self.tell() - try: - rv = os.ftruncate(self.fileno(), size) - except OSError as e: - raise IOError(*e.args) - else: - self.seek(size) # move position&clear buffer - return rv - - def isatty(self): - try: - return os.isatty(self.fileno()) - except OSError as e: - raise IOError(*e.args) - - # import SSL module here so we can refer to greenio.SSL.exceptionclass try: from OpenSSL import SSL diff --git a/eventlet/greenio/py2.py b/eventlet/greenio/py2.py new file mode 100644 index 0000000..6fa6cb7 --- /dev/null +++ b/eventlet/greenio/py2.py @@ -0,0 +1,222 @@ +import errno +import os + +from eventlet.greenio.base import ( + _operation_on_closed_file, + greenpipe_doc, + set_nonblocking, + socket, + SOCKET_BLOCKING, +) +from eventlet.hubs import trampoline, notify_close, notify_opened, IOClosed +from eventlet.support import get_errno, six + +__all__ = ['_fileobject', 'GreenPipe'] + +_fileobject = socket._fileobject + + +class GreenPipe(_fileobject): + + __doc__ = greenpipe_doc + + def __init__(self, f, mode='r', bufsize=-1): + if not isinstance(f, six.string_types + (int, file)): + raise TypeError('f(ile) should be int, str, unicode or file, not %r' % f) + + if isinstance(f, six.string_types): + f = open(f, mode, 0) + + if isinstance(f, int): + fileno = f + self._name = "<fd:%d>" % fileno + else: + fileno = os.dup(f.fileno()) + self._name = f.name + if f.mode != mode: + raise ValueError('file.mode %r does not match mode parameter %r' % (f.mode, mode)) + self._name = f.name + f.close() + + super(GreenPipe, self).__init__(_SocketDuckForFd(fileno), mode) + set_nonblocking(self) + self.softspace = 0 + + @property + def name(self): + return self._name + + def __repr__(self): + return "<%s %s %r, mode %r at 0x%x>" % ( + self.closed and 'closed' or 'open', + self.__class__.__name__, + self.name, + self.mode, + (id(self) < 0) and (sys.maxint + id(self)) or id(self)) + + def close(self): + super(GreenPipe, self).close() + for method in [ + 'fileno', 'flush', 'isatty', 'next', 'read', 'readinto', + 'readline', 'readlines', 'seek', 'tell', 'truncate', + 'write', 'xreadlines', '__iter__', '__next__', 'writelines']: + setattr(self, method, _operation_on_closed_file) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def _get_readahead_len(self): + return len(self._rbuf.getvalue()) + + def _clear_readahead_buf(self): + len = self._get_readahead_len() + if len > 0: + self.read(len) + + def tell(self): + self.flush() + try: + return os.lseek(self.fileno(), 0, 1) - self._get_readahead_len() + except OSError as e: + raise IOError(*e.args) + + def seek(self, offset, whence=0): + self.flush() + if whence == 1 and offset == 0: # tell synonym + return self.tell() + if whence == 1: # adjust offset by what is read ahead + offset -= self._get_readahead_len() + try: + rv = os.lseek(self.fileno(), offset, whence) + except OSError as e: + raise IOError(*e.args) + else: + self._clear_readahead_buf() + return rv + + if getattr(file, "truncate", None): # not all OSes implement truncate + def truncate(self, size=-1): + self.flush() + if size == -1: + size = self.tell() + try: + rv = os.ftruncate(self.fileno(), size) + except OSError as e: + raise IOError(*e.args) + else: + self.seek(size) # move position&clear buffer + return rv + + def isatty(self): + try: + return os.isatty(self.fileno()) + except OSError as e: + raise IOError(*e.args) + + +class _SocketDuckForFd(object): + """Class implementing all socket method used by _fileobject + in cooperative manner using low level os I/O calls. + """ + _refcount = 0 + + def __init__(self, fileno): + self._fileno = fileno + notify_opened(fileno) + self._closed = False + + def _trampoline(self, fd, read=False, write=False, timeout=None, timeout_exc=None): + if self._closed: + # Don't trampoline if we're already closed. + raise IOClosed() + try: + return trampoline(fd, read=read, write=write, timeout=timeout, + timeout_exc=timeout_exc, + mark_as_closed=self._mark_as_closed) + except IOClosed: + # Our fileno has been obsoleted. Defang ourselves to + # prevent spurious closes. + self._mark_as_closed() + raise + + def _mark_as_closed(self): + self._closed = True + + @property + def _sock(self): + return self + + def fileno(self): + return self._fileno + + def recv(self, buflen): + while True: + try: + data = os.read(self._fileno, buflen) + return data + except OSError as e: + if get_errno(e) not in SOCKET_BLOCKING: + raise IOError(*e.args) + self._trampoline(self, read=True) + + def recv_into(self, buf, nbytes=0, flags=0): + if nbytes == 0: + nbytes = len(buf) + data = self.recv(nbytes) + buf[:nbytes] = data + return len(data) + + def send(self, data): + while True: + try: + return os.write(self._fileno, data) + except OSError as e: + if get_errno(e) not in SOCKET_BLOCKING: + raise IOError(*e.args) + else: + trampoline(self, write=True) + + def sendall(self, data): + len_data = len(data) + os_write = os.write + fileno = self._fileno + try: + total_sent = os_write(fileno, data) + except OSError as e: + if get_errno(e) != errno.EAGAIN: + raise IOError(*e.args) + total_sent = 0 + while total_sent < len_data: + self._trampoline(self, write=True) + try: + total_sent += os_write(fileno, data[total_sent:]) + except OSError as e: + if get_errno(e) != errno. EAGAIN: + raise IOError(*e.args) + + def __del__(self): + self._close() + + def _close(self): + notify_close(self._fileno) + self._mark_as_closed() + try: + os.close(self._fileno) + except: + # os.close may fail if __init__ didn't complete + # (i.e file dscriptor passed to popen was invalid + pass + + def __repr__(self): + return "%s:%d" % (self.__class__.__name__, self._fileno) + + def _reuse(self): + self._refcount += 1 + + def _drop(self): + self._refcount -= 1 + if self._refcount == 0: + self._close() diff --git a/eventlet/greenio/py3.py b/eventlet/greenio/py3.py new file mode 100644 index 0000000..22a865e --- /dev/null +++ b/eventlet/greenio/py3.py @@ -0,0 +1,191 @@ +import _pyio as _original_pyio +import errno +import os as _original_os +import socket as _original_socket +from io import ( + BufferedRandom as _OriginalBufferedRandom, + BufferedReader as _OriginalBufferedReader, + BufferedWriter as _OriginalBufferedWriter, + DEFAULT_BUFFER_SIZE, + TextIOWrapper as _OriginalTextIOWrapper, + IOBase as _OriginalIOBase, +) +from types import FunctionType + +from eventlet.greenio.base import ( + _operation_on_closed_file, + greenpipe_doc, + set_nonblocking, + SOCKET_BLOCKING, +) +from eventlet.hubs import notify_close, notify_opened, IOClosed, trampoline +from eventlet.support import get_errno, six + +__all__ = ['_fileobject', 'GreenPipe'] + +# TODO get rid of this, it only seems like the original _fileobject +_fileobject = _original_socket.SocketIO + +# Large part of the following code is copied from the original +# eventlet.greenio module + + +class GreenFileIO(_OriginalIOBase): + def __init__(self, name, mode='r', closefd=True, opener=None): + if isinstance(name, int): + fileno = name + self._name = "<fd:%d>" % fileno + else: + assert isinstance(name, six.string_types) + with open(name, mode) as fd: + self._name = fd.name + fileno = _original_os.dup(fd.fileno()) + + notify_opened(fileno) + self._fileno = fileno + self._mode = mode + self._closed = False + set_nonblocking(self) + self._seekable = None + + @property + def closed(self): + return self._closed + + def seekable(self): + if self._seekable is None: + try: + _original_os.lseek(self._fileno, 0, _original_os.SEEK_CUR) + except IOError as e: + if get_errno(e) == errno.ESPIPE: + self._seekable = False + else: + raise + else: + self._seekable = True + + return self._seekable + + def readable(self): + return 'r' in self._mode or '+' in self._mode + + def writable(self): + return 'w' in self._mode or '+' in self._mode + + def fileno(self): + return self._fileno + + def read(self, buflen): + while True: + try: + return _original_os.read(self._fileno, buflen) + except OSError as e: + if get_errno(e) not in SOCKET_BLOCKING: + raise IOError(*e.args) + self._trampoline(self, read=True) + + def readinto(self, b): + up_to = len(b) + data = self.read(up_to) + bytes_read = len(data) + b[:bytes_read] = data + return bytes_read + + def isatty(self): + try: + return _original_os.isatty(self.fileno()) + except OSError as e: + raise IOError(*e.args) + + def _trampoline(self, fd, read=False, write=False, timeout=None, timeout_exc=None): + if self._closed: + # Don't trampoline if we're already closed. + raise IOClosed() + try: + return trampoline(fd, read=read, write=write, timeout=timeout, + timeout_exc=timeout_exc, + mark_as_closed=self._mark_as_closed) + except IOClosed: + # Our fileno has been obsoleted. Defang ourselves to + # prevent spurious closes. + self._mark_as_closed() + raise + + def _mark_as_closed(self): + """ Mark this socket as being closed """ + self._closed = True + + def write(self, data): + while True: + try: + return _original_os.write(self._fileno, data) + except OSError as e: + if get_errno(e) not in SOCKET_BLOCKING: + raise IOError(*e.args) + else: + trampoline(self, write=True) + + def close(self): + _original_os.close(self._fileno) + notify_close(self._fileno) + self._closed = True + for method in [ + 'fileno', 'flush', 'isatty', 'next', 'read', 'readinto', + 'readline', 'readlines', 'seek', 'tell', 'truncate', + 'write', 'xreadlines', '__iter__', '__next__', 'writelines']: + setattr(self, method, _operation_on_closed_file) + + def truncate(self, size=-1): + if size == -1: + size = self.tell() + try: + rv = _original_os.ftruncate(self._fileno, size) + except OSError as e: + raise IOError(*e.args) + else: + self.seek(size) # move position&clear buffer + return rv + + def seek(self, offset, whence=_original_os.SEEK_SET): + try: + return _original_os.lseek(self._fileno, offset, whence) + except OSError as e: + raise IOError(*e.args) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +_open_environment = dict(globals()) +_open_environment.update(dict( + BufferedRandom=_OriginalBufferedRandom, + BufferedWriter=_OriginalBufferedWriter, + BufferedReader=_OriginalBufferedReader, + TextIOWrapper=_OriginalTextIOWrapper, + FileIO=GreenFileIO, + os=_original_os, +)) + +_open = FunctionType( + six.get_function_code(_original_pyio.open), + _open_environment, +) + + +def GreenPipe(name, mode="r", buffering=-1, encoding=None, errors=None, + newline=None, closefd=True, opener=None): + try: + fileno = name.fileno() + except AttributeError: + pass + else: + fileno = _original_os.dup(fileno) + name.close() + name = fileno + + return _open(name, mode, buffering, encoding, errors, newline, closefd, opener) + +GreenPipe.__doc__ = greenpipe_doc diff --git a/eventlet/wsgi.py b/eventlet/wsgi.py index e69d107..f6add70 100644 --- a/eventlet/wsgi.py +++ b/eventlet/wsgi.py @@ -8,12 +8,13 @@ import warnings from eventlet.green import BaseHTTPServer from eventlet.green import socket -from eventlet.green import urllib from eventlet import greenio from eventlet import greenpool from eventlet import support from eventlet.support import six +from eventlet.support.six.moves.urllib import parse as urllib_parse + DEFAULT_MAX_SIMULTANEOUS_REQUESTS = 1024 DEFAULT_MAX_HTTP_VERSION = 'HTTP/1.1' @@ -395,24 +396,8 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler): towrite.append(six.b("%x" % (len(data),)) + b"\r\n" + data + b"\r\n") else: towrite.append(data) - try: - _writelines(towrite) - length[0] = length[0] + sum(map(len, towrite)) - except UnicodeEncodeError: - self.server.log_message( - "Encountered non-ascii unicode while attempting to write" - "wsgi response: %r" % - [x for x in towrite if isinstance(x, six.text_type)]) - self.server.log_message(traceback.format_exc()) - _writelines( - ["HTTP/1.1 500 Internal Server Error\r\n", - "Connection: close\r\n", - "Content-type: text/plain\r\n", - "Content-length: 98\r\n", - "Date: %s\r\n" % format_date_time(time.time()), - "\r\n", - ("Internal Server Error: wsgi application passed " - "a unicode object to the server instead of a string.")]) + _writelines(towrite) + length[0] = length[0] + sum(map(len, towrite)) def start_response(status, response_headers, exc_info=None): status_code[0] = status.split()[0] @@ -456,6 +441,9 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler): minimum_write_chunk_size = int(self.environ.get( 'eventlet.minimum_write_chunk_size', self.minimum_chunk_size)) for data in result: + if isinstance(data, six.text_type): + data = data.encode('ascii') + towrite.append(data) towrite_size += len(data) if towrite_size >= minimum_write_chunk_size: @@ -472,7 +460,7 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler): self.close_connection = 1 tb = traceback.format_exc() self.server.log_message(tb) - if not headers_set: + if not headers_sent: err_body = six.b(tb) if self.server.debug else b'' start_response("500 Internal Server Error", [('Content-type', 'text/plain'), @@ -522,7 +510,7 @@ class HttpProtocol(BaseHTTPServer.BaseHTTPRequestHandler): pq = self.path.split('?', 1) env['RAW_PATH_INFO'] = pq[0] - env['PATH_INFO'] = urllib.unquote(pq[0]) + env['PATH_INFO'] = urllib_parse.unquote(pq[0]) if len(pq) > 1: env['QUERY_STRING'] = pq[1] diff --git a/tests/greenio_test.py b/tests/greenio_test.py index ffdc534..fa7c160 100644 --- a/tests/greenio_test.py +++ b/tests/greenio_test.py @@ -667,8 +667,8 @@ class TestGreenPipe(LimitedTestCase): def test_pipe(self): r, w = os.pipe() - rf = greenio.GreenPipe(r, 'r') - wf = greenio.GreenPipe(w, 'w', 0) + rf = greenio.GreenPipe(r, 'rb') + wf = greenio.GreenPipe(w, 'wb', 0) def sender(f, content): for ch in map(six.int2byte, six.iterbytes(content)): @@ -690,8 +690,8 @@ class TestGreenPipe(LimitedTestCase): # also ensures that readline() terminates on '\n' and '\r\n' r, w = os.pipe() - r = greenio.GreenPipe(r) - w = greenio.GreenPipe(w, 'w') + r = greenio.GreenPipe(r, 'rb') + w = greenio.GreenPipe(w, 'wb') def writer(): eventlet.sleep(.1) @@ -717,8 +717,8 @@ class TestGreenPipe(LimitedTestCase): def test_pipe_writes_large_messages(self): r, w = os.pipe() - r = greenio.GreenPipe(r) - w = greenio.GreenPipe(w, 'w') + r = greenio.GreenPipe(r, 'rb') + w = greenio.GreenPipe(w, 'wb') large_message = b"".join([1024 * six.int2byte(i) for i in range(65)]) diff --git a/tests/patcher_test.py b/tests/patcher_test.py index 5c9076f..3ace281 100644 --- a/tests/patcher_test.py +++ b/tests/patcher_test.py @@ -9,10 +9,7 @@ from tests import LimitedTestCase, main, run_python, skip_with_pyevent base_module_contents = """ import socket -try: - import urllib.request as urllib -except ImportError: - import urllib +import urllib print("base {0} {1}".format(socket, urllib)) """ @@ -86,7 +83,14 @@ class ImportPatched(ProcessBase): assert 'eventlet.green.httplib' not in lines[2], repr(output) def test_import_patched_defaults(self): - self.write_to_tempfile("base", base_module_contents) + self.write_to_tempfile("base", """ +import socket +try: + import urllib.request as urllib +except ImportError: + import urllib +print("base {0} {1}".format(socket, urllib))""") + new_mod = """ from eventlet import patcher base = patcher.import_patched('base') diff --git a/tests/test__greenness.py b/tests/test__greenness.py index 7d90890..a594b4d 100644 --- a/tests/test__greenness.py +++ b/tests/test__greenness.py @@ -4,8 +4,15 @@ If either operation blocked the whole script would block and timeout. """ import unittest -from eventlet.green import urllib2, BaseHTTPServer +from eventlet.green import BaseHTTPServer from eventlet import spawn, kill +from eventlet.support import six + +if six.PY2: + from eventlet.green.urllib2 import HTTPError, urlopen +else: + from eventlet.green.urllib.request import urlopen + from eventlet.green.urllib.error import HTTPError class QuietHandler(BaseHTTPServer.BaseHTTPRequestHandler): @@ -40,12 +47,12 @@ class TestGreenness(unittest.TestCase): self.server.server_close() kill(self.gthread) - def test_urllib2(self): + def test_urllib(self): self.assertEqual(self.server.request_count, 0) try: - urllib2.urlopen('http://127.0.0.1:%s' % self.port) + urlopen('http://127.0.0.1:%s' % self.port) assert False, 'should not get there' - except urllib2.HTTPError as ex: + except HTTPError as ex: assert ex.code == 501, repr(ex) self.assertEqual(self.server.request_count, 1) diff --git a/tests/wsgi_test.py b/tests/wsgi_test.py index 179881d..8587803 100644 --- a/tests/wsgi_test.py +++ b/tests/wsgi_test.py @@ -1330,11 +1330,24 @@ class TestHttpd(_TestBase): self.assertEqual(result.headers_lower['connection'], 'close') assert 'transfer-encoding' not in result.headers_lower - def test_unicode_raises_error(self): + def test_unicode_with_only_ascii_characters_works(self): def wsgi_app(environ, start_response): start_response("200 OK", []) - yield u"oh hai" - yield u"non-encodable unicode: \u0230" + yield b"oh hai, " + yield u"xxx" + self.site.application = wsgi_app + sock = eventlet.connect(('localhost', self.port)) + fd = sock.makefile('rwb') + fd.write(b'GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + fd.flush() + result = read_http(sock) + assert b'xxx' in result.body + + def test_unicode_with_nonascii_characters_raises_error(self): + def wsgi_app(environ, start_response): + start_response("200 OK", []) + yield b"oh hai, " + yield u"xxx \u0230" self.site.application = wsgi_app sock = eventlet.connect(('localhost', self.port)) fd = sock.makefile('rwb') @@ -1343,7 +1356,6 @@ class TestHttpd(_TestBase): result = read_http(sock) self.assertEqual(result.status, 'HTTP/1.1 500 Internal Server Error') self.assertEqual(result.headers_lower['connection'], 'close') - assert b'unicode' in result.body def test_path_info_decoding(self): def wsgi_app(environ, start_response): @@ -1454,11 +1466,11 @@ class TestHttpd(_TestBase): # (if eventlet stops using file.readline() to read HTTP headers, # for instance) for runlog in sections[1:]: - debug = False if "debug set to: False" in runlog else True + debug = False if b"debug set to: False" in runlog else True if debug: - self.assertTrue("timed out" in runlog) - self.assertTrue("BOOM" in runlog) - self.assertFalse("Traceback" in runlog) + self.assertTrue(b"timed out" in runlog) + self.assertTrue(b"BOOM" in runlog) + self.assertFalse(b"Traceback" in runlog) def test_server_socket_timeout(self): self.spawn_server(socket_timeout=0.1) @@ -1743,7 +1755,9 @@ class TestChunkedInput(_TestBase): fd = self.connect() fd.sendall(req.encode()) fd.close() - eventlet.sleep(0.0) + + # TODO changed from 0 to make Python 3 tests pass, not sure if ok + eventlet.sleep(0.01) finally: signal.alarm(0) signal.signal(signal.SIGALRM, signal.SIG_DFL) diff --git a/tests/wsgi_test_conntimeout.py b/tests/wsgi_test_conntimeout.py index d925a04..01ecc0e 100644 --- a/tests/wsgi_test_conntimeout.py +++ b/tests/wsgi_test_conntimeout.py @@ -25,6 +25,7 @@ connection makefile() file objects - ExplodingSocketFile <-- these raise from __future__ import print_function import eventlet +from eventlet.support import six import socket import sys @@ -96,7 +97,8 @@ class ExplodingConnectionWrap(object): class ExplodingSocketFile(eventlet.greenio._fileobject): def __init__(self, sock, mode='rb', bufsize=-1, close=False): - super(self.__class__, self).__init__(sock, mode, bufsize, close) + args = [bufsize, close] if six.PY2 else [] + super(self.__class__, self).__init__(sock, mode, *args) self.armed = False def arm(self): |