diff options
| -rw-r--r-- | kafka/client_async.py | 14 | ||||
| -rw-r--r-- | kafka/socketpair.py | 57 |
2 files changed, 65 insertions, 6 deletions
diff --git a/kafka/client_async.py b/kafka/client_async.py index 57aea66..ae9dbb4 100644 --- a/kafka/client_async.py +++ b/kafka/client_async.py @@ -4,9 +4,9 @@ import copy import heapq import itertools import logging -import os import random import select +import socket import time import six @@ -18,6 +18,7 @@ from .conn import BrokerConnection, ConnectionStates, collect_hosts from .future import Future from .protocol.metadata import MetadataRequest from .protocol.produce import ProduceRequest +from . import socketpair from .version import __version__ if six.PY2: @@ -97,11 +98,11 @@ class KafkaClient(object): self._last_bootstrap = 0 self._bootstrap_fails = 0 self._bootstrap(collect_hosts(self.config['bootstrap_servers'])) - self._wake_r, self._wake_w = os.pipe() + self._wake_r, self._wake_w = socket.socketpair() def __del__(self): - os.close(self._wake_r) - os.close(self._wake_w) + self._wake_r.close() + self._wake_w.close() def _bootstrap(self, hosts): # Exponential backoff if bootstrap fails @@ -674,14 +675,15 @@ class KafkaClient(object): return version def wakeup(self): - os.write(self._wake_w, b'x') + if self._wake_w.send(b'x') != 1: + log.warning('Unable to send to wakeup socket!') def _clear_wake_fd(self): while True: fds, _, _ = select.select([self._wake_r], [], [], 0) if not fds: break - os.read(self._wake_r, 1) + self._wake_r.recv(1) class DelayedTaskQueue(object): diff --git a/kafka/socketpair.py b/kafka/socketpair.py new file mode 100644 index 0000000..6a87c4d --- /dev/null +++ b/kafka/socketpair.py @@ -0,0 +1,57 @@ +# pylint: skip-file +# vendored from https://github.com/mhils/backports.socketpair + +import sys +import socket +import errno + +_LOCALHOST = '127.0.0.1' +_LOCALHOST_V6 = '::1' + +if not hasattr(socket, "socketpair"): + # Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. + def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): + if family == socket.AF_INET: + host = _LOCALHOST + elif family == socket.AF_INET6: + host = _LOCALHOST_V6 + else: + raise ValueError("Only AF_INET and AF_INET6 socket address families " + "are supported") + if type != socket.SOCK_STREAM: + raise ValueError("Only SOCK_STREAM socket type is supported") + if proto != 0: + raise ValueError("Only protocol zero is supported") + + # We create a connected TCP socket. Note the trick with + # setblocking(False) that prevents us from having to create a thread. + lsock = socket.socket(family, type, proto) + try: + lsock.bind((host, 0)) + lsock.listen(min(socket.SOMAXCONN, 128)) + # On IPv6, ignore flow_info and scope_id + addr, port = lsock.getsockname()[:2] + csock = socket.socket(family, type, proto) + try: + csock.setblocking(False) + if sys.version_info >= (3, 0): + try: + csock.connect((addr, port)) + except (BlockingIOError, InterruptedError): + pass + else: + try: + csock.connect((addr, port)) + except socket.error as e: + if e.errno != errno.WSAEWOULDBLOCK: + raise + csock.setblocking(True) + ssock, _ = lsock.accept() + except: + csock.close() + raise + finally: + lsock.close() + return (ssock, csock) + + socket.socketpair = socketpair |