diff options
| author | John Schwarz <jschwarz@redhat.com> | 2016-07-14 13:43:25 +0300 |
|---|---|---|
| committer | John Schwarz <jschwarz@redhat.com> | 2016-07-27 11:14:57 +0300 |
| commit | e5c530a2cdd1fefbe468d9ac8d8eb8ceed4ec262 (patch) | |
| tree | 2346c180bc46d952b27925b579bb2710c41cc114 | |
| parent | 324482fa96344410cec56254d61ccf91ea5e9a8b (diff) | |
| download | tooz-e5c530a2cdd1fefbe468d9ac8d8eb8ceed4ec262.tar.gz | |
etcd: don't run heartbeat() concurrently
As the bug specifies, running heartbeat() concurrently to other actions
(such as release()) is raceful as using heartbeat() can and will change
the modifiedIndex field of a lock, thus breaking the compare function of
the Compare-and-Delete used by release(). This patch, therefore, uses
threading's Lock object to make sure only a single action is done
concurrently in tooz' etcd driver.
Closes-Bug: #1603005
Change-Id: Ief32a3d5add90ebbd4adf825d78fca3dd57e5d9e
| -rw-r--r-- | tooz/drivers/etcd.py | 12 |
1 files changed, 12 insertions, 0 deletions
diff --git a/tooz/drivers/etcd.py b/tooz/drivers/etcd.py index 81fb179..9d6531e 100644 --- a/tooz/drivers/etcd.py +++ b/tooz/drivers/etcd.py @@ -13,7 +13,9 @@ # under the License. import logging +import threading +import fasteners from oslo_utils import encodeutils from oslo_utils import timeutils import requests @@ -94,11 +96,18 @@ class EtcdLock(locking.Lock): self._lock_url = lock_url self._node = None + # NOTE(jschwarz): this lock is mainly used to prevent concurrent runs + # of hearthbeat() with another function. For more details, see + # https://bugs.launchpad.net/python-tooz/+bug/1603005. + self._lock = threading.Lock() + @_translate_failures + @fasteners.locked def break_(self): reply = self.client.delete(self._lock_url, make_url=False) return reply.get('errorCode') is None + @fasteners.locked def acquire(self, blocking=True): blocking, timeout = utils.convert_blocking(blocking) if timeout is not None: @@ -141,6 +150,7 @@ class EtcdLock(locking.Lock): return False @_translate_failures + @fasteners.locked def release(self): if self in self.coord._acquired_locks: lock_url = self._lock_url @@ -157,6 +167,7 @@ class EtcdLock(locking.Lock): return False @_translate_failures + @fasteners.locked def heartbeat(self): """Keep the lock alive.""" poked = self.client.put(self._lock_url, @@ -167,6 +178,7 @@ class EtcdLock(locking.Lock): LOG.warning("Unable to heartbeat by updating key '%s' with " "extended expiry of %s seconds: %d, %s", self.name, self.ttl, errorcode, poked.get("message")) + self._node = poked['node'] class EtcdDriver(coordination.CoordinationDriver): |