test_lock_fence.sh, rbdrw.py: rbd lock/fence test

qa/workunits/rbd/test_lock_fence.sh runs using test/rbdrw.py

rbdrw.py creates an image, locks it, and runs an I/O loop;
test_lock_fence.sh runs it, waits, and then blacklists that client,
which causes rbdrw.py to get ESHUTDOWN on operations thereafter.
Currently doesn't work with rbd caching enabled.

rbd.py gets new exception type for ESHUTDOWN

Fixes: #3190
Signed-off-by: Dan Mick <dan.mick@inktank.com>
Signed-off-by: Josh Durgin <josh.durgin@inktank.com>

3 files changed, 85 insertions, 1 deletions

diff --git a/qa/workunits/rbd/test_lock_fence.sh b/qa/workunits/rbd/test_lock_fence.sh
new file mode 100755
index 00000000000..ee015043fbd
--- /dev/null
+++ b/qa/workunits/rbd/test_lock_fence.sh
@@ -0,0 +1,49 @@
+#!/bin/bash  -x
+# can't use -e because of background process
+
+IMAGE=rbdrw-image
+LOCKID=rbdrw
+RBDRW=rbdrw.py
+CEPH_REF=${CEPH_REF:-master}
+
+wget -O $RBDRW "https://ceph.com/git/?p=ceph.git;a=blob_plain;hb=$CEPH_REF;f=src/test/librbd/rbdrw.py"
+
+rbd create $IMAGE --size 10 --image-format 2 || exit 1
+
+# rbdrw loops doing I/O to $IMAGE after locking with lockid $LOCKID
+python $RBDRW $IMAGE $LOCKID &
+iochild=$!
+
+# give client time to lock and start reading/writing
+LOCKS='{}'
+while [ "$LOCKS" == "{}" ]
+do
+    LOCKS=$(rbd lock list $IMAGE --format json)
+    sleep 1
+done
+
+clientaddr=$(rbd lock list $IMAGE | tail -1 | awk '{print $NF;}')
+clientid=$(rbd lock list $IMAGE | tail -1 | awk '{print $1;}')
+echo "clientaddr: $clientaddr"
+echo "clientid: $clientid"
+
+ceph osd blacklist add $clientaddr || exit 1
+
+wait $iochild
+rbdrw_exitcode=$?
+if [ $rbdrw_exitcode != 108 ]
+then
+	echo "wrong exitcode from rbdrw: $rbdrw_exitcode"
+	exit 1
+else
+	echo "rbdrw stopped with ESHUTDOWN"
+fi
+
+set -e
+ceph osd blacklist rm $clientaddr
+rbd lock remove $IMAGE $LOCKID "$clientid"
+# rbdrw will have exited with an existing watch, so, until #3527 is fixed,
+# hang out until the watch expires
+sleep 30
+rbd rm $IMAGE
+echo OK
diff --git a/src/pybind/rbd.py b/src/pybind/rbd.py
index f4d844a17ec..4ce1a0e819b 100644
--- a/src/pybind/rbd.py
+++ b/src/pybind/rbd.py
@@ -68,6 +68,9 @@ class FunctionNotSupported(Error):
 class ArgumentOutOfRange(Error):
     pass
 
+class ConnectionShutdown(Error):
+    pass
+
 def make_ex(ret, msg):
     """
     Translate a librbd return code into an exception.
@@ -89,7 +92,8 @@ def make_ex(ret, msg):
         errno.EBUSY     : ImageBusy,
         errno.ENOTEMPTY : ImageHasSnapshots,
         errno.ENOSYS    : FunctionNotSupported,
-        errno.EDOM      : ArgumentOutOfRange
+        errno.EDOM      : ArgumentOutOfRange,
+        errno.ESHUTDOWN : ConnectionShutdown
         }
     ret = abs(ret)
     if ret in errors:
diff --git a/src/test/librbd/rbdrw.py b/src/test/librbd/rbdrw.py
new file mode 100755
index 00000000000..6034ad055cf
--- /dev/null
+++ b/src/test/librbd/rbdrw.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+"""
+Loop writing/reading the first 4k of image argv[1] in pool rbd,
+after acquiring exclusive lock named argv[2].  When an exception
+happens, split off the last number in the exception 'args' string
+and use it as the process exit code, if it's convertible to a number.
+
+Designed to run against a blacklist operation and verify the
+ESHUTDOWN expected from the image operation.
+
+Note: this cannot be run with writeback caching on, currently, as
+writeback errors cause reads be marked dirty rather than error, and
+even if they were marked as errored, ObjectCacher would retry them
+rather than note them as errored.
+"""
+
+import rados, rbd, sys
+
+with rados.Rados(conffile='') as r:
+    with r.open_ioctx('rbd') as ioctx:
+        with rbd.Image(ioctx, sys.argv[1]) as image:
+            image.lock_exclusive(sys.argv[2])
+            while True:
+                try:
+                    image.write('A' * 4096, 0)
+                    r = image.read(0, 4096)
+                except rbd.ConnectionShutdown:
+                    # it so happens that the errno here is 108, but
+                    # anything recognizable would do
+                    exit(108)
+