ceph-disk: use separate lock files for prepare, activate

Use a separate lock file for prepare and activate to avoid deadlock.  This
didn't seem to trigger on all machines, but in many cases, the prepare
process would take the file lock and later trigger a udev event and the
activate would then block on the same lock, either when we explicitly call
'udevadm settle --timeout=10' or when partprobe does it on our behalf
(without a timeout!).   Avoid this by using separate locks for prepare
and activate.  We only care if multiple activates race; it is
okay for a prepare to be in progress and for an activate to be kicked
off.

Signed-off-by: Sage Weil <sage@inktank.com>

1 files changed, 8 insertions, 7 deletions

diff --git a/src/ceph-disk b/src/ceph-disk
index f1380be8701..c5f16a401e1 100755
--- a/src/ceph-disk
+++ b/src/ceph-disk
@@ -63,7 +63,8 @@ if LOG_NAME == '__main__':
     LOG_NAME = os.path.basename(sys.argv[0])
 LOG = logging.getLogger(LOG_NAME)
 
-lock = lockfile.FileLock('/var/lib/ceph/tmp/ceph-disk.lock')
+prepare_lock = lockfile.FileLock('/var/lib/ceph/tmp/ceph-disk.prepare.lock')
+activate_lock = lockfile.FileLock('/var/lib/ceph/tmp/ceph-disk.activate.lock')
 
 ###### exceptions ########
 
@@ -1060,7 +1061,7 @@ def main_prepare(args):
     osd_dm_keypath = None
 
     try:
-        lock.acquire()
+        prepare_lock.acquire()
         if not os.path.exists(args.data):
             raise Error('data path does not exist', args.data)
 
@@ -1189,14 +1190,14 @@ def main_prepare(args):
                 )
         else:
             raise Error('not a dir or block device', args.data)
-        lock.release()
+        prepare_lock.release()
 
     except Error as e:
         if journal_dm_keypath:
             os.unlink(journal_dm_keypath)
         if osd_dm_keypath:
             os.unlink(osd_dm_keypath)
-        lock.release()
+        prepare_lock.release()
         raise e
 
 
@@ -1591,7 +1592,7 @@ def main_activate(args):
     if not os.path.exists(args.path):
         raise Error('%s does not exist', args.path)
 
-    lock.acquire()
+    activate_lock.acquire()
     try:
         mode = os.stat(args.path).st_mode
         if stat.S_ISBLK(mode):
@@ -1613,10 +1614,10 @@ def main_activate(args):
             cluster=cluster,
             osd_id=osd_id,
             )
-        lock.release()
+        activate_lock.release()
 
     except:
-        lock.release()
+        activate_lock.release()