From a71af7e8ccc3b576216e1f804701a3c564fb53fb Mon Sep 17 00:00:00 2001 From: Gary Lowell Date: Thu, 11 Apr 2013 09:42:13 -0700 Subject: ceph-disk: OSD hotplug fixes for Centos Two fixes for Centos 6.3 and other systems with udev versions prior to 172. The disk peristant name using the GPT UUID does not exist, so use the by_path persistent name instead for the journal symlink. The gpt label fields are not available for use in udev rules. Add ceph-disk-udev wrapper script that extracts the partition type guid from the label and calls ceph-disk-activate if it is a ceph guid type. (Bug #4632) Signed-off-by: Gary Lowell --- Makefile.am | 3 +- ceph.spec.in | 8 ++++- debian/control | 2 ++ src/Makefile.am | 2 ++ src/ceph-disk | 78 +++++++++++++++++++++++++++++++++------------- src/ceph-disk-udev | 56 +++++++++++++++++++++++++++++++++ udev/95-ceph-osd-alt.rules | 5 +++ 7 files changed, 131 insertions(+), 23 deletions(-) create mode 100755 src/ceph-disk-udev create mode 100644 udev/95-ceph-osd-alt.rules diff --git a/Makefile.am b/Makefile.am index adeb4e57728..03cb914079f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,7 +10,8 @@ EXTRA_DIST += \ src/test/cli \ src/test/downloads \ udev/50-rbd.rules \ - udev/95-ceph-osd.rules + udev/95-ceph-osd.rules \ + udev/95-ceph-osd-alt.rules all-local: diff --git a/ceph.spec.in b/ceph.spec.in index a2b504d2d2f..023258cbe61 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -25,10 +25,11 @@ Requires: librbd1 = %{version}-%{release} Requires: librados2 = %{version}-%{release} Requires: libcephfs1 = %{version}-%{release} Requires: python +Requires: python-argparse +Requires: python-lockfile Requires: cryptsetup Requires: parted Requires: util-linux -Requires: python-argparse Requires: hdparm Requires(post): binutils BuildRoot: %{_tmppath}/%{name}-%{version}-build @@ -301,7 +302,11 @@ chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.fetch_config # udev rules install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/50-rbd.rules +%if 0%{?centos} +install -m 0644 -D udev/95-ceph-osd-alt.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules +%else install -m 0644 -D udev/95-ceph-osd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules +%endif #set up placeholder directories mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/ceph @@ -386,6 +391,7 @@ fi %{_sbindir}/ceph-disk %{_sbindir}/ceph-disk-activate %{_sbindir}/ceph-disk-prepare +%{_sbindir}/ceph-disk-udev %{_sbindir}/ceph-create-keys %{_sbindir}/rcceph /sbin/mkcephfs diff --git a/debian/control b/debian/control index 98d0b37ffc1..efa86377c52 100644 --- a/debian/control +++ b/debian/control @@ -45,6 +45,8 @@ Depends: binutils, gdisk, parted, python, + python-argparse, + python-lockfile, sdparm | hdparm, uuid-runtime, xfsprogs, diff --git a/src/Makefile.am b/src/Makefile.am index d528b78a1be..0b223f20299 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -33,6 +33,7 @@ ceph_sbin_SCRIPTS = \ ceph-disk \ ceph-disk-prepare \ ceph-disk-activate \ + ceph-disk-udev \ ceph-create-keys sbin_SCRIPTS = \ @@ -1192,6 +1193,7 @@ EXTRA_DIST += \ ceph-disk \ ceph-disk-prepare \ ceph-disk-activate \ + ceph-disk-udev \ ceph-create-keys \ mount.fuse.ceph diff --git a/src/ceph-disk b/src/ceph-disk index 68a171938c5..c9515997ded 100755 --- a/src/ceph-disk +++ b/src/ceph-disk @@ -12,6 +12,7 @@ import stat import sys import tempfile import uuid +import lockfile CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026' @@ -62,6 +63,7 @@ if LOG_NAME == '__main__': LOG_NAME = os.path.basename(sys.argv[0]) LOG = logging.getLogger(LOG_NAME) +lock = lockfile.FileLock('/var/lib/ceph/tmp/ceph-disk.lock') ###### exceptions ######## @@ -683,6 +685,16 @@ def zap(dev): raise Error(e) +def get_udev_version(): + version = _check_output( + args=[ + 'udevadm', + '--version', + ], + ) + return int(version) + + def prepare_journal_dev( data, journal, @@ -761,9 +773,24 @@ def prepare_journal_dev( ], ) - journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format( - journal_uuid=journal_uuid, - ) + if get_udev_version() >= 172: + journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format( + journal_uuid=journal_uuid, + ) + else: + # udev prior to version 172 doesn't create by-partuuid directory + # use by-path symlink instead. This is the third symlink returned + # by udevadm when queried. + LOG.debug('Using alternate persistant name for journal symlink') + symlinks = _check_output( + args=[ + 'udevadm', + 'info', + '--query=symlink', + '--name={name}'.format(name=os.path.basename(journal)), + ], + ) + journal_symlink='/dev/{symlink}-part{num}'.format(symlink=symlinks.split()[2], num=num) journal_dmcrypt = None if journal_dm_keypath: @@ -1035,6 +1062,7 @@ def main_prepare(args): osd_dm_keypath = None try: + lock.acquire() if not os.path.exists(args.data): raise Error('data path does not exist', args.data) @@ -1163,12 +1191,14 @@ def main_prepare(args): ) else: raise Error('not a dir or block device', args.data) + lock.release() except Error as e: if journal_dm_keypath: os.unlink(journal_dm_keypath) if osd_dm_keypath: os.unlink(osd_dm_keypath) + lock.release() raise e @@ -1559,26 +1589,32 @@ def main_activate(args): if not os.path.exists(args.path): raise Error('%s does not exist', args.path) - mode = os.stat(args.path).st_mode - if stat.S_ISBLK(mode): - (cluster, osd_id) = mount_activate( - dev=args.path, - activate_key_template=args.activate_key_template, - init=args.mark_init, - ) - elif stat.S_ISDIR(mode): - (cluster, osd_id) = activate_dir( - path=args.path, - activate_key_template=args.activate_key_template, - init=args.mark_init, + lock.acquire() + try: + mode = os.stat(args.path).st_mode + if stat.S_ISBLK(mode): + (cluster, osd_id) = mount_activate( + dev=args.path, + activate_key_template=args.activate_key_template, + init=args.mark_init, + ) + elif stat.S_ISDIR(mode): + (cluster, osd_id) = activate_dir( + path=args.path, + activate_key_template=args.activate_key_template, + init=args.mark_init, + ) + else: + raise Error('%s is not a directory or block device', args.path) + + start_daemon( + cluster=cluster, + osd_id=osd_id, ) - else: - raise Error('%s is not a directory or block device', args.path) + lock.release() - start_daemon( - cluster=cluster, - osd_id=osd_id, - ) + except: + lock.release() diff --git a/src/ceph-disk-udev b/src/ceph-disk-udev new file mode 100755 index 00000000000..a96c21652aa --- /dev/null +++ b/src/ceph-disk-udev @@ -0,0 +1,56 @@ +#! /bin/sh + +# Wrapper for the ceph udev rules. Since older versions of udev do not support gpt label fields, this shell +# script is invoked from the udev rule to read the needed gpt label fields and call the appropriate ceph +# OSD functions. + +PARTNO=$1 +NAME=$2 +PARENT_NAME=$3 + +# Get GPT partition type guid +ID_PART_ENTRY_TYPE=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition GUID code" | awk '{print $4}' | tr '[:upper:]' '[:lower:]') +case $ID_PART_ENTRY_TYPE in + +45b0969e-9b03-4f30-b4c6-b4b80ceff106) + # JOURNAL_UUID + ;; + +45b0969e-9b03-4f30-b4c6-5ec00ceff106) + # DMCRYPT_JOURNAL_UUID + # Map journal if using dm-crypt + ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]') + /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME} + ;; + +4fbd7e29-9d25-41b8-afd0-062c0ceff05d) + # OSD_UUID + # activate ceph-tagged partitions. + /usr/sbin/ceph-disk -v activate --mount /dev/${NAME} + ;; + +4fbd7e29-9d25-41b8-afd0-5ec00ceff05d) + # DMCRYPT_OSD_UUID + # Map data device and activate ceph-tagged partitions + # for dm-crypted data devices + ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]') + /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME} + bash -c 'while [ ! -e /dev/mapper/${ID_PART_ENTRY_UUID} ];do sleep 1; done' + /usr/sbin/ceph-disk-activate --mount /dev/mapper/${ID_PART_ENTRY_UUID} + ;; + +89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be) + # TOBE_UUID + ;; + +89c57f98-2fe5-4dc0-89c1-5ec00ceff2be) + # DMCRYPT_TOBE_UUID + ;; + +*) + # Not a Ceph device + ;; + +esac + +exit diff --git a/udev/95-ceph-osd-alt.rules b/udev/95-ceph-osd-alt.rules new file mode 100644 index 00000000000..702ceea5d39 --- /dev/null +++ b/udev/95-ceph-osd-alt.rules @@ -0,0 +1,5 @@ +# Check gpt partion for ceph tags and activate +ACTION=="add", SUBSYSTEM=="block", \ + ENV{DEVTYPE}=="partition", \ + ENV{ID_PART_TABLE_TYPE}=="gpt", \ + RUN+="/usr/sbin/ceph-disk-udev $number $name" -- cgit v1.2.1