From 9900876d263ea141c3371a1f004c9e478a967be2 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 7 Aug 2013 12:59:34 -0700 Subject: create a setup.py file Signed-off-by: Alfredo Deza --- src/pybind/setup.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 src/pybind/setup.py diff --git a/src/pybind/setup.py b/src/pybind/setup.py new file mode 100644 index 00000000000..6814ba09d87 --- /dev/null +++ b/src/pybind/setup.py @@ -0,0 +1,32 @@ +from setuptools import setup +import os + + +def long_description(): + readme = os.path.join(os.path.dirname(__file__), 'README.rst') + return open(readme).read() + + +setup( + name = 'ceph', + description = 'Bindings for Ceph', + packages = ['ceph'], + author = 'Inktank', + author_email = 'ceph-devel@vger.kernel.org', + version = '0.0.1', #XXX Fix version + license = "GPLv2", + zip_safe = False, + keywords = "ceph, bindings, api, cli", + long_description = "", #XXX Long description should come from the README.rst + classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', + 'Topic :: Software Development :: Libraries', + 'Topic :: Utilities', + 'Topic :: System :: Filesystems', + 'Operating System :: POSIX', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], +) -- cgit v1.2.1 From d6e1479e09c571113fd725185c27e48bb1f251bd Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 7 Aug 2013 12:59:55 -0700 Subject: create a MANIFEST file Signed-off-by: Alfredo Deza --- src/pybind/MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/pybind/MANIFEST.in diff --git a/src/pybind/MANIFEST.in b/src/pybind/MANIFEST.in new file mode 100644 index 00000000000..f1cb7376052 --- /dev/null +++ b/src/pybind/MANIFEST.in @@ -0,0 +1 @@ +include setup.py -- cgit v1.2.1 From e0d15b1fc53ccfaf81ca73d8907c0138994d16e5 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 7 Aug 2013 13:01:25 -0700 Subject: base tox.ini file for testing Signed-off-by: Alfredo Deza --- src/pybind/tox.ini | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/pybind/tox.ini diff --git a/src/pybind/tox.ini b/src/pybind/tox.ini new file mode 100644 index 00000000000..9c63bb9d884 --- /dev/null +++ b/src/pybind/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py26, py27 + +[testenv] +deps= +commands= -- cgit v1.2.1 From 0c171f45a5f1ff1d4339076d8890c2c8c8a672f4 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 16 Aug 2013 13:22:35 -0400 Subject: add a python specific gitignore Signed-off-by: Alfredo Deza --- src/pybind/.gitignore | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 src/pybind/.gitignore diff --git a/src/pybind/.gitignore b/src/pybind/.gitignore new file mode 100644 index 00000000000..45b83ef4807 --- /dev/null +++ b/src/pybind/.gitignore @@ -0,0 +1,34 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject -- cgit v1.2.1 From fcffdec8a86b867c2acb128b7bcfe18e0c39b291 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 16 Aug 2013 13:23:13 -0400 Subject: create the cephfs package Signed-off-by: Alfredo Deza --- src/pybind/cephfs/MANIFEST.in | 2 + src/pybind/cephfs/README.rst | 0 src/pybind/cephfs/cephfs.py | 340 ++++++++++++++++++++++++++++++++++++++++++ src/pybind/cephfs/setup.py | 32 ++++ src/pybind/cephfs/tox.ini | 6 + 5 files changed, 380 insertions(+) create mode 100644 src/pybind/cephfs/MANIFEST.in create mode 100644 src/pybind/cephfs/README.rst create mode 100644 src/pybind/cephfs/cephfs.py create mode 100644 src/pybind/cephfs/setup.py create mode 100644 src/pybind/cephfs/tox.ini diff --git a/src/pybind/cephfs/MANIFEST.in b/src/pybind/cephfs/MANIFEST.in new file mode 100644 index 00000000000..3c01b12a5a5 --- /dev/null +++ b/src/pybind/cephfs/MANIFEST.in @@ -0,0 +1,2 @@ +include setup.py +include README.rst diff --git a/src/pybind/cephfs/README.rst b/src/pybind/cephfs/README.rst new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/cephfs/cephfs.py b/src/pybind/cephfs/cephfs.py new file mode 100644 index 00000000000..80b7e4b773f --- /dev/null +++ b/src/pybind/cephfs/cephfs.py @@ -0,0 +1,340 @@ +""" +This module is a thin wrapper around libcephfs. +""" +from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_int, c_long, c_uint, c_ulong, \ + create_string_buffer, byref, Structure +import errno + +class Error(Exception): + pass + +class PermissionError(Error): + pass + +class ObjectNotFound(Error): + pass + +class NoData(Error): + pass + +class ObjectExists(Error): + pass + +class IOError(Error): + pass + +class NoSpace(Error): + pass + +class IncompleteWriteError(Error): + pass + +class LibCephFSStateError(Error): + pass + +def make_ex(ret, msg): + """ + Translate a libcephfs return code into an exception. + + :param ret: the return code + :type ret: int + :param msg: the error message to use + :type msg: str + :returns: a subclass of :class:`Error` + """ + + errors = { + errno.EPERM : PermissionError, + errno.ENOENT : ObjectNotFound, + errno.EIO : IOError, + errno.ENOSPC : NoSpace, + errno.EEXIST : ObjectExists, + errno.ENODATA : NoData + } + ret = abs(ret) + if ret in errors: + return errors[ret](msg) + else: + return Error(msg + (": error code %d" % ret)) + +class cephfs_statvfs(Structure): + _fields_ = [("f_bsize", c_uint), + ("f_frsize", c_uint), + ("f_blocks", c_uint), + ("f_bfree", c_uint), + ("f_bavail", c_uint), + ("f_files", c_uint), + ("f_ffree", c_uint), + ("f_favail", c_uint), + ("f_fsid", c_uint), + ("f_flag", c_uint), + ("f_namemax", c_uint)] + +# struct timespec { +# long int tv_sec; +# long int tv_nsec; +# } +class cephfs_timespec(Structure): + _fields_ = [('tv_sec', c_long), + ('tv_nsec', c_long)] + +# struct stat { +# unsigned long st_dev; +# unsigned long st_ino; +# unsigned long st_nlink; +# unsigned int st_mode; +# unsigned int st_uid; +# unsigned int st_gid; +# int __pad0; +# unsigned long st_rdev; +# long int st_size; +# long int st_blksize; +# long int st_blocks; +# struct timespec st_atim; +# struct timespec st_mtim; +# struct timespec st_ctim; +# long int __unused[3]; +# }; +class cephfs_stat(Structure): + _fields_ = [('st_dev', c_ulong), # ID of device containing file + ('st_ino', c_ulong), # inode number + ('st_nlink', c_ulong), # number of hard links + ('st_mode', c_uint), # protection + ('st_uid', c_uint), # user ID of owner + ('st_gid', c_uint), # group ID of owner + ('__pad0', c_int), + ('st_rdev', c_ulong), # device ID (if special file) + ('st_size', c_long), # total size, in bytes + ('st_blksize', c_long), # blocksize for file system I/O + ('st_blocks', c_long), # number of 512B blocks allocated + ('st_atime', cephfs_timespec), # time of last access + ('st_mtime', cephfs_timespec), # time of last modification + ('st_ctime', cephfs_timespec), # time of last status change + ('__unused1', c_long), + ('__unused2', c_long), + ('__unused3', c_long) ] + +class LibCephFS(object): + """libcephfs python wrapper""" + def require_state(self, *args): + for a in args: + if self.state == a: + return + raise LibCephFSStateError("You cannot perform that operation on a " + "CephFS object in state %s." % (self.state)) + + def __init__(self, conf=None, conffile=None): + self.libcephfs = CDLL('libcephfs.so.1') + self.cluster = c_void_p() + + if conffile is not None and not isinstance(conffile, str): + raise TypeError('conffile must be a string or None') + ret = self.libcephfs.ceph_create(byref(self.cluster), c_char_p(0)) + if ret != 0: + raise Error("libcephfs_initialize failed with error code: %d" %ret) + self.state = "configuring" + if conffile is not None: + # read the default conf file when '' is given + if conffile == '': + conffile = None + self.conf_read_file(conffile) + if conf is not None: + for key, value in conf.iteritems(): + self.conf_set(key, value) + + def conf_read_file(self, conffile=None): + if conffile is not None and not isinstance(conffile, str): + raise TypeError('conffile param must be a string') + ret = self.libcephfs.ceph_conf_read_file(self.cluster, c_char_p(conffile)) + if ret != 0: + raise make_ex(ret, "error calling conf_read_file") + + def shutdown(self): + """ + Unmount and destroy the ceph mount handle. + """ + if self.state != "shutdown": + self.libcephfs.ceph_shutdown(self.cluster) + self.state = "shutdown" + + def __enter__(self): + self.mount() + return self + + def __exit__(self, type_, value, traceback): + self.shutdown() + return False + + def __del__(self): + self.shutdown() + + def version(self): + """ + Get the version number of the ``libcephfs`` C library. + + :returns: a tuple of ``(major, minor, extra)`` components of the + libcephfs version + """ + major = c_int(0) + minor = c_int(0) + extra = c_int(0) + self.libcephfs.ceph_version(byref(major), byref(minor), byref(extra)) + return (major.value, minor.value, extra.value) + + def conf_get(self, option): + self.require_state("configuring", "connected") + if not isinstance(option, str): + raise TypeError('option must be a string') + length = 20 + while True: + ret_buf = create_string_buffer(length) + ret = self.libcephfs.ceph_conf_get(self.cluster, option, + ret_buf, c_size_t(length)) + if ret == 0: + return ret_buf.value + elif ret == -errno.ENAMETOOLONG: + length = length * 2 + elif ret == -errno.ENOENT: + return None + else: + raise make_ex(ret, "error calling conf_get") + + def conf_set(self, option, val): + self.require_state("configuring", "connected") + if not isinstance(option, str): + raise TypeError('option must be a string') + if not isinstance(val, str): + raise TypeError('val must be a string') + ret = self.libcephfs.ceph_conf_set(self.cluster, c_char_p(option), + c_char_p(val)) + if ret != 0: + raise make_ex(ret, "error calling conf_set") + + def mount(self): + self.require_state("configuring") + ret = self.libcephfs.ceph_mount(self.cluster, "/") + if ret != 0: + raise make_ex(ret, "error calling ceph_mount") + self.state = "mounted" + + def statfs(self, path): + self.require_state("mounted") + statbuf = cephfs_statvfs() + ret = self.libcephfs.ceph_statfs(self.cluster, c_char_p(path), byref(statbuf)) + if ret < 0: + raise make_ex(ret, "statfs failed: %s" % path) + return {'f_bsize': statbuf.f_bsize, + 'f_frsize': statbuf.f_frsize, + 'f_blocks': statbuf.f_blocks, + 'f_bfree': statbuf.f_bfree, + 'f_bavail': statbuf.f_bavail, + 'f_files': statbuf.f_files, + 'f_ffree': statbuf.f_ffree, + 'f_favail': statbuf.f_favail, + 'f_fsid': statbuf.f_fsid, + 'f_flag': statbuf.f_flag, + 'f_namemax': statbuf.f_namemax } + + def sync_fs(self): + self.require_state("mounted") + ret = self.libcephfs.ceph_sync_fs(self.cluster) + if ret < 0: + raise make_ex(ret, "sync_fs failed") + + def getcwd(self): + self.require_state("mounted") + return self.libcephfs.ceph_getcwd(self.cluster) + + def chdir(self, path): + self.require_state("mounted") + ret = self.libcephfs.ceph_chdir(self.cluster, c_char_p(path)) + if ret < 0: + raise make_ex(ret, "chdir failed") + + def mkdir(self, path, mode): + self.require_state("mounted") + if not isinstance(path, str): + raise TypeError('path must be a string') + ret = self.libcephfs.ceph_mkdir(self.cluster, c_char_p(path), c_int(mode)) + if ret < 0: + raise make_ex(ret, "error in mkdir '%s'" % path) + + def mkdirs(self, path, mode): + self.require_state("mounted") + if not isinstance(path, str): + raise TypeError('path must be a string') + if not isinstance(mode, int): + raise TypeError('mode must be an int') + ret = self.libcephfs.ceph_mkdir(self.cluster, c_char_p(path), c_int(mode)) + if ret < 0: + raise make_ex(ret, "error in mkdirs '%s'" % path) + + def open(self, path, flags, mode): + self.require_state("mounted") + if not isinstance(path, str): + raise TypeError('path must be a string') + if not isinstance(mode, int): + raise TypeError('mode must be an int') + if not isinstance(flags, int): + raise TypeError('flags must be an int') + ret = self.libcephfs.ceph_open(self.cluster, c_char_p(path), c_int(flags), c_int(mode)) + if ret < 0: + raise make_ex(ret, "error in open '%s'" % path) + return ret + + def close(self, fd): + self.require_state("mounted") + ret = self.libcephfs.ceph_close(self.cluster, c_int(fd)) + if ret < 0: + raise make_ex(ret, "error in close") + + def setxattr(self, path, name, value, flags): + if not isinstance(path, str): + raise TypeError('path must be a string') + if not isinstance(name, str): + raise TypeError('name must be a string') + if not isinstance(value, str): + raise TypeError('value must be a string') + self.require_state("mounted") + ret = self.libcephfs.ceph_setxattr( + self.cluster, + c_char_p(path), + c_char_p(name), + c_void_p(value), + c_size_t(len(value)), + c_int(flags)) + if ret < 0: + raise make_ex(ret, "error in setxattr") + + def stat(self, path): + self.require_state("mounted") + if not isinstance(path, str): + raise TypeError('path must be a string') + statbuf = cephfs_stat() + ret = self.libcephfs.ceph_stat( + self.cluster, + c_char_p(path), + byref(statbuf)) + if ret < 0: + raise make_ex(ret, "error in stat: %s" % path) + return {'st_dev': statbuf.st_dev, + 'st_ino': statbuf.st_ino, + 'st_mode': statbuf.st_mode, + 'st_nlink': statbuf.st_nlink, + 'st_uid': statbuf.st_uid, + 'st_gid': statbuf.st_gid, + 'st_rdev': statbuf.st_rdev, + 'st_size': statbuf.st_size, + 'st_blksize': statbuf.st_blksize, + 'st_blocks': statbuf.st_blocks, + 'st_atime': statbuf.st_atime, + 'st_mtime': statbuf.st_mtime, + 'st_ctime': statbuf.st_ctime } + + def unlink(self, path): + self.require_state("mounted") + ret = self.libcephfs.ceph_unlink( + self.cluster, + c_char_p(path)) + if ret < 0: + raise make_ex(ret, "error in unlink: %s" % path) diff --git a/src/pybind/cephfs/setup.py b/src/pybind/cephfs/setup.py new file mode 100644 index 00000000000..bbd96dbb9ee --- /dev/null +++ b/src/pybind/cephfs/setup.py @@ -0,0 +1,32 @@ +from setuptools import setup, find_packages +import os + + +def long_description(): + readme = os.path.join(os.path.dirname(__file__), 'README.rst') + return open(readme).read() + + +setup( + name = 'cephfs', + description = 'Bindings for cephfs [ceph]', + packages=find_packages(), + author = 'Inktank', + author_email = 'ceph-devel@vger.kernel.org', + version = '0.67', #XXX Fix version + license = "LGPL2", + zip_safe = False, + keywords = "ceph, cephfs, bindings, api, cli", + long_description = long_description(), + classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)', + 'Topic :: Software Development :: Libraries', + 'Topic :: Utilities', + 'Topic :: System :: Filesystems', + 'Operating System :: POSIX', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], +) diff --git a/src/pybind/cephfs/tox.ini b/src/pybind/cephfs/tox.ini new file mode 100644 index 00000000000..9c63bb9d884 --- /dev/null +++ b/src/pybind/cephfs/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py26, py27 + +[testenv] +deps= +commands= -- cgit v1.2.1 From e46bcd6a6c482b4cb87919631743365814a54256 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 16 Aug 2013 13:33:47 -0400 Subject: move cephfs.py Signed-off-by: Alfredo Deza --- src/pybind/cephfs.py | 340 --------------------------------------------- src/pybind/cephfs/setup.py | 2 +- 2 files changed, 1 insertion(+), 341 deletions(-) delete mode 100644 src/pybind/cephfs.py diff --git a/src/pybind/cephfs.py b/src/pybind/cephfs.py deleted file mode 100644 index 80b7e4b773f..00000000000 --- a/src/pybind/cephfs.py +++ /dev/null @@ -1,340 +0,0 @@ -""" -This module is a thin wrapper around libcephfs. -""" -from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_int, c_long, c_uint, c_ulong, \ - create_string_buffer, byref, Structure -import errno - -class Error(Exception): - pass - -class PermissionError(Error): - pass - -class ObjectNotFound(Error): - pass - -class NoData(Error): - pass - -class ObjectExists(Error): - pass - -class IOError(Error): - pass - -class NoSpace(Error): - pass - -class IncompleteWriteError(Error): - pass - -class LibCephFSStateError(Error): - pass - -def make_ex(ret, msg): - """ - Translate a libcephfs return code into an exception. - - :param ret: the return code - :type ret: int - :param msg: the error message to use - :type msg: str - :returns: a subclass of :class:`Error` - """ - - errors = { - errno.EPERM : PermissionError, - errno.ENOENT : ObjectNotFound, - errno.EIO : IOError, - errno.ENOSPC : NoSpace, - errno.EEXIST : ObjectExists, - errno.ENODATA : NoData - } - ret = abs(ret) - if ret in errors: - return errors[ret](msg) - else: - return Error(msg + (": error code %d" % ret)) - -class cephfs_statvfs(Structure): - _fields_ = [("f_bsize", c_uint), - ("f_frsize", c_uint), - ("f_blocks", c_uint), - ("f_bfree", c_uint), - ("f_bavail", c_uint), - ("f_files", c_uint), - ("f_ffree", c_uint), - ("f_favail", c_uint), - ("f_fsid", c_uint), - ("f_flag", c_uint), - ("f_namemax", c_uint)] - -# struct timespec { -# long int tv_sec; -# long int tv_nsec; -# } -class cephfs_timespec(Structure): - _fields_ = [('tv_sec', c_long), - ('tv_nsec', c_long)] - -# struct stat { -# unsigned long st_dev; -# unsigned long st_ino; -# unsigned long st_nlink; -# unsigned int st_mode; -# unsigned int st_uid; -# unsigned int st_gid; -# int __pad0; -# unsigned long st_rdev; -# long int st_size; -# long int st_blksize; -# long int st_blocks; -# struct timespec st_atim; -# struct timespec st_mtim; -# struct timespec st_ctim; -# long int __unused[3]; -# }; -class cephfs_stat(Structure): - _fields_ = [('st_dev', c_ulong), # ID of device containing file - ('st_ino', c_ulong), # inode number - ('st_nlink', c_ulong), # number of hard links - ('st_mode', c_uint), # protection - ('st_uid', c_uint), # user ID of owner - ('st_gid', c_uint), # group ID of owner - ('__pad0', c_int), - ('st_rdev', c_ulong), # device ID (if special file) - ('st_size', c_long), # total size, in bytes - ('st_blksize', c_long), # blocksize for file system I/O - ('st_blocks', c_long), # number of 512B blocks allocated - ('st_atime', cephfs_timespec), # time of last access - ('st_mtime', cephfs_timespec), # time of last modification - ('st_ctime', cephfs_timespec), # time of last status change - ('__unused1', c_long), - ('__unused2', c_long), - ('__unused3', c_long) ] - -class LibCephFS(object): - """libcephfs python wrapper""" - def require_state(self, *args): - for a in args: - if self.state == a: - return - raise LibCephFSStateError("You cannot perform that operation on a " - "CephFS object in state %s." % (self.state)) - - def __init__(self, conf=None, conffile=None): - self.libcephfs = CDLL('libcephfs.so.1') - self.cluster = c_void_p() - - if conffile is not None and not isinstance(conffile, str): - raise TypeError('conffile must be a string or None') - ret = self.libcephfs.ceph_create(byref(self.cluster), c_char_p(0)) - if ret != 0: - raise Error("libcephfs_initialize failed with error code: %d" %ret) - self.state = "configuring" - if conffile is not None: - # read the default conf file when '' is given - if conffile == '': - conffile = None - self.conf_read_file(conffile) - if conf is not None: - for key, value in conf.iteritems(): - self.conf_set(key, value) - - def conf_read_file(self, conffile=None): - if conffile is not None and not isinstance(conffile, str): - raise TypeError('conffile param must be a string') - ret = self.libcephfs.ceph_conf_read_file(self.cluster, c_char_p(conffile)) - if ret != 0: - raise make_ex(ret, "error calling conf_read_file") - - def shutdown(self): - """ - Unmount and destroy the ceph mount handle. - """ - if self.state != "shutdown": - self.libcephfs.ceph_shutdown(self.cluster) - self.state = "shutdown" - - def __enter__(self): - self.mount() - return self - - def __exit__(self, type_, value, traceback): - self.shutdown() - return False - - def __del__(self): - self.shutdown() - - def version(self): - """ - Get the version number of the ``libcephfs`` C library. - - :returns: a tuple of ``(major, minor, extra)`` components of the - libcephfs version - """ - major = c_int(0) - minor = c_int(0) - extra = c_int(0) - self.libcephfs.ceph_version(byref(major), byref(minor), byref(extra)) - return (major.value, minor.value, extra.value) - - def conf_get(self, option): - self.require_state("configuring", "connected") - if not isinstance(option, str): - raise TypeError('option must be a string') - length = 20 - while True: - ret_buf = create_string_buffer(length) - ret = self.libcephfs.ceph_conf_get(self.cluster, option, - ret_buf, c_size_t(length)) - if ret == 0: - return ret_buf.value - elif ret == -errno.ENAMETOOLONG: - length = length * 2 - elif ret == -errno.ENOENT: - return None - else: - raise make_ex(ret, "error calling conf_get") - - def conf_set(self, option, val): - self.require_state("configuring", "connected") - if not isinstance(option, str): - raise TypeError('option must be a string') - if not isinstance(val, str): - raise TypeError('val must be a string') - ret = self.libcephfs.ceph_conf_set(self.cluster, c_char_p(option), - c_char_p(val)) - if ret != 0: - raise make_ex(ret, "error calling conf_set") - - def mount(self): - self.require_state("configuring") - ret = self.libcephfs.ceph_mount(self.cluster, "/") - if ret != 0: - raise make_ex(ret, "error calling ceph_mount") - self.state = "mounted" - - def statfs(self, path): - self.require_state("mounted") - statbuf = cephfs_statvfs() - ret = self.libcephfs.ceph_statfs(self.cluster, c_char_p(path), byref(statbuf)) - if ret < 0: - raise make_ex(ret, "statfs failed: %s" % path) - return {'f_bsize': statbuf.f_bsize, - 'f_frsize': statbuf.f_frsize, - 'f_blocks': statbuf.f_blocks, - 'f_bfree': statbuf.f_bfree, - 'f_bavail': statbuf.f_bavail, - 'f_files': statbuf.f_files, - 'f_ffree': statbuf.f_ffree, - 'f_favail': statbuf.f_favail, - 'f_fsid': statbuf.f_fsid, - 'f_flag': statbuf.f_flag, - 'f_namemax': statbuf.f_namemax } - - def sync_fs(self): - self.require_state("mounted") - ret = self.libcephfs.ceph_sync_fs(self.cluster) - if ret < 0: - raise make_ex(ret, "sync_fs failed") - - def getcwd(self): - self.require_state("mounted") - return self.libcephfs.ceph_getcwd(self.cluster) - - def chdir(self, path): - self.require_state("mounted") - ret = self.libcephfs.ceph_chdir(self.cluster, c_char_p(path)) - if ret < 0: - raise make_ex(ret, "chdir failed") - - def mkdir(self, path, mode): - self.require_state("mounted") - if not isinstance(path, str): - raise TypeError('path must be a string') - ret = self.libcephfs.ceph_mkdir(self.cluster, c_char_p(path), c_int(mode)) - if ret < 0: - raise make_ex(ret, "error in mkdir '%s'" % path) - - def mkdirs(self, path, mode): - self.require_state("mounted") - if not isinstance(path, str): - raise TypeError('path must be a string') - if not isinstance(mode, int): - raise TypeError('mode must be an int') - ret = self.libcephfs.ceph_mkdir(self.cluster, c_char_p(path), c_int(mode)) - if ret < 0: - raise make_ex(ret, "error in mkdirs '%s'" % path) - - def open(self, path, flags, mode): - self.require_state("mounted") - if not isinstance(path, str): - raise TypeError('path must be a string') - if not isinstance(mode, int): - raise TypeError('mode must be an int') - if not isinstance(flags, int): - raise TypeError('flags must be an int') - ret = self.libcephfs.ceph_open(self.cluster, c_char_p(path), c_int(flags), c_int(mode)) - if ret < 0: - raise make_ex(ret, "error in open '%s'" % path) - return ret - - def close(self, fd): - self.require_state("mounted") - ret = self.libcephfs.ceph_close(self.cluster, c_int(fd)) - if ret < 0: - raise make_ex(ret, "error in close") - - def setxattr(self, path, name, value, flags): - if not isinstance(path, str): - raise TypeError('path must be a string') - if not isinstance(name, str): - raise TypeError('name must be a string') - if not isinstance(value, str): - raise TypeError('value must be a string') - self.require_state("mounted") - ret = self.libcephfs.ceph_setxattr( - self.cluster, - c_char_p(path), - c_char_p(name), - c_void_p(value), - c_size_t(len(value)), - c_int(flags)) - if ret < 0: - raise make_ex(ret, "error in setxattr") - - def stat(self, path): - self.require_state("mounted") - if not isinstance(path, str): - raise TypeError('path must be a string') - statbuf = cephfs_stat() - ret = self.libcephfs.ceph_stat( - self.cluster, - c_char_p(path), - byref(statbuf)) - if ret < 0: - raise make_ex(ret, "error in stat: %s" % path) - return {'st_dev': statbuf.st_dev, - 'st_ino': statbuf.st_ino, - 'st_mode': statbuf.st_mode, - 'st_nlink': statbuf.st_nlink, - 'st_uid': statbuf.st_uid, - 'st_gid': statbuf.st_gid, - 'st_rdev': statbuf.st_rdev, - 'st_size': statbuf.st_size, - 'st_blksize': statbuf.st_blksize, - 'st_blocks': statbuf.st_blocks, - 'st_atime': statbuf.st_atime, - 'st_mtime': statbuf.st_mtime, - 'st_ctime': statbuf.st_ctime } - - def unlink(self, path): - self.require_state("mounted") - ret = self.libcephfs.ceph_unlink( - self.cluster, - c_char_p(path)) - if ret < 0: - raise make_ex(ret, "error in unlink: %s" % path) diff --git a/src/pybind/cephfs/setup.py b/src/pybind/cephfs/setup.py index bbd96dbb9ee..596212e4867 100644 --- a/src/pybind/cephfs/setup.py +++ b/src/pybind/cephfs/setup.py @@ -13,7 +13,7 @@ setup( packages=find_packages(), author = 'Inktank', author_email = 'ceph-devel@vger.kernel.org', - version = '0.67', #XXX Fix version + version = '0.67', license = "LGPL2", zip_safe = False, keywords = "ceph, cephfs, bindings, api, cli", -- cgit v1.2.1 From e65af11999a80667972add12265c0aded151d7d6 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 16 Aug 2013 13:35:29 -0400 Subject: create the rados package Signed-off-by: Alfredo Deza --- src/pybind/rados/MANIFEST.in | 2 + src/pybind/rados/README.rst | 0 src/pybind/rados/rados.py | 1648 ++++++++++++++++++++++++++++++++++++++++++ src/pybind/rados/setup.py | 32 + src/pybind/rados/tox.ini | 6 + 5 files changed, 1688 insertions(+) create mode 100644 src/pybind/rados/MANIFEST.in create mode 100644 src/pybind/rados/README.rst create mode 100644 src/pybind/rados/rados.py create mode 100644 src/pybind/rados/setup.py create mode 100644 src/pybind/rados/tox.ini diff --git a/src/pybind/rados/MANIFEST.in b/src/pybind/rados/MANIFEST.in new file mode 100644 index 00000000000..3c01b12a5a5 --- /dev/null +++ b/src/pybind/rados/MANIFEST.in @@ -0,0 +1,2 @@ +include setup.py +include README.rst diff --git a/src/pybind/rados/README.rst b/src/pybind/rados/README.rst new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/rados/rados.py b/src/pybind/rados/rados.py new file mode 100644 index 00000000000..7768f8c39d3 --- /dev/null +++ b/src/pybind/rados/rados.py @@ -0,0 +1,1648 @@ +""" +This module is a thin wrapper around librados. + +Copyright 2011, Hannu Valtonen +""" +from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_char, c_int, c_long, \ + c_ulong, create_string_buffer, byref, Structure, c_uint64, c_ubyte, \ + pointer, CFUNCTYPE +import ctypes +import errno +import threading +import time +from datetime import datetime + +ANONYMOUS_AUID = 0xffffffffffffffff +ADMIN_AUID = 0 + +class Error(Exception): + """ `Error` class, derived from `Exception` """ + pass + +class PermissionError(Error): + """ `PermissionError` class, derived from `Error` """ + pass + +class ObjectNotFound(Error): + """ `ObjectNotFound` class, derived from `Error` """ + pass + +class NoData(Error): + """ `NoData` class, derived from `Error` """ + pass + +class ObjectExists(Error): + """ `ObjectExists` class, derived from `Error` """ + pass + +class IOError(Error): + """ `IOError` class, derived from `Error` """ + pass + +class NoSpace(Error): + """ `NoSpace` class, derived from `Error` """ + pass + +class IncompleteWriteError(Error): + """ `IncompleteWriteError` class, derived from `Error` """ + pass + +class RadosStateError(Error): + """ `RadosStateError` class, derived from `Error` """ + pass + +class IoctxStateError(Error): + """ `IoctxStateError` class, derived from `Error` """ + pass + +class ObjectStateError(Error): + """ `ObjectStateError` class, derived from `Error` """ + pass + +class LogicError(Error): + """ `` class, derived from `Error` """ + pass + +def make_ex(ret, msg): + """ + Translate a librados return code into an exception. + + :param ret: the return code + :type ret: int + :param msg: the error message to use + :type msg: str + :returns: a subclass of :class:`Error` + """ + + errors = { + errno.EPERM : PermissionError, + errno.ENOENT : ObjectNotFound, + errno.EIO : IOError, + errno.ENOSPC : NoSpace, + errno.EEXIST : ObjectExists, + errno.ENODATA : NoData + } + ret = abs(ret) + if ret in errors: + return errors[ret](msg) + else: + return Error(msg + (": errno %s" % errno.errorcode[ret])) + +class rados_pool_stat_t(Structure): + """ Usage information for a pool """ + _fields_ = [("num_bytes", c_uint64), + ("num_kb", c_uint64), + ("num_objects", c_uint64), + ("num_object_clones", c_uint64), + ("num_object_copies", c_uint64), + ("num_objects_missing_on_primary", c_uint64), + ("num_objects_unfound", c_uint64), + ("num_objects_degraded", c_uint64), + ("num_rd", c_uint64), + ("num_rd_kb", c_uint64), + ("num_wr", c_uint64), + ("num_wr_kb", c_uint64)] + +class rados_cluster_stat_t(Structure): + """ Cluster-wide usage information """ + _fields_ = [("kb", c_uint64), + ("kb_used", c_uint64), + ("kb_avail", c_uint64), + ("num_objects", c_uint64)] + +class Version(object): + """ Version information """ + def __init__(self, major, minor, extra): + self.major = major + self.minor = minor + self.extra = extra + + def __str__(self): + return "%d.%d.%d" % (self.major, self.minor, self.extra) + +class RadosThread(threading.Thread): + def __init__(self, target, args=None): + self.args = args + self.target = target + threading.Thread.__init__(self) + + def run(self): + self.retval = self.target(*self.args) + +# time in seconds between each call to t.join() for child thread +POLL_TIME_INCR = 0.5 + +def run_in_thread(target, args, timeout=0): + import sys + interrupt = False + + countdown = timeout + t = RadosThread(target, args) + + # allow the main thread to exit (presumably, avoid a join() on this + # subthread) before this thread terminates. This allows SIGINT + # exit of a blocked call. See below. + t.daemon = True + + t.start() + try: + # poll for thread exit + while t.is_alive(): + t.join(POLL_TIME_INCR) + if timeout: + countdown = countdown - POLL_TIME_INCR + if countdown <= 0: + raise KeyboardInterrupt + + t.join() # in case t exits before reaching the join() above + except KeyboardInterrupt: + # ..but allow SIGINT to terminate the waiting. Note: this + # relies on the Linux kernel behavior of delivering the signal + # to the main thread in preference to any subthread (all that's + # strictly guaranteed is that *some* thread that has the signal + # unblocked will receive it). But there doesn't seem to be + # any interface to create t with SIGINT blocked. + interrupt = True + + if interrupt: + t.retval = -errno.EINTR + return t.retval + +class Rados(object): + """librados python wrapper""" + def require_state(self, *args): + """ + Checks if the Rados object is in a special state + + :raises: RadosStateError + """ + for a in args: + if self.state == a: + return + raise RadosStateError("You cannot perform that operation on a \ +Rados object in state %s." % (self.state)) + + def __init__(self, rados_id=None, name=None, clustername=None, + conf_defaults=None, conffile=None, conf=None, flags=0): + self.librados = CDLL('librados.so.2') + self.cluster = c_void_p() + self.rados_id = rados_id + if rados_id is not None and not isinstance(rados_id, str): + raise TypeError('rados_id must be a string or None') + if conffile is not None and not isinstance(conffile, str): + raise TypeError('conffile must be a string or None') + if name is not None and not isinstance(name, str): + raise TypeError('name must be a string or None') + if clustername is not None and not isinstance(clustername, str): + raise TypeError('clustername must be a string or None') + if rados_id and name: + raise Error("Rados(): can't supply both rados_id and name") + if rados_id: + name = 'client.' + rados_id + if name is None: + name = 'client.admin' + if clustername is None: + clustername = 'ceph' + ret = run_in_thread(self.librados.rados_create2, + (byref(self.cluster), c_char_p(clustername), + c_char_p(name), c_uint64(flags))) + + if ret != 0: + raise Error("rados_initialize failed with error code: %d" % ret) + self.state = "configuring" + # order is important: conf_defaults, then conffile, then conf + if conf_defaults: + for key, value in conf_defaults.iteritems(): + self.conf_set(key, value) + if conffile is not None: + # read the default conf file when '' is given + if conffile == '': + conffile = None + self.conf_read_file(conffile) + if conf: + for key, value in conf.iteritems(): + self.conf_set(key, value) + + def shutdown(self): + """ + Disconnects from the cluster. + """ + if (self.__dict__.has_key("state") and self.state != "shutdown"): + run_in_thread(self.librados.rados_shutdown, (self.cluster,)) + self.state = "shutdown" + + def __enter__(self): + self.connect() + return self + + def __exit__(self, type_, value, traceback): + self.shutdown() + return False + + def __del__(self): + self.shutdown() + + def version(self): + """ + Get the version number of the ``librados`` C library. + + :returns: a tuple of ``(major, minor, extra)`` components of the + librados version + """ + major = c_int(0) + minor = c_int(0) + extra = c_int(0) + run_in_thread(self.librados.rados_version, + (byref(major), byref(minor), byref(extra))) + return Version(major.value, minor.value, extra.value) + + def conf_read_file(self, path=None): + """ + Configure the cluster handle using a Ceph config file. + + :param path: path to the config file + :type path: str + """ + self.require_state("configuring", "connected") + if path is not None and not isinstance(path, str): + raise TypeError('path must be a string') + ret = run_in_thread(self.librados.rados_conf_read_file, + (self.cluster, c_char_p(path))) + if (ret != 0): + raise make_ex(ret, "error calling conf_read_file") + + def conf_parse_argv(self, args): + """ + Parse known arguments from args, and remove; returned + args contain only those unknown to ceph + """ + self.require_state("configuring", "connected") + if not args: + return + # create instances of arrays of c_char_p's, both len(args) long + # cretargs will always be a subset of cargs (perhaps identical) + cargs = (c_char_p * len(args))(*args) + cretargs = (c_char_p * len(args))() + ret = run_in_thread(self.librados.rados_conf_parse_argv_remainder, + (self.cluster, len(args), cargs, cretargs)) + if ret: + raise make_ex(ret, "error calling conf_parse_argv_remainder") + + # cretargs was allocated with fixed length; collapse return + # list to eliminate any missing args + + retargs = [a for a in cretargs if a is not None] + return retargs + + def conf_get(self, option): + """ + Get the value of a configuration option + + :param option: which option to read + :type option: str + + :returns: str - value of the option or None + :raises: :class:`TypeError` + """ + self.require_state("configuring", "connected") + if not isinstance(option, str): + raise TypeError('option must be a string') + length = 20 + while True: + ret_buf = create_string_buffer(length) + ret = run_in_thread(self.librados.rados_conf_get, + (self.cluster, c_char_p(option), ret_buf, + c_size_t(length))) + if (ret == 0): + return ret_buf.value + elif (ret == -errno.ENAMETOOLONG): + length = length * 2 + elif (ret == -errno.ENOENT): + return None + else: + raise make_ex(ret, "error calling conf_get") + + def conf_set(self, option, val): + """ + Set the value of a configuration option + + :param option: which option to set + :type option: str + :param option: value of the option + :type option: str + + :raises: :class:`TypeError`, :class:`ObjectNotFound` + """ + self.require_state("configuring", "connected") + if not isinstance(option, str): + raise TypeError('option must be a string') + if not isinstance(val, str): + raise TypeError('val must be a string') + ret = run_in_thread(self.librados.rados_conf_set, + (self.cluster, c_char_p(option), c_char_p(val))) + if (ret != 0): + raise make_ex(ret, "error calling conf_set") + + def connect(self, timeout=0): + """ + Connect to the cluster. + """ + self.require_state("configuring") + ret = run_in_thread(self.librados.rados_connect, (self.cluster,), + timeout) + if (ret != 0): + raise make_ex(ret, "error calling connect") + self.state = "connected" + + def get_cluster_stats(self): + """ + Read usage info about the cluster + + This tells you total space, space used, space available, and number + of objects. These are not updated immediately when data is written, + they are eventually consistent. + + :returns: dict - contains the following keys: + + *``kb`` (int) - total space + + *``kb_used`` (int) - space used + + *``kb_avail`` (int) - free space available + + *``num_objects`` (int) - number of objects + + """ + stats = rados_cluster_stat_t() + ret = run_in_thread(self.librados.rados_cluster_stat, + (self.cluster, byref(stats))) + if ret < 0: + raise make_ex( + ret, "Rados.get_cluster_stats(%s): get_stats failed" % self.rados_id) + return {'kb': stats.kb, + 'kb_used': stats.kb_used, + 'kb_avail': stats.kb_avail, + 'num_objects': stats.num_objects} + + def pool_exists(self, pool_name): + """ + Checks if a given pool exists. + + :param pool_name: name of the pool to check + :type pool_name: str + + :raises: :class:`TypeError`, :class:`Error` + :returns: bool - whether the pool exists, false otherwise. + """ + self.require_state("connected") + if not isinstance(pool_name, str): + raise TypeError('pool_name must be a string') + ret = run_in_thread(self.librados.rados_pool_lookup, + (self.cluster, c_char_p(pool_name))) + if (ret >= 0): + return True + elif (ret == -errno.ENOENT): + return False + else: + raise make_ex(ret, "error looking up pool '%s'" % pool_name) + + def create_pool(self, pool_name, auid=None, crush_rule=None): + """ + Create a pool: + - with default settings: if auid=None and crush_rule=None + - owned by a specific auid: auid given and crush_rule=None + - with a specific CRUSH rule: if auid=None and crush_rule given + - with a specific CRUSH rule and auid: if auid and crush_rule given + + :param pool_name: name of the pool to create + :type pool_name: str + :param auid: the id of the owner of the new pool + :type auid: int + :param crush_rule: rule to use for placement in the new pool + :type crush_rule: str + + :raises: :class:`TypeError`, :class:`Error` + """ + self.require_state("connected") + if not isinstance(pool_name, str): + raise TypeError('pool_name must be a string') + if crush_rule is not None and not isinstance(crush_rule, str): + raise TypeError('cruse_rule must be a string') + if (auid == None): + if (crush_rule == None): + ret = run_in_thread(self.librados.rados_pool_create, + (self.cluster, c_char_p(pool_name))) + else: + ret = run_in_thread(self.librados.\ + rados_pool_create_with_crush_rule, + (self.cluster, c_char_p(pool_name), + c_ubyte(crush_rule))) + + elif (crush_rule == None): + ret = run_in_thread(self.librados.rados_pool_create_with_auid, + (self.cluster, c_char_p(pool_name), + c_uint64(auid))) + else: + ret = run_in_thread(self.librados.rados_pool_create_with_all, + (self.cluster, c_char_p(pool_name), + c_uint64(auid), c_ubyte(crush_rule))) + if ret < 0: + raise make_ex(ret, "error creating pool '%s'" % pool_name) + + def delete_pool(self, pool_name): + """ + Delete a pool and all data inside it. + + The pool is removed from the cluster immediately, + but the actual data is deleted in the background. + + :param pool_name: name of the pool to delete + :type pool_name: str + + :raises: :class:`TypeError`, :class:`Error` + """ + self.require_state("connected") + if not isinstance(pool_name, str): + raise TypeError('pool_name must be a string') + ret = run_in_thread(self.librados.rados_pool_delete, + (self.cluster, c_char_p(pool_name))) + if ret < 0: + raise make_ex(ret, "error deleting pool '%s'" % pool_name) + + def list_pools(self): + """ + Gets a list of pool names. + + :returns: list - of pool names. + """ + self.require_state("connected") + size = c_size_t(512) + while True: + c_names = create_string_buffer(size.value) + ret = run_in_thread(self.librados.rados_pool_list, + (self.cluster, byref(c_names), size)) + if ret > size.value: + size = c_size_t(ret) + else: + break + return filter(lambda name: name != '', c_names.raw.split('\0')) + + def get_fsid(self): + """ + Get the fsid of the cluster as a hexadecimal string. + + :raises: :class:`Error` + :returns: str - cluster fsid + """ + self.require_state("connected") + buf_len = 37 + fsid = create_string_buffer(buf_len) + ret = run_in_thread(self.librados.rados_cluster_fsid, + (self.cluster, byref(fsid), c_size_t(buf_len))) + if ret < 0: + raise make_ex(ret, "error getting cluster fsid") + return fsid.value + + def open_ioctx(self, ioctx_name): + """ + Create an io context + + The io context allows you to perform operations within a particular + pool. + + :param ioctx_name: name of the pool + :type ioctx_name: str + + :raises: :class:`TypeError`, :class:`Error` + :returns: Ioctx - Rados Ioctx object + """ + self.require_state("connected") + if not isinstance(ioctx_name, str): + raise TypeError('ioctx_name must be a string') + ioctx = c_void_p() + ret = run_in_thread(self.librados.rados_ioctx_create, + (self.cluster, c_char_p(ioctx_name), byref(ioctx))) + if ret < 0: + raise make_ex(ret, "error opening ioctx '%s'" % ioctx_name) + return Ioctx(ioctx_name, self.librados, ioctx) + + def mon_command(self, cmd, inbuf, timeout=0, target=None): + """ + mon_command[_target](cmd, inbuf, outbuf, outbuflen, outs, outslen) + returns (int ret, string outbuf, string outs) + """ + import sys + self.require_state("connected") + outbufp = pointer(pointer(c_char())) + outbuflen = c_long() + outsp = pointer(pointer(c_char())) + outslen = c_long() + cmdarr = (c_char_p * len(cmd))(*cmd) + + if target: + ret = run_in_thread(self.librados.rados_mon_command_target, + (self.cluster, c_char_p(target), cmdarr, + len(cmd), c_char_p(inbuf), len(inbuf), + outbufp, byref(outbuflen), outsp, + byref(outslen)), timeout) + else: + ret = run_in_thread(self.librados.rados_mon_command, + (self.cluster, cmdarr, len(cmd), + c_char_p(inbuf), len(inbuf), + outbufp, byref(outbuflen), outsp, byref(outslen)), + timeout) + + # copy returned memory (ctypes makes a copy, not a reference) + my_outbuf = outbufp.contents[:(outbuflen.value)] + my_outs = outsp.contents[:(outslen.value)] + + # free callee's allocations + if outbuflen.value: + run_in_thread(self.librados.rados_buffer_free, (outbufp.contents,)) + if outslen.value: + run_in_thread(self.librados.rados_buffer_free, (outsp.contents,)) + + return (ret, my_outbuf, my_outs) + + def osd_command(self, osdid, cmd, inbuf, timeout=0): + """ + osd_command(osdid, cmd, inbuf, outbuf, outbuflen, outs, outslen) + returns (int ret, string outbuf, string outs) + """ + import sys + self.require_state("connected") + outbufp = pointer(pointer(c_char())) + outbuflen = c_long() + outsp = pointer(pointer(c_char())) + outslen = c_long() + cmdarr = (c_char_p * len(cmd))(*cmd) + ret = run_in_thread(self.librados.rados_osd_command, + (self.cluster, osdid, cmdarr, len(cmd), + c_char_p(inbuf), len(inbuf), + outbufp, byref(outbuflen), outsp, byref(outslen)), + timeout) + + # copy returned memory (ctypes makes a copy, not a reference) + my_outbuf = outbufp.contents[:(outbuflen.value)] + my_outs = outsp.contents[:(outslen.value)] + + # free callee's allocations + if outbuflen.value: + run_in_thread(self.librados.rados_buffer_free, (outbufp.contents,)) + if outslen.value: + run_in_thread(self.librados.rados_buffer_free, (outsp.contents,)) + + return (ret, my_outbuf, my_outs) + + def pg_command(self, pgid, cmd, inbuf, timeout=0): + """ + pg_command(pgid, cmd, inbuf, outbuf, outbuflen, outs, outslen) + returns (int ret, string outbuf, string outs) + """ + import sys + self.require_state("connected") + outbufp = pointer(pointer(c_char())) + outbuflen = c_long() + outsp = pointer(pointer(c_char())) + outslen = c_long() + cmdarr = (c_char_p * len(cmd))(*cmd) + ret = run_in_thread(self.librados.rados_pg_command, + (self.cluster, c_char_p(pgid), cmdarr, len(cmd), + c_char_p(inbuf), len(inbuf), + outbufp, byref(outbuflen), outsp, byref(outslen)), + timeout) + + # copy returned memory (ctypes makes a copy, not a reference) + my_outbuf = outbufp.contents[:(outbuflen.value)] + my_outs = outsp.contents[:(outslen.value)] + + # free callee's allocations + if outbuflen.value: + run_in_thread(self.librados.rados_buffer_free, (outbufp.contents,)) + if outslen.value: + run_in_thread(self.librados.rados_buffer_free, (outsp.contents,)) + + return (ret, my_outbuf, my_outs) + +class ObjectIterator(object): + """rados.Ioctx Object iterator""" + def __init__(self, ioctx): + self.ioctx = ioctx + self.ctx = c_void_p() + ret = run_in_thread(self.ioctx.librados.rados_objects_list_open, + (self.ioctx.io, byref(self.ctx))) + if ret < 0: + raise make_ex(ret, "error iterating over the objects in ioctx '%s'" \ + % self.ioctx.name) + + def __iter__(self): + return self + + def next(self): + """ + Get the next object name and locator in the pool + + :raises: StopIteration + :returns: next rados.Ioctx Object + """ + key = c_char_p() + locator = c_char_p() + ret = run_in_thread(self.ioctx.librados.rados_objects_list_next, + (self.ctx, byref(key), byref(locator))) + if ret < 0: + raise StopIteration() + return Object(self.ioctx, key.value, locator.value) + + def __del__(self): + run_in_thread(self.ioctx.librados.rados_objects_list_close, (self.ctx,)) + +class XattrIterator(object): + """Extended attribute iterator""" + def __init__(self, ioctx, it, oid): + self.ioctx = ioctx + self.it = it + self.oid = oid + + def __iter__(self): + return self + + def next(self): + """ + Get the next xattr on the object + + :raises: StopIteration + :returns: pair - of name and value of the next Xattr + """ + name_ = c_char_p(0) + val_ = c_char_p(0) + len_ = c_int(0) + ret = run_in_thread(self.ioctx.librados.rados_getxattrs_next, + (self.it, byref(name_), byref(val_), byref(len_))) + if (ret != 0): + raise make_ex(ret, "error iterating over the extended attributes \ +in '%s'" % self.oid) + if name_.value == None: + raise StopIteration() + name = ctypes.string_at(name_) + val = ctypes.string_at(val_, len_) + return (name, val) + + def __del__(self): + run_in_thread(self.ioctx.librados.rados_getxattrs_end, (self.it,)) + +class SnapIterator(object): + """Snapshot iterator""" + def __init__(self, ioctx): + self.ioctx = ioctx + # We don't know how big a buffer we need until we've called the + # function. So use the exponential doubling strategy. + num_snaps = 10 + while True: + self.snaps = (ctypes.c_uint64 * num_snaps)() + ret = run_in_thread(self.ioctx.librados.rados_ioctx_snap_list, + (self.ioctx.io, self.snaps, c_int(num_snaps))) + if (ret >= 0): + self.max_snap = ret + break + elif (ret != -errno.ERANGE): + raise make_ex(ret, "error calling rados_snap_list for \ +ioctx '%s'" % self.ioctx.name) + num_snaps = num_snaps * 2 + self.cur_snap = 0 + + def __iter__(self): + return self + + def next(self): + """ + Get the next Snapshot + + :raises: :class:`Error`, StopIteration + :returns: Snap - next snapshot + """ + if (self.cur_snap >= self.max_snap): + raise StopIteration + snap_id = self.snaps[self.cur_snap] + name_len = 10 + while True: + name = create_string_buffer(name_len) + ret = run_in_thread(self.ioctx.librados.rados_ioctx_snap_get_name, + (self.ioctx.io, c_uint64(snap_id), byref(name), + c_int(name_len))) + if (ret == 0): + name_len = ret + break + elif (ret != -errno.ERANGE): + raise make_ex(ret, "rados_snap_get_name error") + name_len = name_len * 2 + snap = Snap(self.ioctx, name.value, snap_id) + self.cur_snap = self.cur_snap + 1 + return snap + +class Snap(object): + """Snapshot object""" + def __init__(self, ioctx, name, snap_id): + self.ioctx = ioctx + self.name = name + self.snap_id = snap_id + + def __str__(self): + return "rados.Snap(ioctx=%s,name=%s,snap_id=%d)" \ + % (str(self.ioctx), self.name, self.snap_id) + + def get_timestamp(self): + """ + Find when a snapshot in the current pool occurred + + :raises: :class:`Error` + :returns: datetime - the data and time the snapshot was created + """ + snap_time = c_long(0) + ret = run_in_thread(self.ioctx.librados.rados_ioctx_snap_get_stamp, + (self.ioctx.io, self.snap_id, byref(snap_time))) + if (ret != 0): + raise make_ex(ret, "rados_ioctx_snap_get_stamp error") + return datetime.fromtimestamp(snap_time.value) + +class Completion(object): + """completion object""" + def __init__(self, ioctx, rados_comp, oncomplete, onsafe): + self.rados_comp = rados_comp + self.oncomplete = oncomplete + self.onsafe = onsafe + self.ioctx = ioctx + + def wait_for_safe(self): + """ + Is an asynchronous operation safe? + + This does not imply that the safe callback has finished. + + :returns: whether the operation is safe + """ + return run_in_thread(self.ioctx.librados.rados_aio_is_safe, + (self.rados_comp,)) + + def wait_for_complete(self): + """ + Has an asynchronous operation completed? + + This does not imply that the safe callback has finished. + + :returns: whether the operation is completed + """ + return run_in_thread(self.ioctx.librados.rados_aio_is_complete, + (self.rados_comp,)) + + def get_return_value(self): + """ + Get the return value of an asychronous operation + + The return value is set when the operation is complete or safe, + whichever comes first. + + :returns: int - return value of the operation + """ + return run_in_thread(self.ioctx.librados.rados_aio_get_return_value, + (self.rados_comp,)) + + def __del__(self): + """ + Release a completion + + Call this when you no longer need the completion. It may not be + freed immediately if the operation is not acked and committed. + """ + run_in_thread(self.ioctx.librados.rados_aio_release, + (self.rados_comp,)) + +class Ioctx(object): + """rados.Ioctx object""" + def __init__(self, name, librados, io): + self.name = name + self.librados = librados + self.io = io + self.state = "open" + self.locator_key = "" + self.safe_cbs = {} + self.complete_cbs = {} + RADOS_CB = CFUNCTYPE(c_int, c_void_p, c_void_p) + self.__aio_safe_cb_c = RADOS_CB(self.__aio_safe_cb) + self.__aio_complete_cb_c = RADOS_CB(self.__aio_complete_cb) + self.lock = threading.Lock() + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + self.close() + return False + + def __del__(self): + self.close() + + def __aio_safe_cb(self, completion, _): + """ + Callback to onsafe() for asynchronous operations + """ + cb = None + with self.lock: + cb = self.safe_cbs[completion] + del self.safe_cbs[completion] + cb.onsafe(cb) + return 0 + + def __aio_complete_cb(self, completion, _): + """ + Callback to oncomplete() for asynchronous operations + """ + cb = None + with self.lock: + cb = self.complete_cbs[completion] + del self.complete_cbs[completion] + cb.oncomplete(cb) + return 0 + + def __get_completion(self, oncomplete, onsafe): + """ + Constructs a completion to use with asynchronous operations + + :param oncomplete: what to do when the write is safe and complete in memory + on all replicas + :type oncomplete: completion + :param onsafe: what to do when the write is safe and complete on storage + on all replicas + :type onsafe: completion + + :raises: :class:`Error` + :returns: completion object + """ + completion = c_void_p(0) + complete_cb = None + safe_cb = None + if oncomplete: + complete_cb = self.__aio_complete_cb_c + if onsafe: + safe_cb = self.__aio_safe_cb_c + ret = run_in_thread(self.librados.rados_aio_create_completion, + (c_void_p(0), complete_cb, safe_cb, + byref(completion))) + if ret < 0: + raise make_ex(ret, "error getting a completion") + with self.lock: + completion_obj = Completion(self, completion, oncomplete, onsafe) + if oncomplete: + self.complete_cbs[completion.value] = completion_obj + if onsafe: + self.safe_cbs[completion.value] = completion_obj + return completion_obj + + def aio_write(self, object_name, to_write, offset=0, + oncomplete=None, onsafe=None): + """ + Write data to an object asynchronously + + Queues the write and returns. + + :param object_name: name of the object + :type object_name: str + :param to_write: data to write + :type to_write: str + :param offset: byte offset in the object to begin writing at + :type offset: int + :param oncomplete: what to do when the write is safe and complete in memory + on all replicas + :type oncomplete: completion + :param onsafe: what to do when the write is safe and complete on storage + on all replicas + :type onsafe: completion + + :raises: :class:`Error` + :returns: completion object + """ + completion = self.__get_completion(oncomplete, onsafe) + ret = run_in_thread(self.librados.rados_aio_write, + (self.io, c_char_p(object_name), + completion.rados_comp, c_char_p(to_write), + c_size_t(len(to_write)), c_uint64(offset))) + if ret < 0: + raise make_ex(ret, "error writing object %s" % object_name) + return completion + + def aio_write_full(self, object_name, to_write, + oncomplete=None, onsafe=None): + """ + Asychronously write an entire object + + The object is filled with the provided data. If the object exists, + it is atomically truncated and then written. + Queues the write and returns. + + :param object_name: name of the object + :type object_name: str + :param to_write: data to write + :type to_write: str + :param oncomplete: what to do when the write is safe and complete in memory + on all replicas + :type oncomplete: completion + :param onsafe: what to do when the write is safe and complete on storage + on all replicas + :type onsafe: completion + + :raises: :class:`Error` + :returns: completion object + """ + completion = self.__get_completion(oncomplete, onsafe) + ret = run_in_thread(self.librados.rados_aio_write_full, + (self.io, c_char_p(object_name), + completion.rados_comp, c_char_p(to_write), + c_size_t(len(to_write)))) + if ret < 0: + raise make_ex(ret, "error writing object %s" % object_name) + return completion + + def aio_append(self, object_name, to_append, oncomplete=None, onsafe=None): + """ + Asychronously append data to an object + + Queues the write and returns. + + :param object_name: name of the object + :type object_name: str + :param to_append: data to append + :type to_append: str + :param offset: byte offset in the object to begin writing at + :type offset: int + :param oncomplete: what to do when the write is safe and complete in memory + on all replicas + :type oncomplete: completion + :param onsafe: what to do when the write is safe and complete on storage + on all replicas + :type onsafe: completion + + :raises: :class:`Error` + :returns: completion object + """ + completion = self.__get_completion(oncomplete, onsafe) + ret = run_in_thread(self.librados.rados_aio_append, + (self.io, c_char_p(object_name), + completion.rados_comp, c_char_p(to_append), + c_size_t(len(to_append)))) + if ret < 0: + raise make_ex(ret, "error appending to object %s" % object_name) + return completion + + def aio_flush(self): + """ + Block until all pending writes in an io context are safe + + :raises: :class:`Error` + """ + ret = run_in_thread(self.librados.rados_aio_flush, (self.io,)) + if ret < 0: + raise make_ex(ret, "error flushing") + + def aio_read(self, object_name, length, offset, oncomplete): + """ + Asychronously read data from an object + + oncomplete will be called with the returned read value as + well as the completion: + + oncomplete(completion, data_read) + + :param object_name: name of the object to read from + :type object_name: str + :param length: the number of bytes to read + :type length: int + :param offset: byte offset in the object to begin reading from + :type offset: int + :param oncomplete: what to do when the read is complete + :type oncomplete: completion + + :raises: :class:`Error` + :returns: completion object + """ + buf = create_string_buffer(length) + def oncomplete_(completion): + return oncomplete(completion, buf.value) + completion = self.__get_completion(oncomplete_, None) + ret = run_in_thread(self.librados.rados_aio_read, + (self.io, c_char_p(object_name), + completion.rados_comp, buf, c_size_t(length), + c_uint64(offset))) + if ret < 0: + raise make_ex(ret, "error reading %s" % object_name) + return completion + + def require_ioctx_open(self): + """ + Checks if the rados.Ioctx object state is 'open' + + :raises: IoctxStateError + """ + if self.state != "open": + raise IoctxStateError("The pool is %s" % self.state) + + def change_auid(self, auid): + """ + Attempt to change an io context's associated auid "owner." + + Requires that you have write permission on both the current and new + auid. + + :raises: :class:`Error` + """ + self.require_ioctx_open() + ret = run_in_thread(self.librados.rados_ioctx_pool_set_auid, + (self.io, ctypes.c_uint64(auid))) + if ret < 0: + raise make_ex(ret, "error changing auid of '%s' to %d" %\ + (self.name, auid)) + + def set_locator_key(self, loc_key): + """ + Set the key for mapping objects to pgs within an io context. + + The key is used instead of the object name to determine which + placement groups an object is put in. This affects all subsequent + operations of the io context - until a different locator key is + set, all objects in this io context will be placed in the same pg. + + :param loc_key: the key to use as the object locator, or NULL to discard + any previously set key + :type loc_key: str + + :raises: :class:`TypeError` + """ + self.require_ioctx_open() + if not isinstance(loc_key, str): + raise TypeError('loc_key must be a string') + run_in_thread(self.librados.rados_ioctx_locator_set_key, + (self.io, c_char_p(loc_key))) + self.locator_key = loc_key + + def get_locator_key(self): + """ + Get the locator_key of context + + :returns: locator_key + """ + return self.locator_key + + def close(self): + """ + Close a rados.Ioctx object. + + This just tells librados that you no longer need to use the io context. + It may not be freed immediately if there are pending asynchronous + requests on it, but you should not use an io context again after + calling this function on it. + """ + if self.state == "open": + self.require_ioctx_open() + run_in_thread(self.librados.rados_ioctx_destroy, (self.io,)) + self.state = "closed" + + def write(self, key, data, offset=0): + """ + Write data to an object synchronously + + :param key: name of the object + :type key: str + :param data: data to write + :type data: str + :param offset: byte offset in the object to begin writing at + :type offset: int + + :raises: :class:`TypeError` + :raises: :class:`IncompleteWriteError` + :raises: :class:`LogicError` + :returns: int - number of bytes written + """ + self.require_ioctx_open() + if not isinstance(data, str): + raise TypeError('data must be a string') + length = len(data) + ret = run_in_thread(self.librados.rados_write, + (self.io, c_char_p(key), c_char_p(data), + c_size_t(length), c_uint64(offset))) + if ret == length: + return ret + elif ret < 0: + raise make_ex(ret, "Ioctx.write(%s): failed to write %s" % \ + (self.name, key)) + elif ret < length: + raise IncompleteWriteError("Wrote only %d out of %d bytes" % \ + (ret, length)) + else: + raise LogicError("Ioctx.write(%s): rados_write \ +returned %d, but %d was the maximum number of bytes it could have \ +written." % (self.name, ret, length)) + + def write_full(self, key, data): + """ + Write an entire object synchronously. + + The object is filled with the provided data. If the object exists, + it is atomically truncated and then written. + + :param key: name of the object + :type key: str + :param data: data to write + :type data: str + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: int - 0 on success + """ + self.require_ioctx_open() + if not isinstance(key, str): + raise TypeError('key must be a string') + if not isinstance(data, str): + raise TypeError('data must be a string') + length = len(data) + ret = run_in_thread(self.librados.rados_write_full, + (self.io, c_char_p(key), c_char_p(data), + c_size_t(length))) + if ret == 0: + return ret + else: + raise make_ex(ret, "Ioctx.write(%s): failed to write_full %s" % \ + (self.name, key)) + + def read(self, key, length=8192, offset=0): + """ + Write data to an object synchronously + + :param key: name of the object + :type key: str + :param length: the number of bytes to read (default=8192) + :type length: int + :param offset: byte offset in the object to begin reading at + :type offset: int + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: str - data read from object + """ + self.require_ioctx_open() + if not isinstance(key, str): + raise TypeError('key must be a string') + ret_buf = create_string_buffer(length) + ret = run_in_thread(self.librados.rados_read, + (self.io, c_char_p(key), ret_buf, c_size_t(length), + c_uint64(offset))) + if ret < 0: + raise make_ex(ret, "Ioctx.read(%s): failed to read %s" % (self.name, key)) + return ctypes.string_at(ret_buf, ret) + + def get_stats(self): + """ + Get pool usage statistics + + :returns: dict - contains the following keys: + + *``num_bytes`` (int) - size of pool in bytes + + *``num_kb`` (int) - size of pool in kbytes + + *``num_objects`` (int) - number of objects in the pool + + *``num_object_clones`` (int) - number of object clones + + *``num_object_copies`` (int) - number of object copies + + *``num_objects_missing_on_primary`` (int) - number of objets + missing on primary + + *``num_objects_unfound`` (int) - number of unfound objects + + *``num_objects_degraded`` (int) - number of degraded objects + + *``num_rd`` (int) - bytes read + + *``num_rd_kb`` (int) - kbytes read + + *``num_wr`` (int) - bytes written + + *``num_wr_kb`` (int) - kbytes written + """ + self.require_ioctx_open() + stats = rados_pool_stat_t() + ret = run_in_thread(self.librados.rados_ioctx_pool_stat, + (self.io, byref(stats))) + if ret < 0: + raise make_ex(ret, "Ioctx.get_stats(%s): get_stats failed" % self.name) + return {'num_bytes': stats.num_bytes, + 'num_kb': stats.num_kb, + 'num_objects': stats.num_objects, + 'num_object_clones': stats.num_object_clones, + 'num_object_copies': stats.num_object_copies, + "num_objects_missing_on_primary": stats.num_objects_missing_on_primary, + "num_objects_unfound": stats.num_objects_unfound, + "num_objects_degraded": stats.num_objects_degraded, + "num_rd": stats.num_rd, + "num_rd_kb": stats.num_rd_kb, + "num_wr": stats.num_wr, + "num_wr_kb": stats.num_wr_kb } + + def remove_object(self, key): + """ + Delete an object + + This does not delete any snapshots of the object. + + :param key: the name of the object to delete + :type key: str + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: bool - True on success + """ + self.require_ioctx_open() + if not isinstance(key, str): + raise TypeError('key must be a string') + ret = run_in_thread(self.librados.rados_remove, + (self.io, c_char_p(key))) + if ret < 0: + raise make_ex(ret, "Failed to remove '%s'" % key) + return True + + def trunc(self, key, size): + """ + Resize an object + + If this enlarges the object, the new area is logically filled with + zeroes. If this shrinks the object, the excess data is removed. + + :param key: the name of the object to resize + :type key: str + :param size: the new size of the object in bytes + :type size: int + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: int - 0 on success, otherwise raises error + """ + + self.require_ioctx_open() + if not isinstance(key, str): + raise TypeError('key must be a string') + ret = run_in_thread(self.librados.rados_trunc, + (self.io, c_char_p(key), c_uint64(size))) + if ret < 0: + raise make_ex(ret, "Ioctx.trunc(%s): failed to truncate %s" % (self.name, key)) + return ret + + def stat(self, key): + """ + Get object stats (size/mtime) + + :param key: the name of the object to get stats from + :type key: str + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: (size,timestamp) + """ + self.require_ioctx_open() + if not isinstance(key, str): + raise TypeError('key must be a string') + psize = c_uint64() + pmtime = c_uint64() + + ret = run_in_thread(self.librados.rados_stat, + (self.io, c_char_p(key), pointer(psize), + pointer(pmtime))) + if ret < 0: + raise make_ex(ret, "Failed to stat %r" % key) + return psize.value, time.localtime(pmtime.value) + + def get_xattr(self, key, xattr_name): + """ + Get the value of an extended attribute on an object. + + :param key: the name of the object to get xattr from + :type key: str + :param xattr_name: which extended attribute to read + :type xattr_name: str + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: str - value of the xattr + """ + self.require_ioctx_open() + if not isinstance(xattr_name, str): + raise TypeError('xattr_name must be a string') + ret_length = 4096 + while ret_length < 4096 * 1024 * 1024: + ret_buf = create_string_buffer(ret_length) + ret = run_in_thread(self.librados.rados_getxattr, + (self.io, c_char_p(key), c_char_p(xattr_name), + ret_buf, c_size_t(ret_length))) + if (ret == -errno.ERANGE): + ret_length *= 2 + elif ret < 0: + raise make_ex(ret, "Failed to get xattr %r" % xattr_name) + else: + break + return ctypes.string_at(ret_buf, ret) + + def get_xattrs(self, oid): + """ + Start iterating over xattrs on an object. + + :param oid: the name of the object to get xattrs from + :type key: str + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: XattrIterator + """ + self.require_ioctx_open() + if not isinstance(oid, str): + raise TypeError('oid must be a string') + it = c_void_p(0) + ret = run_in_thread(self.librados.rados_getxattrs, + (self.io, oid, byref(it))) + if ret != 0: + raise make_ex(ret, "Failed to get rados xattrs for object %r" % oid) + return XattrIterator(self, it, oid) + + def set_xattr(self, key, xattr_name, xattr_value): + """ + Set an extended attribute on an object. + + :param key: the name of the object to set xattr to + :type key: str + :param xattr_name: which extended attribute to set + :type xattr_name: str + :param xattr_value: the value of the extended attribute + :type xattr_value: str + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: bool - True on success, otherwise raise an error + """ + self.require_ioctx_open() + if not isinstance(key, str): + raise TypeError('key must be a string') + if not isinstance(xattr_name, str): + raise TypeError('xattr_name must be a string') + if not isinstance(xattr_value, str): + raise TypeError('xattr_value must be a string') + ret = run_in_thread(self.librados.rados_setxattr, + (self.io, c_char_p(key), c_char_p(xattr_name), + c_char_p(xattr_value), c_size_t(len(xattr_value)))) + if ret < 0: + raise make_ex(ret, "Failed to set xattr %r" % xattr_name) + return True + + def rm_xattr(self, key, xattr_name): + """ + Removes an extended attribute on from an object. + + :param key: the name of the object to remove xattr from + :type key: str + :param xattr_name: which extended attribute to remove + :type xattr_name: str + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: bool - True on success, otherwise raise an error + """ + self.require_ioctx_open() + if not isinstance(key, str): + raise TypeError('key must be a string') + if not isinstance(xattr_name, str): + raise TypeError('xattr_name must be a string') + ret = run_in_thread(self.librados.rados_rmxattr, + (self.io, c_char_p(key), c_char_p(xattr_name))) + if ret < 0: + raise make_ex(ret, "Failed to delete key %r xattr %r" % + (key, xattr_name)) + return True + + def list_objects(self): + """ + Get ObjectIterator on rados.Ioctx object. + + :returns: ObjectIterator + """ + self.require_ioctx_open() + return ObjectIterator(self) + + def list_snaps(self): + """ + Get SnapIterator on rados.Ioctx object. + + :returns: SnapIterator + """ + self.require_ioctx_open() + return SnapIterator(self) + + def create_snap(self, snap_name): + """ + Create a pool-wide snapshot + + :param snap_name: the name of the snapshot + :type snap_name: str + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + if not isinstance(snap_name, str): + raise TypeError('snap_name must be a string') + ret = run_in_thread(self.librados.rados_ioctx_snap_create, + (self.io, c_char_p(snap_name))) + if (ret != 0): + raise make_ex(ret, "Failed to create snap %s" % snap_name) + + def remove_snap(self, snap_name): + """ + Removes a pool-wide snapshot + + :param snap_name: the name of the snapshot + :type snap_name: str + + :raises: :class:`TypeError` + :raises: :class:`Error` + """ + self.require_ioctx_open() + if not isinstance(snap_name, str): + raise TypeError('snap_name must be a string') + ret = run_in_thread(self.librados.rados_ioctx_snap_remove, + (self.io, c_char_p(snap_name))) + if (ret != 0): + raise make_ex(ret, "Failed to remove snap %s" % snap_name) + + def lookup_snap(self, snap_name): + """ + Get the id of a pool snapshot + + :param snap_name: the name of the snapshot to lookop + :type snap_name: str + + :raises: :class:`TypeError` + :raises: :class:`Error` + :returns: Snap - on success + """ + self.require_ioctx_open() + if not isinstance(snap_name, str): + raise TypeError('snap_name must be a string') + snap_id = c_uint64() + ret = run_in_thread(self.librados.rados_ioctx_snap_lookup, + (self.io, c_char_p(snap_name), byref(snap_id))) + if (ret != 0): + raise make_ex(ret, "Failed to lookup snap %s" % snap_name) + return Snap(self, snap_name, snap_id) + + def get_last_version(self): + """ + Return the version of the last object read or written to. + + This exposes the internal version number of the last object read or + written via this io context + + :returns: version of the last object used + """ + self.require_ioctx_open() + return run_in_thread(self.librados.rados_get_last_version, (self.io,)) + +def set_object_locator(func): + def retfunc(self, *args, **kwargs): + if self.locator_key is not None: + old_locator = self.ioctx.get_locator_key() + self.ioctx.set_locator_key(self.locator_key) + retval = func(self, *args, **kwargs) + self.ioctx.set_locator_key(old_locator) + return retval + else: + return func(self, *args, **kwargs) + return retfunc + +class Object(object): + """Rados object wrapper, makes the object look like a file""" + def __init__(self, ioctx, key, locator_key=None): + self.key = key + self.ioctx = ioctx + self.offset = 0 + self.state = "exists" + self.locator_key = locator_key + + def __str__(self): + return "rados.Object(ioctx=%s,key=%s)" % (str(self.ioctx), self.key) + + def require_object_exists(self): + if self.state != "exists": + raise ObjectStateError("The object is %s" % self.state) + + @set_object_locator + def read(self, length = 1024*1024): + self.require_object_exists() + ret = self.ioctx.read(self.key, length, self.offset) + self.offset += len(ret) + return ret + + @set_object_locator + def write(self, string_to_write): + self.require_object_exists() + ret = self.ioctx.write(self.key, string_to_write, self.offset) + self.offset += ret + return ret + + @set_object_locator + def remove(self): + self.require_object_exists() + self.ioctx.remove_object(self.key) + self.state = "removed" + + @set_object_locator + def stat(self): + self.require_object_exists() + return self.ioctx.stat(self.key) + + def seek(self, position): + self.require_object_exists() + self.offset = position + + @set_object_locator + def get_xattr(self, xattr_name): + self.require_object_exists() + return self.ioctx.get_xattr(self.key, xattr_name) + + @set_object_locator + def get_xattrs(self): + self.require_object_exists() + return self.ioctx.get_xattrs(self.key) + + @set_object_locator + def set_xattr(self, xattr_name, xattr_value): + self.require_object_exists() + return self.ioctx.set_xattr(self.key, xattr_name, xattr_value) + + @set_object_locator + def rm_xattr(self, xattr_name): + self.require_object_exists() + return self.ioctx.rm_xattr(self.key, xattr_name) + +MONITOR_LEVELS = [ + "debug", + "info", + "warn", "warning", + "err", "error", + "sec", + ] + + +class MonitorLog(object): + """ + For watching cluster log messages. Instantiate an object and keep + it around while callback is periodically called. Construct with + 'level' to monitor 'level' messages (one of MONITOR_LEVELS). + arg will be passed to the callback. + + callback will be called with: + arg (given to __init__) + line (the full line, including timestamp, who, level, msg) + who (which entity issued the log message) + timestamp_sec (sec of a struct timespec) + timestamp_nsec (sec of a struct timespec) + seq (sequence number) + level (string representing the level of the log message) + msg (the message itself) + callback's return value is ignored + """ + + def monitor_log_callback(self, arg, line, who, sec, nsec, seq, level, msg): + """ + Local callback wrapper, in case we decide to do something + """ + self.callback(arg, line, who, sec, nsec, seq, level, msg) + return 0 + + def __init__(self, cluster, level, callback, arg): + if level not in MONITOR_LEVELS: + raise LogicError("invalid monitor level " + level) + if not callable(callback): + raise LogicError("callback must be a callable function") + self.level = level + self.callback = callback + self.arg = arg + callback_factory = CFUNCTYPE(c_int, # return type (really void) + c_void_p, # arg + c_char_p, # line + c_char_p, # who + c_uint64, # timestamp_sec + c_uint64, # timestamp_nsec + c_ulong, # seq + c_char_p, # level + c_char_p) # msg + self.internal_callback = callback_factory(self.monitor_log_callback) + + r = run_in_thread(cluster.librados.rados_monitor_log, + (cluster.cluster, level, self.internal_callback, arg)) + if r: + raise make_ex(r, 'error calling rados_monitor_log') diff --git a/src/pybind/rados/setup.py b/src/pybind/rados/setup.py new file mode 100644 index 00000000000..a437f06bac2 --- /dev/null +++ b/src/pybind/rados/setup.py @@ -0,0 +1,32 @@ +from setuptools import setup, find_packages +import os + + +def long_description(): + readme = os.path.join(os.path.dirname(__file__), 'README.rst') + return open(readme).read() + + +setup( + name = 'rados', + description = 'Bindings for rados [ceph]', + packages=find_packages(), + author = 'Inktank', + author_email = 'ceph-devel@vger.kernel.org', + version = '0.67', + license = "LGPL2", + zip_safe = False, + keywords = "ceph, rados, bindings, api, cli", + long_description = long_description(), + classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)', + 'Topic :: Software Development :: Libraries', + 'Topic :: Utilities', + 'Topic :: System :: Filesystems', + 'Operating System :: POSIX', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], +) diff --git a/src/pybind/rados/tox.ini b/src/pybind/rados/tox.ini new file mode 100644 index 00000000000..9c63bb9d884 --- /dev/null +++ b/src/pybind/rados/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py26, py27 + +[testenv] +deps= +commands= -- cgit v1.2.1 From 333447a24e171838e920a6839d9f557536b48664 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 16 Aug 2013 13:37:04 -0400 Subject: create the rbd package Signed-off-by: Alfredo Deza --- src/pybind/rbd.py | 934 --------------------------------------------- src/pybind/rbd/MANIFEST.in | 2 + src/pybind/rbd/README.rst | 0 src/pybind/rbd/rbd.py | 934 +++++++++++++++++++++++++++++++++++++++++++++ src/pybind/rbd/setup.py | 32 ++ src/pybind/rbd/tox.ini | 6 + 6 files changed, 974 insertions(+), 934 deletions(-) delete mode 100644 src/pybind/rbd.py create mode 100644 src/pybind/rbd/MANIFEST.in create mode 100644 src/pybind/rbd/README.rst create mode 100644 src/pybind/rbd/rbd.py create mode 100644 src/pybind/rbd/setup.py create mode 100644 src/pybind/rbd/tox.ini diff --git a/src/pybind/rbd.py b/src/pybind/rbd.py deleted file mode 100644 index 6e9ca8a2252..00000000000 --- a/src/pybind/rbd.py +++ /dev/null @@ -1,934 +0,0 @@ -""" -This module is a thin wrapper around librbd. - -It currently provides all the synchronous methods of librbd that do -not use callbacks. - -Error codes from librbd are turned into exceptions that subclass -:class:`Error`. Almost all methods may raise :class:`Error` -(the base class of all rbd exceptions), :class:`PermissionError` -and :class:`IOError`, in addition to those documented for the -method. - -A number of methods have string arguments, which must not be unicode -to interact correctly with librbd. If unicode is passed to these -methods, a :class:`TypeError` will be raised. -""" -# Copyright 2011 Josh Durgin -from ctypes import CDLL, c_char, c_char_p, c_size_t, c_void_p, c_int, \ - create_string_buffer, byref, Structure, c_uint64, c_int64, c_uint8, \ - CFUNCTYPE -import ctypes -import errno - -ANONYMOUS_AUID = 0xffffffffffffffff -ADMIN_AUID = 0 - -RBD_FEATURE_LAYERING = 1 -RBD_FEATURE_STRIPINGV2 = 2 - -class Error(Exception): - pass - -class PermissionError(Error): - pass - -class ImageNotFound(Error): - pass - -class ImageExists(Error): - pass - -class IOError(Error): - pass - -class NoSpace(Error): - pass - -class IncompleteWriteError(Error): - pass - -class InvalidArgument(Error): - pass - -class LogicError(Error): - pass - -class ReadOnlyImage(Error): - pass - -class ImageBusy(Error): - pass - -class ImageHasSnapshots(Error): - pass - -class FunctionNotSupported(Error): - pass - -class ArgumentOutOfRange(Error): - pass - -class ConnectionShutdown(Error): - pass - -def make_ex(ret, msg): - """ - Translate a librbd return code into an exception. - - :param ret: the return code - :type ret: int - :param msg: the error message to use - :type msg: str - :returns: a subclass of :class:`Error` - """ - errors = { - errno.EPERM : PermissionError, - errno.ENOENT : ImageNotFound, - errno.EIO : IOError, - errno.ENOSPC : NoSpace, - errno.EEXIST : ImageExists, - errno.EINVAL : InvalidArgument, - errno.EROFS : ReadOnlyImage, - errno.EBUSY : ImageBusy, - errno.ENOTEMPTY : ImageHasSnapshots, - errno.ENOSYS : FunctionNotSupported, - errno.EDOM : ArgumentOutOfRange, - errno.ESHUTDOWN : ConnectionShutdown - } - ret = abs(ret) - if ret in errors: - return errors[ret](msg) - else: - return Error(msg + (": error code %d" % ret)) - -class rbd_image_info_t(Structure): - _fields_ = [("size", c_uint64), - ("obj_size", c_uint64), - ("num_objs", c_uint64), - ("order", c_int), - ("block_name_prefix", c_char * 24), - ("parent_pool", c_int64), - ("parent_name", c_char * 96)] - -class rbd_snap_info_t(Structure): - _fields_ = [("id", c_uint64), - ("size", c_uint64), - ("name", c_char_p)] - -class RBD(object): - """ - This class wraps librbd CRUD functions. - """ - def __init__(self): - self.librbd = CDLL('librbd.so.1') - - def version(self): - """ - Get the version number of the ``librbd`` C library. - - :returns: a tuple of ``(major, minor, extra)`` components of the - librbd version - """ - major = c_int(0) - minor = c_int(0) - extra = c_int(0) - self.librbd.rbd_version(byref(major), byref(minor), byref(extra)) - return (major.value, minor.value, extra.value) - - def create(self, ioctx, name, size, order=None, old_format=True, - features=0, stripe_unit=0, stripe_count=0): - """ - Create an rbd image. - - :param ioctx: the context in which to create the image - :type ioctx: :class:`rados.Ioctx` - :param name: what the image is called - :type name: str - :param size: how big the image is in bytes - :type size: int - :param order: the image is split into (2**order) byte objects - :type order: int - :param old_format: whether to create an old-style image that - is accessible by old clients, but can't - use more advanced features like layering. - :type old_format: bool - :param features: bitmask of features to enable - :type features: int - :param stripe_unit: stripe unit in bytes (default 0 for object size) - :type stripe_unit: int - :param stripe_count: objects to stripe over before looping - :type stripe_count: int - :raises: :class:`ImageExists` - :raises: :class:`TypeError` - :raises: :class:`InvalidArgument` - :raises: :class:`FunctionNotSupported` - """ - if order is None: - order = 0 - if not isinstance(name, str): - raise TypeError('name must be a string') - if old_format: - if features != 0 or stripe_unit != 0 or stripe_count != 0: - raise InvalidArgument('format 1 images do not support feature' - ' masks or non-default striping') - ret = self.librbd.rbd_create(ioctx.io, c_char_p(name), - c_uint64(size), - byref(c_int(order))) - else: - if not hasattr(self.librbd, 'rbd_create2'): - raise FunctionNotSupported('installed version of librbd does' - ' not support format 2 images') - has_create3 = hasattr(self.librbd, 'rbd_create3') - if (stripe_unit != 0 or stripe_count != 0) and not has_create3: - raise FunctionNotSupported('installed version of librbd does' - ' not support stripe unit or count') - if has_create3: - ret = self.librbd.rbd_create3(ioctx.io, c_char_p(name), - c_uint64(size), - c_uint64(features), - byref(c_int(order)), - c_uint64(stripe_unit), - c_uint64(stripe_count)) - else: - ret = self.librbd.rbd_create2(ioctx.io, c_char_p(name), - c_uint64(size), - c_uint64(features), - byref(c_int(order))) - if ret < 0: - raise make_ex(ret, 'error creating image') - - def clone(self, p_ioctx, p_name, p_snapname, c_ioctx, c_name, - features=0, order=None): - """ - Clone a parent rbd snapshot into a COW sparse child. - - :param p_ioctx: the parent context that represents the parent snap - :type ioctx: :class:`rados.Ioctx` - :param p_name: the parent image name - :type name: str - :param p_snapname: the parent image snapshot name - :type name: str - :param c_ioctx: the child context that represents the new clone - :type ioctx: :class:`rados.Ioctx` - :param c_name: the clone (child) name - :type name: str - :param features: bitmask of features to enable; if set, must include layering - :type features: int - :param order: the image is split into (2**order) byte objects - :type order: int - :raises: :class:`TypeError` - :raises: :class:`InvalidArgument` - :raises: :class:`ImageExists` - :raises: :class:`FunctionNotSupported` - :raises: :class:`ArgumentOutOfRange` - """ - if order is None: - order = 0 - if not isinstance(p_snapname, str) or not isinstance(p_name, str): - raise TypeError('parent name and snapname must be strings') - if not isinstance(c_name, str): - raise TypeError('child name must be a string') - - ret = self.librbd.rbd_clone(p_ioctx.io, c_char_p(p_name), - c_char_p(p_snapname), - c_ioctx.io, c_char_p(c_name), - c_uint64(features), - byref(c_int(order))) - if ret < 0: - raise make_ex(ret, 'error creating clone') - - def list(self, ioctx): - """ - List image names. - - :param ioctx: determines which RADOS pool is read - :type ioctx: :class:`rados.Ioctx` - :returns: list -- a list of image names - """ - size = c_size_t(512) - while True: - c_names = create_string_buffer(size.value) - ret = self.librbd.rbd_list(ioctx.io, byref(c_names), byref(size)) - if ret >= 0: - break - elif ret != -errno.ERANGE: - raise make_ex(ret, 'error listing images') - return filter(lambda name: name != '', c_names.raw.split('\0')) - - def remove(self, ioctx, name): - """ - Delete an RBD image. This may take a long time, since it does - not return until every object that comprises the image has - been deleted. Note that all snapshots must be deleted before - the image can be removed. If there are snapshots left, - :class:`ImageHasSnapshots` is raised. If the image is still - open, or the watch from a crashed client has not expired, - :class:`ImageBusy` is raised. - - :param ioctx: determines which RADOS pool the image is in - :type ioctx: :class:`rados.Ioctx` - :param name: the name of the image to remove - :type name: str - :raises: :class:`ImageNotFound`, :class:`ImageBusy`, - :class:`ImageHasSnapshots` - """ - if not isinstance(name, str): - raise TypeError('name must be a string') - ret = self.librbd.rbd_remove(ioctx.io, c_char_p(name)) - if ret != 0: - raise make_ex(ret, 'error removing image') - - def rename(self, ioctx, src, dest): - """ - Rename an RBD image. - - :param ioctx: determines which RADOS pool the image is in - :type ioctx: :class:`rados.Ioctx` - :param src: the current name of the image - :type src: str - :param dest: the new name of the image - :type dest: str - :raises: :class:`ImageNotFound`, :class:`ImageExists` - """ - if not isinstance(src, str) or not isinstance(dest, str): - raise TypeError('src and dest must be strings') - ret = self.librbd.rbd_rename(ioctx.io, c_char_p(src), c_char_p(dest)) - if ret != 0: - raise make_ex(ret, 'error renaming image') - -class Image(object): - """ - This class represents an RBD image. It is used to perform I/O on - the image and interact with snapshots. - - **Note**: Any method of this class may raise :class:`ImageNotFound` - if the image has been deleted. - """ - - def __init__(self, ioctx, name, snapshot=None, read_only=False): - """ - Open the image at the given snapshot. - If a snapshot is specified, the image will be read-only, unless - :func:`Image.set_snap` is called later. - - If read-only mode is used, metadata for the :class:`Image` - object (such as which snapshots exist) may become obsolete. See - the C api for more details. - - To clean up from opening the image, :func:`Image.close` should - be called. For ease of use, this is done automatically when - an :class:`Image` is used as a context manager (see :pep:`343`). - - :param ioctx: determines which RADOS pool the image is in - :type ioctx: :class:`rados.Ioctx` - :param name: the name of the image - :type name: str - :param snapshot: which snapshot to read from - :type snaphshot: str - :param read_only: whether to open the image in read-only mode - :type read_only: bool - """ - self.closed = True - self.librbd = CDLL('librbd.so.1') - self.image = c_void_p() - self.name = name - if not isinstance(name, str): - raise TypeError('name must be a string') - if snapshot is not None and not isinstance(snapshot, str): - raise TypeError('snapshot must be a string or None') - if read_only: - if not hasattr(self.librbd, 'rbd_open_read_only'): - raise FunctionNotSupported('installed version of librbd does ' - 'not support open in read-only mode') - ret = self.librbd.rbd_open_read_only(ioctx.io, c_char_p(name), - byref(self.image), - c_char_p(snapshot)) - else: - ret = self.librbd.rbd_open(ioctx.io, c_char_p(name), - byref(self.image), c_char_p(snapshot)) - if ret != 0: - raise make_ex(ret, 'error opening image %s at snapshot %s' % (name, snapshot)) - self.closed = False - - def __enter__(self): - return self - - def __exit__(self, type_, value, traceback): - """ - Closes the image. See :func:`close` - """ - self.close() - return False - - def close(self): - """ - Release the resources used by this image object. - - After this is called, this object should not be used. - """ - if not self.closed: - self.closed = True - self.librbd.rbd_close(self.image) - - def __del__(self): - self.close() - - def __str__(self): - s = "rbd.Image(" + dict.__repr__(self.__dict__) + ")" - return s - - def resize(self, size): - """ - Change the size of the image. - - :param size: the new size of the image - :type size: int - """ - ret = self.librbd.rbd_resize(self.image, c_uint64(size)) - if ret < 0: - raise make_ex(ret, 'error resizing image %s' % (self.name,)) - - def stat(self): - """ - Get information about the image. Currently parent pool and - parent name are always -1 and ''. - - :returns: dict - contains the following keys: - - * ``size`` (int) - the size of the image in bytes - - * ``obj_size`` (int) - the size of each object that comprises the - image - - * ``num_objs`` (int) - the number of objects in the image - - * ``order`` (int) - log_2(object_size) - - * ``block_name_prefix`` (str) - the prefix of the RADOS objects used - to store the image - - * ``parent_pool`` (int) - deprecated - - * ``parent_name`` (str) - deprecated - - See also :meth:`format` and :meth:`features`. - - """ - info = rbd_image_info_t() - ret = self.librbd.rbd_stat(self.image, byref(info), ctypes.sizeof(info)) - if ret != 0: - raise make_ex(ret, 'error getting info for image %s' % (self.name,)) - return { - 'size' : info.size, - 'obj_size' : info.obj_size, - 'num_objs' : info.num_objs, - 'order' : info.order, - 'block_name_prefix' : info.block_name_prefix, - 'parent_pool' : info.parent_pool, - 'parent_name' : info.parent_name - } - - def parent_info(self): - ret = -errno.ERANGE - size = 8 - while ret == -errno.ERANGE and size < 128: - pool = create_string_buffer(size) - name = create_string_buffer(size) - snapname = create_string_buffer(size) - ret = self.librbd.rbd_get_parent_info(self.image, pool, len(pool), - name, len(name), snapname, len(snapname)) - if ret == -errno.ERANGE: - size *= 2 - - if (ret != 0): - raise make_ex(ret, 'error getting parent info for image %s' % (self.name,)) - return (pool.value, name.value, snapname.value) - - def old_format(self): - old = c_uint8() - ret = self.librbd.rbd_get_old_format(self.image, byref(old)) - if (ret != 0): - raise make_ex(ret, 'error getting old_format for image' % (self.name)) - return old.value != 0 - - def size(self): - """ - Get the size of the image. If open to a snapshot, returns the - size of that snapshot. - - :returns: the size of the image in bytes - """ - image_size = c_uint64() - ret = self.librbd.rbd_get_size(self.image, byref(image_size)) - if (ret != 0): - raise make_ex(ret, 'error getting size for image' % (self.name)) - return image_size.value - - def features(self): - features = c_uint64() - ret = self.librbd.rbd_get_features(self.image, byref(features)) - if (ret != 0): - raise make_ex(ret, 'error getting features for image' % (self.name)) - return features.value - - def overlap(self): - overlap = c_uint64() - ret = self.librbd.rbd_get_overlap(self.image, byref(overlap)) - if (ret != 0): - raise make_ex(ret, 'error getting overlap for image' % (self.name)) - return overlap.value - - def copy(self, dest_ioctx, dest_name): - """ - Copy the image to another location. - - :param dest_ioctx: determines which pool to copy into - :type dest_ioctx: :class:`rados.Ioctx` - :param dest_name: the name of the copy - :type dest_name: str - :raises: :class:`ImageExists` - """ - if not isinstance(dest_name, str): - raise TypeError('dest_name must be a string') - ret = self.librbd.rbd_copy(self.image, dest_ioctx.io, c_char_p(dest_name)) - if ret < 0: - raise make_ex(ret, 'error copying image %s to %s' % (self.name, dest_name)) - - def list_snaps(self): - """ - Iterate over the snapshots of an image. - - :returns: :class:`SnapIterator` - """ - return SnapIterator(self) - - def create_snap(self, name): - """ - Create a snapshot of the image. - - :param name: the name of the snapshot - :type name: str - :raises: :class:`ImageExists` - """ - if not isinstance(name, str): - raise TypeError('name must be a string') - ret = self.librbd.rbd_snap_create(self.image, c_char_p(name)) - if ret != 0: - raise make_ex(ret, 'error creating snapshot %s from %s' % (name, self.name)) - - def remove_snap(self, name): - """ - Delete a snapshot of the image. - - :param name: the name of the snapshot - :type name: str - :raises: :class:`IOError`, :class:`ImageBusy` - """ - if not isinstance(name, str): - raise TypeError('name must be a string') - ret = self.librbd.rbd_snap_remove(self.image, c_char_p(name)) - if ret != 0: - raise make_ex(ret, 'error removing snapshot %s from %s' % (name, self.name)) - - def rollback_to_snap(self, name): - """ - Revert the image to its contents at a snapshot. This is a - potentially expensive operation, since it rolls back each - object individually. - - :param name: the snapshot to rollback to - :type name: str - :raises: :class:`IOError` - """ - if not isinstance(name, str): - raise TypeError('name must be a string') - ret = self.librbd.rbd_snap_rollback(self.image, c_char_p(name)) - if ret != 0: - raise make_ex(ret, 'error rolling back image %s to snapshot %s' % (self.name, name)) - - def protect_snap(self, name): - """ - Mark a snapshot as protected. This means it can't be deleted - until it is unprotected. - - :param name: the snapshot to protect - :type name: str - :raises: :class:`IOError`, :class:`ImageNotFound` - """ - if not isinstance(name, str): - raise TypeError('name must be a string') - ret = self.librbd.rbd_snap_protect(self.image, c_char_p(name)) - if ret != 0: - raise make_ex(ret, 'error protecting snapshot %s@%s' % (self.name, name)) - - def unprotect_snap(self, name): - """ - Mark a snapshot unprotected. This allows it to be deleted if - it was protected. - - :param name: the snapshot to unprotect - :type name: str - :raises: :class:`IOError`, :class:`ImageNotFound` - """ - if not isinstance(name, str): - raise TypeError('name must be a string') - ret = self.librbd.rbd_snap_unprotect(self.image, c_char_p(name)) - if ret != 0: - raise make_ex(ret, 'error unprotecting snapshot %s@%s' % (self.name, name)) - - def is_protected_snap(self, name): - """ - Find out whether a snapshot is protected from deletion. - - :param name: the snapshot to check - :type name: str - :returns: bool - whether the snapshot is protected - :raises: :class:`IOError`, :class:`ImageNotFound` - """ - if not isinstance(name, str): - raise TypeError('name must be a string') - is_protected = c_int() - ret = self.librbd.rbd_snap_is_protected(self.image, c_char_p(name), - byref(is_protected)) - if ret != 0: - raise make_ex(ret, 'error checking if snapshot %s@%s is protected' % (self.name, name)) - return is_protected.value == 1 - - def set_snap(self, name): - """ - Set the snapshot to read from. Writes will raise ReadOnlyImage - while a snapshot is set. Pass None to unset the snapshot - (reads come from the current image) , and allow writing again. - - :param name: the snapshot to read from, or None to unset the snapshot - :type name: str or None - """ - if name is not None and not isinstance(name, str): - raise TypeError('name must be a string') - ret = self.librbd.rbd_snap_set(self.image, c_char_p(name)) - if ret != 0: - raise make_ex(ret, 'error setting image %s to snapshot %s' % (self.name, name)) - - def read(self, offset, length): - """ - Read data from the image. Raises :class:`InvalidArgument` if - part of the range specified is outside the image. - - :param offset: the offset to start reading at - :type offset: int - :param length: how many bytes to read - :type length: int - :returns: str - the data read - :raises: :class:`InvalidArgument`, :class:`IOError` - """ - ret_buf = create_string_buffer(length) - ret = self.librbd.rbd_read(self.image, c_uint64(offset), - c_size_t(length), byref(ret_buf)) - if ret < 0: - raise make_ex(ret, 'error reading %s %ld~%ld' % (self.image, offset, length)) - return ctypes.string_at(ret_buf, ret) - - def diff_iterate(self, offset, length, from_snapshot, iterate_cb): - """ - Iterate over the changed extents of an image. - - This will call iterate_cb with three arguments: - - (offset, length, exists) - - where the changed extent starts at offset bytes, continues for - length bytes, and is full of data (if exists is True) or zeroes - (if exists is False). - - If from_snapshot is None, it is interpreted as the beginning - of time and this generates all allocated extents. - - The end version is whatever is currently selected (via set_snap) - for the image. - - Raises :class:`InvalidArgument` if from_snapshot is after - the currently set snapshot. - - Raises :class:`ImageNotFound` if from_snapshot is not the name - of a snapshot of the image. - - :param offset: start offset in bytes - :type offset: int - :param length: size of region to report on, in bytes - :type length: int - :param from_snapshot: starting snapshot name, or None - :type from_snapshot: str or None - :param iterate_cb: function to call for each extent - :type iterate_cb: function acception arguments for offset, - length, and exists - :raises: :class:`InvalidArgument`, :class:`IOError`, - :class:`ImageNotFound` - """ - if from_snapshot is not None and not isinstance(from_snapshot, str): - raise TypeError('client must be a string') - - RBD_DIFF_CB = CFUNCTYPE(c_int, c_uint64, c_size_t, c_int, c_void_p) - cb_holder = DiffIterateCB(iterate_cb) - cb = RBD_DIFF_CB(cb_holder.callback) - ret = self.librbd.rbd_diff_iterate(self.image, - c_char_p(from_snapshot), - c_uint64(offset), - c_uint64(length), - cb, - c_void_p(None)) - if ret < 0: - msg = 'error generating diff from snapshot %s' % from_snapshot - raise make_ex(ret, msg) - - def write(self, data, offset): - """ - Write data to the image. Raises :class:`InvalidArgument` if - part of the write would fall outside the image. - - :param data: the data to be written - :type data: str - :param offset: where to start writing data - :type offset: int - :returns: int - the number of bytes written - :raises: :class:`IncompleteWriteError`, :class:`LogicError`, - :class:`InvalidArgument`, :class:`IOError` - """ - if not isinstance(data, str): - raise TypeError('data must be a string') - length = len(data) - ret = self.librbd.rbd_write(self.image, c_uint64(offset), - c_size_t(length), c_char_p(data)) - if ret == length: - return ret - elif ret < 0: - raise make_ex(ret, "error writing to %s" % (self.name,)) - elif ret < length: - raise IncompleteWriteError("Wrote only %ld out of %ld bytes" % (ret, length)) - else: - raise LogicError("logic error: rbd_write(%s) \ -returned %d, but %d was the maximum number of bytes it could have \ -written." % (self.name, ret, length)) - - def discard(self, offset, length): - """ - Trim the range from the image. It will be logically filled - with zeroes. - """ - ret = self.librbd.rbd_discard(self.image, - c_uint64(offset), - c_uint64(length)) - if ret < 0: - msg = 'error discarding region %d~%d' % (offset, length) - raise make_ex(ret, msg) - - def flush(self): - """ - Block until all writes are fully flushed if caching is enabled. - """ - ret = self.librbd.rbd_flush(self.image) - if ret < 0: - raise make_ex(ret, 'error flushing image') - - def stripe_unit(self): - """ - Returns the stripe unit used for the image. - """ - stripe_unit = c_uint64() - ret = self.librbd.rbd_get_stripe_unit(self.image, byref(stripe_unit)) - if ret != 0: - raise make_ex(ret, 'error getting stripe unit for image' % (self.name)) - return stripe_unit.value - - def stripe_count(self): - """ - Returns the stripe count used for the image. - """ - stripe_count = c_uint64() - ret = self.librbd.rbd_get_stripe_count(self.image, byref(stripe_count)) - if ret != 0: - raise make_ex(ret, 'error getting stripe count for image' % (self.name)) - return stripe_count.value - - def flatten(self): - """ - Flatten clone image (copy all blocks from parent to child) - """ - ret = self.librbd.rbd_flatten(self.image) - if (ret < 0): - raise make_ex(ret, "error flattening %s" % self.name) - - def list_children(self): - """ - List children of the currently set snapshot (set via set_snap()). - - :returns: list - a list of (pool name, image name) tuples - """ - pools_size = c_size_t(512) - images_size = c_size_t(512) - while True: - c_pools = create_string_buffer(pools_size.value) - c_images = create_string_buffer(images_size.value) - ret = self.librbd.rbd_list_children(self.image, - byref(c_pools), - byref(pools_size), - byref(c_images), - byref(images_size)) - if ret >= 0: - break - elif ret != -errno.ERANGE: - raise make_ex(ret, 'error listing images') - if ret == 0: - return [] - pools = c_pools.raw[:pools_size.value - 1].split('\0') - images = c_images.raw[:images_size.value - 1].split('\0') - return zip(pools, images) - - def list_lockers(self): - """ - List clients that have locked the image and information - about the lock. - - :returns: dict - contains the following keys: - - * ``tag`` - the tag associated with the lock (every - additional locker must use the same tag) - * ``exclusive`` - boolean indicating whether the - lock is exclusive or shared - * ``lockers`` - a list of (client, cookie, address) - tuples - """ - clients_size = c_size_t(512) - cookies_size = c_size_t(512) - addrs_size = c_size_t(512) - tag_size = c_size_t(512) - exclusive = c_int(0) - - while True: - c_clients = create_string_buffer(clients_size.value) - c_cookies = create_string_buffer(cookies_size.value) - c_addrs = create_string_buffer(addrs_size.value) - c_tag = create_string_buffer(tag_size.value) - ret = self.librbd.rbd_list_lockers(self.image, - byref(exclusive), - byref(c_tag), - byref(tag_size), - byref(c_clients), - byref(clients_size), - byref(c_cookies), - byref(cookies_size), - byref(c_addrs), - byref(addrs_size)) - if ret >= 0: - break - elif ret != -errno.ERANGE: - raise make_ex(ret, 'error listing images') - if ret == 0: - return [] - clients = c_clients.raw[:clients_size.value - 1].split('\0') - cookies = c_cookies.raw[:cookies_size.value - 1].split('\0') - addrs = c_addrs.raw[:addrs_size.value - 1].split('\0') - return { - 'tag' : c_tag.value, - 'exclusive' : exclusive.value == 1, - 'lockers' : zip(clients, cookies, addrs), - } - - def lock_exclusive(self, cookie): - """ - Take an exclusive lock on the image. - - :raises: :class:`ImageBusy` if a different client or cookie locked it - :class:`ImageExists` if the same client and cookie locked it - """ - if not isinstance(cookie, str): - raise TypeError('cookie must be a string') - ret = self.librbd.rbd_lock_exclusive(self.image, c_char_p(cookie)) - if ret < 0: - raise make_ex(ret, 'error acquiring exclusive lock on image') - - def lock_shared(self, cookie, tag): - """ - Take a shared lock on the image. The tag must match - that of the existing lockers, if any. - - :raises: :class:`ImageBusy` if a different client or cookie locked it - :class:`ImageExists` if the same client and cookie locked it - """ - if not isinstance(cookie, str): - raise TypeError('cookie must be a string') - if not isinstance(tag, str): - raise TypeError('tag must be a string') - ret = self.librbd.rbd_lock_shared(self.image, c_char_p(cookie), - c_char_p(tag)) - if ret < 0: - raise make_ex(ret, 'error acquiring shared lock on image') - - def unlock(self, cookie): - """ - Release a lock on the image that was locked by this rados client. - """ - if not isinstance(cookie, str): - raise TypeError('cookie must be a string') - ret = self.librbd.rbd_unlock(self.image, c_char_p(cookie)) - if ret < 0: - raise make_ex(ret, 'error unlocking image') - - def break_lock(self, client, cookie): - """ - Release a lock held by another rados client. - """ - if not isinstance(client, str): - raise TypeError('client must be a string') - if not isinstance(cookie, str): - raise TypeError('cookie must be a string') - ret = self.librbd.rbd_break_lock(self.image, c_char_p(client), - c_char_p(cookie)) - if ret < 0: - raise make_ex(ret, 'error unlocking image') - -class DiffIterateCB(object): - def __init__(self, cb): - self.cb = cb - - def callback(self, offset, length, exists, unused): - self.cb(offset, length, exists == 1) - return 0 - -class SnapIterator(object): - """ - Iterator over snapshot info for an image. - - Yields a dictionary containing information about a snapshot. - - Keys are: - - * ``id`` (int) - numeric identifier of the snapshot - - * ``size`` (int) - size of the image at the time of snapshot (in bytes) - - * ``name`` (str) - name of the snapshot - """ - def __init__(self, image): - self.librbd = image.librbd - num_snaps = c_int(10) - while True: - self.snaps = (rbd_snap_info_t * num_snaps.value)() - ret = self.librbd.rbd_snap_list(image.image, byref(self.snaps), - byref(num_snaps)) - if ret >= 0: - self.num_snaps = ret - break - elif ret != -errno.ERANGE: - raise make_ex(ret, 'error listing snapshots for image %s' % (image.name,)) - - def __iter__(self): - for i in xrange(self.num_snaps): - yield { - 'id' : self.snaps[i].id, - 'size' : self.snaps[i].size, - 'name' : self.snaps[i].name, - } - - def __del__(self): - self.librbd.rbd_snap_list_end(self.snaps) diff --git a/src/pybind/rbd/MANIFEST.in b/src/pybind/rbd/MANIFEST.in new file mode 100644 index 00000000000..3c01b12a5a5 --- /dev/null +++ b/src/pybind/rbd/MANIFEST.in @@ -0,0 +1,2 @@ +include setup.py +include README.rst diff --git a/src/pybind/rbd/README.rst b/src/pybind/rbd/README.rst new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/rbd/rbd.py b/src/pybind/rbd/rbd.py new file mode 100644 index 00000000000..6e9ca8a2252 --- /dev/null +++ b/src/pybind/rbd/rbd.py @@ -0,0 +1,934 @@ +""" +This module is a thin wrapper around librbd. + +It currently provides all the synchronous methods of librbd that do +not use callbacks. + +Error codes from librbd are turned into exceptions that subclass +:class:`Error`. Almost all methods may raise :class:`Error` +(the base class of all rbd exceptions), :class:`PermissionError` +and :class:`IOError`, in addition to those documented for the +method. + +A number of methods have string arguments, which must not be unicode +to interact correctly with librbd. If unicode is passed to these +methods, a :class:`TypeError` will be raised. +""" +# Copyright 2011 Josh Durgin +from ctypes import CDLL, c_char, c_char_p, c_size_t, c_void_p, c_int, \ + create_string_buffer, byref, Structure, c_uint64, c_int64, c_uint8, \ + CFUNCTYPE +import ctypes +import errno + +ANONYMOUS_AUID = 0xffffffffffffffff +ADMIN_AUID = 0 + +RBD_FEATURE_LAYERING = 1 +RBD_FEATURE_STRIPINGV2 = 2 + +class Error(Exception): + pass + +class PermissionError(Error): + pass + +class ImageNotFound(Error): + pass + +class ImageExists(Error): + pass + +class IOError(Error): + pass + +class NoSpace(Error): + pass + +class IncompleteWriteError(Error): + pass + +class InvalidArgument(Error): + pass + +class LogicError(Error): + pass + +class ReadOnlyImage(Error): + pass + +class ImageBusy(Error): + pass + +class ImageHasSnapshots(Error): + pass + +class FunctionNotSupported(Error): + pass + +class ArgumentOutOfRange(Error): + pass + +class ConnectionShutdown(Error): + pass + +def make_ex(ret, msg): + """ + Translate a librbd return code into an exception. + + :param ret: the return code + :type ret: int + :param msg: the error message to use + :type msg: str + :returns: a subclass of :class:`Error` + """ + errors = { + errno.EPERM : PermissionError, + errno.ENOENT : ImageNotFound, + errno.EIO : IOError, + errno.ENOSPC : NoSpace, + errno.EEXIST : ImageExists, + errno.EINVAL : InvalidArgument, + errno.EROFS : ReadOnlyImage, + errno.EBUSY : ImageBusy, + errno.ENOTEMPTY : ImageHasSnapshots, + errno.ENOSYS : FunctionNotSupported, + errno.EDOM : ArgumentOutOfRange, + errno.ESHUTDOWN : ConnectionShutdown + } + ret = abs(ret) + if ret in errors: + return errors[ret](msg) + else: + return Error(msg + (": error code %d" % ret)) + +class rbd_image_info_t(Structure): + _fields_ = [("size", c_uint64), + ("obj_size", c_uint64), + ("num_objs", c_uint64), + ("order", c_int), + ("block_name_prefix", c_char * 24), + ("parent_pool", c_int64), + ("parent_name", c_char * 96)] + +class rbd_snap_info_t(Structure): + _fields_ = [("id", c_uint64), + ("size", c_uint64), + ("name", c_char_p)] + +class RBD(object): + """ + This class wraps librbd CRUD functions. + """ + def __init__(self): + self.librbd = CDLL('librbd.so.1') + + def version(self): + """ + Get the version number of the ``librbd`` C library. + + :returns: a tuple of ``(major, minor, extra)`` components of the + librbd version + """ + major = c_int(0) + minor = c_int(0) + extra = c_int(0) + self.librbd.rbd_version(byref(major), byref(minor), byref(extra)) + return (major.value, minor.value, extra.value) + + def create(self, ioctx, name, size, order=None, old_format=True, + features=0, stripe_unit=0, stripe_count=0): + """ + Create an rbd image. + + :param ioctx: the context in which to create the image + :type ioctx: :class:`rados.Ioctx` + :param name: what the image is called + :type name: str + :param size: how big the image is in bytes + :type size: int + :param order: the image is split into (2**order) byte objects + :type order: int + :param old_format: whether to create an old-style image that + is accessible by old clients, but can't + use more advanced features like layering. + :type old_format: bool + :param features: bitmask of features to enable + :type features: int + :param stripe_unit: stripe unit in bytes (default 0 for object size) + :type stripe_unit: int + :param stripe_count: objects to stripe over before looping + :type stripe_count: int + :raises: :class:`ImageExists` + :raises: :class:`TypeError` + :raises: :class:`InvalidArgument` + :raises: :class:`FunctionNotSupported` + """ + if order is None: + order = 0 + if not isinstance(name, str): + raise TypeError('name must be a string') + if old_format: + if features != 0 or stripe_unit != 0 or stripe_count != 0: + raise InvalidArgument('format 1 images do not support feature' + ' masks or non-default striping') + ret = self.librbd.rbd_create(ioctx.io, c_char_p(name), + c_uint64(size), + byref(c_int(order))) + else: + if not hasattr(self.librbd, 'rbd_create2'): + raise FunctionNotSupported('installed version of librbd does' + ' not support format 2 images') + has_create3 = hasattr(self.librbd, 'rbd_create3') + if (stripe_unit != 0 or stripe_count != 0) and not has_create3: + raise FunctionNotSupported('installed version of librbd does' + ' not support stripe unit or count') + if has_create3: + ret = self.librbd.rbd_create3(ioctx.io, c_char_p(name), + c_uint64(size), + c_uint64(features), + byref(c_int(order)), + c_uint64(stripe_unit), + c_uint64(stripe_count)) + else: + ret = self.librbd.rbd_create2(ioctx.io, c_char_p(name), + c_uint64(size), + c_uint64(features), + byref(c_int(order))) + if ret < 0: + raise make_ex(ret, 'error creating image') + + def clone(self, p_ioctx, p_name, p_snapname, c_ioctx, c_name, + features=0, order=None): + """ + Clone a parent rbd snapshot into a COW sparse child. + + :param p_ioctx: the parent context that represents the parent snap + :type ioctx: :class:`rados.Ioctx` + :param p_name: the parent image name + :type name: str + :param p_snapname: the parent image snapshot name + :type name: str + :param c_ioctx: the child context that represents the new clone + :type ioctx: :class:`rados.Ioctx` + :param c_name: the clone (child) name + :type name: str + :param features: bitmask of features to enable; if set, must include layering + :type features: int + :param order: the image is split into (2**order) byte objects + :type order: int + :raises: :class:`TypeError` + :raises: :class:`InvalidArgument` + :raises: :class:`ImageExists` + :raises: :class:`FunctionNotSupported` + :raises: :class:`ArgumentOutOfRange` + """ + if order is None: + order = 0 + if not isinstance(p_snapname, str) or not isinstance(p_name, str): + raise TypeError('parent name and snapname must be strings') + if not isinstance(c_name, str): + raise TypeError('child name must be a string') + + ret = self.librbd.rbd_clone(p_ioctx.io, c_char_p(p_name), + c_char_p(p_snapname), + c_ioctx.io, c_char_p(c_name), + c_uint64(features), + byref(c_int(order))) + if ret < 0: + raise make_ex(ret, 'error creating clone') + + def list(self, ioctx): + """ + List image names. + + :param ioctx: determines which RADOS pool is read + :type ioctx: :class:`rados.Ioctx` + :returns: list -- a list of image names + """ + size = c_size_t(512) + while True: + c_names = create_string_buffer(size.value) + ret = self.librbd.rbd_list(ioctx.io, byref(c_names), byref(size)) + if ret >= 0: + break + elif ret != -errno.ERANGE: + raise make_ex(ret, 'error listing images') + return filter(lambda name: name != '', c_names.raw.split('\0')) + + def remove(self, ioctx, name): + """ + Delete an RBD image. This may take a long time, since it does + not return until every object that comprises the image has + been deleted. Note that all snapshots must be deleted before + the image can be removed. If there are snapshots left, + :class:`ImageHasSnapshots` is raised. If the image is still + open, or the watch from a crashed client has not expired, + :class:`ImageBusy` is raised. + + :param ioctx: determines which RADOS pool the image is in + :type ioctx: :class:`rados.Ioctx` + :param name: the name of the image to remove + :type name: str + :raises: :class:`ImageNotFound`, :class:`ImageBusy`, + :class:`ImageHasSnapshots` + """ + if not isinstance(name, str): + raise TypeError('name must be a string') + ret = self.librbd.rbd_remove(ioctx.io, c_char_p(name)) + if ret != 0: + raise make_ex(ret, 'error removing image') + + def rename(self, ioctx, src, dest): + """ + Rename an RBD image. + + :param ioctx: determines which RADOS pool the image is in + :type ioctx: :class:`rados.Ioctx` + :param src: the current name of the image + :type src: str + :param dest: the new name of the image + :type dest: str + :raises: :class:`ImageNotFound`, :class:`ImageExists` + """ + if not isinstance(src, str) or not isinstance(dest, str): + raise TypeError('src and dest must be strings') + ret = self.librbd.rbd_rename(ioctx.io, c_char_p(src), c_char_p(dest)) + if ret != 0: + raise make_ex(ret, 'error renaming image') + +class Image(object): + """ + This class represents an RBD image. It is used to perform I/O on + the image and interact with snapshots. + + **Note**: Any method of this class may raise :class:`ImageNotFound` + if the image has been deleted. + """ + + def __init__(self, ioctx, name, snapshot=None, read_only=False): + """ + Open the image at the given snapshot. + If a snapshot is specified, the image will be read-only, unless + :func:`Image.set_snap` is called later. + + If read-only mode is used, metadata for the :class:`Image` + object (such as which snapshots exist) may become obsolete. See + the C api for more details. + + To clean up from opening the image, :func:`Image.close` should + be called. For ease of use, this is done automatically when + an :class:`Image` is used as a context manager (see :pep:`343`). + + :param ioctx: determines which RADOS pool the image is in + :type ioctx: :class:`rados.Ioctx` + :param name: the name of the image + :type name: str + :param snapshot: which snapshot to read from + :type snaphshot: str + :param read_only: whether to open the image in read-only mode + :type read_only: bool + """ + self.closed = True + self.librbd = CDLL('librbd.so.1') + self.image = c_void_p() + self.name = name + if not isinstance(name, str): + raise TypeError('name must be a string') + if snapshot is not None and not isinstance(snapshot, str): + raise TypeError('snapshot must be a string or None') + if read_only: + if not hasattr(self.librbd, 'rbd_open_read_only'): + raise FunctionNotSupported('installed version of librbd does ' + 'not support open in read-only mode') + ret = self.librbd.rbd_open_read_only(ioctx.io, c_char_p(name), + byref(self.image), + c_char_p(snapshot)) + else: + ret = self.librbd.rbd_open(ioctx.io, c_char_p(name), + byref(self.image), c_char_p(snapshot)) + if ret != 0: + raise make_ex(ret, 'error opening image %s at snapshot %s' % (name, snapshot)) + self.closed = False + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + """ + Closes the image. See :func:`close` + """ + self.close() + return False + + def close(self): + """ + Release the resources used by this image object. + + After this is called, this object should not be used. + """ + if not self.closed: + self.closed = True + self.librbd.rbd_close(self.image) + + def __del__(self): + self.close() + + def __str__(self): + s = "rbd.Image(" + dict.__repr__(self.__dict__) + ")" + return s + + def resize(self, size): + """ + Change the size of the image. + + :param size: the new size of the image + :type size: int + """ + ret = self.librbd.rbd_resize(self.image, c_uint64(size)) + if ret < 0: + raise make_ex(ret, 'error resizing image %s' % (self.name,)) + + def stat(self): + """ + Get information about the image. Currently parent pool and + parent name are always -1 and ''. + + :returns: dict - contains the following keys: + + * ``size`` (int) - the size of the image in bytes + + * ``obj_size`` (int) - the size of each object that comprises the + image + + * ``num_objs`` (int) - the number of objects in the image + + * ``order`` (int) - log_2(object_size) + + * ``block_name_prefix`` (str) - the prefix of the RADOS objects used + to store the image + + * ``parent_pool`` (int) - deprecated + + * ``parent_name`` (str) - deprecated + + See also :meth:`format` and :meth:`features`. + + """ + info = rbd_image_info_t() + ret = self.librbd.rbd_stat(self.image, byref(info), ctypes.sizeof(info)) + if ret != 0: + raise make_ex(ret, 'error getting info for image %s' % (self.name,)) + return { + 'size' : info.size, + 'obj_size' : info.obj_size, + 'num_objs' : info.num_objs, + 'order' : info.order, + 'block_name_prefix' : info.block_name_prefix, + 'parent_pool' : info.parent_pool, + 'parent_name' : info.parent_name + } + + def parent_info(self): + ret = -errno.ERANGE + size = 8 + while ret == -errno.ERANGE and size < 128: + pool = create_string_buffer(size) + name = create_string_buffer(size) + snapname = create_string_buffer(size) + ret = self.librbd.rbd_get_parent_info(self.image, pool, len(pool), + name, len(name), snapname, len(snapname)) + if ret == -errno.ERANGE: + size *= 2 + + if (ret != 0): + raise make_ex(ret, 'error getting parent info for image %s' % (self.name,)) + return (pool.value, name.value, snapname.value) + + def old_format(self): + old = c_uint8() + ret = self.librbd.rbd_get_old_format(self.image, byref(old)) + if (ret != 0): + raise make_ex(ret, 'error getting old_format for image' % (self.name)) + return old.value != 0 + + def size(self): + """ + Get the size of the image. If open to a snapshot, returns the + size of that snapshot. + + :returns: the size of the image in bytes + """ + image_size = c_uint64() + ret = self.librbd.rbd_get_size(self.image, byref(image_size)) + if (ret != 0): + raise make_ex(ret, 'error getting size for image' % (self.name)) + return image_size.value + + def features(self): + features = c_uint64() + ret = self.librbd.rbd_get_features(self.image, byref(features)) + if (ret != 0): + raise make_ex(ret, 'error getting features for image' % (self.name)) + return features.value + + def overlap(self): + overlap = c_uint64() + ret = self.librbd.rbd_get_overlap(self.image, byref(overlap)) + if (ret != 0): + raise make_ex(ret, 'error getting overlap for image' % (self.name)) + return overlap.value + + def copy(self, dest_ioctx, dest_name): + """ + Copy the image to another location. + + :param dest_ioctx: determines which pool to copy into + :type dest_ioctx: :class:`rados.Ioctx` + :param dest_name: the name of the copy + :type dest_name: str + :raises: :class:`ImageExists` + """ + if not isinstance(dest_name, str): + raise TypeError('dest_name must be a string') + ret = self.librbd.rbd_copy(self.image, dest_ioctx.io, c_char_p(dest_name)) + if ret < 0: + raise make_ex(ret, 'error copying image %s to %s' % (self.name, dest_name)) + + def list_snaps(self): + """ + Iterate over the snapshots of an image. + + :returns: :class:`SnapIterator` + """ + return SnapIterator(self) + + def create_snap(self, name): + """ + Create a snapshot of the image. + + :param name: the name of the snapshot + :type name: str + :raises: :class:`ImageExists` + """ + if not isinstance(name, str): + raise TypeError('name must be a string') + ret = self.librbd.rbd_snap_create(self.image, c_char_p(name)) + if ret != 0: + raise make_ex(ret, 'error creating snapshot %s from %s' % (name, self.name)) + + def remove_snap(self, name): + """ + Delete a snapshot of the image. + + :param name: the name of the snapshot + :type name: str + :raises: :class:`IOError`, :class:`ImageBusy` + """ + if not isinstance(name, str): + raise TypeError('name must be a string') + ret = self.librbd.rbd_snap_remove(self.image, c_char_p(name)) + if ret != 0: + raise make_ex(ret, 'error removing snapshot %s from %s' % (name, self.name)) + + def rollback_to_snap(self, name): + """ + Revert the image to its contents at a snapshot. This is a + potentially expensive operation, since it rolls back each + object individually. + + :param name: the snapshot to rollback to + :type name: str + :raises: :class:`IOError` + """ + if not isinstance(name, str): + raise TypeError('name must be a string') + ret = self.librbd.rbd_snap_rollback(self.image, c_char_p(name)) + if ret != 0: + raise make_ex(ret, 'error rolling back image %s to snapshot %s' % (self.name, name)) + + def protect_snap(self, name): + """ + Mark a snapshot as protected. This means it can't be deleted + until it is unprotected. + + :param name: the snapshot to protect + :type name: str + :raises: :class:`IOError`, :class:`ImageNotFound` + """ + if not isinstance(name, str): + raise TypeError('name must be a string') + ret = self.librbd.rbd_snap_protect(self.image, c_char_p(name)) + if ret != 0: + raise make_ex(ret, 'error protecting snapshot %s@%s' % (self.name, name)) + + def unprotect_snap(self, name): + """ + Mark a snapshot unprotected. This allows it to be deleted if + it was protected. + + :param name: the snapshot to unprotect + :type name: str + :raises: :class:`IOError`, :class:`ImageNotFound` + """ + if not isinstance(name, str): + raise TypeError('name must be a string') + ret = self.librbd.rbd_snap_unprotect(self.image, c_char_p(name)) + if ret != 0: + raise make_ex(ret, 'error unprotecting snapshot %s@%s' % (self.name, name)) + + def is_protected_snap(self, name): + """ + Find out whether a snapshot is protected from deletion. + + :param name: the snapshot to check + :type name: str + :returns: bool - whether the snapshot is protected + :raises: :class:`IOError`, :class:`ImageNotFound` + """ + if not isinstance(name, str): + raise TypeError('name must be a string') + is_protected = c_int() + ret = self.librbd.rbd_snap_is_protected(self.image, c_char_p(name), + byref(is_protected)) + if ret != 0: + raise make_ex(ret, 'error checking if snapshot %s@%s is protected' % (self.name, name)) + return is_protected.value == 1 + + def set_snap(self, name): + """ + Set the snapshot to read from. Writes will raise ReadOnlyImage + while a snapshot is set. Pass None to unset the snapshot + (reads come from the current image) , and allow writing again. + + :param name: the snapshot to read from, or None to unset the snapshot + :type name: str or None + """ + if name is not None and not isinstance(name, str): + raise TypeError('name must be a string') + ret = self.librbd.rbd_snap_set(self.image, c_char_p(name)) + if ret != 0: + raise make_ex(ret, 'error setting image %s to snapshot %s' % (self.name, name)) + + def read(self, offset, length): + """ + Read data from the image. Raises :class:`InvalidArgument` if + part of the range specified is outside the image. + + :param offset: the offset to start reading at + :type offset: int + :param length: how many bytes to read + :type length: int + :returns: str - the data read + :raises: :class:`InvalidArgument`, :class:`IOError` + """ + ret_buf = create_string_buffer(length) + ret = self.librbd.rbd_read(self.image, c_uint64(offset), + c_size_t(length), byref(ret_buf)) + if ret < 0: + raise make_ex(ret, 'error reading %s %ld~%ld' % (self.image, offset, length)) + return ctypes.string_at(ret_buf, ret) + + def diff_iterate(self, offset, length, from_snapshot, iterate_cb): + """ + Iterate over the changed extents of an image. + + This will call iterate_cb with three arguments: + + (offset, length, exists) + + where the changed extent starts at offset bytes, continues for + length bytes, and is full of data (if exists is True) or zeroes + (if exists is False). + + If from_snapshot is None, it is interpreted as the beginning + of time and this generates all allocated extents. + + The end version is whatever is currently selected (via set_snap) + for the image. + + Raises :class:`InvalidArgument` if from_snapshot is after + the currently set snapshot. + + Raises :class:`ImageNotFound` if from_snapshot is not the name + of a snapshot of the image. + + :param offset: start offset in bytes + :type offset: int + :param length: size of region to report on, in bytes + :type length: int + :param from_snapshot: starting snapshot name, or None + :type from_snapshot: str or None + :param iterate_cb: function to call for each extent + :type iterate_cb: function acception arguments for offset, + length, and exists + :raises: :class:`InvalidArgument`, :class:`IOError`, + :class:`ImageNotFound` + """ + if from_snapshot is not None and not isinstance(from_snapshot, str): + raise TypeError('client must be a string') + + RBD_DIFF_CB = CFUNCTYPE(c_int, c_uint64, c_size_t, c_int, c_void_p) + cb_holder = DiffIterateCB(iterate_cb) + cb = RBD_DIFF_CB(cb_holder.callback) + ret = self.librbd.rbd_diff_iterate(self.image, + c_char_p(from_snapshot), + c_uint64(offset), + c_uint64(length), + cb, + c_void_p(None)) + if ret < 0: + msg = 'error generating diff from snapshot %s' % from_snapshot + raise make_ex(ret, msg) + + def write(self, data, offset): + """ + Write data to the image. Raises :class:`InvalidArgument` if + part of the write would fall outside the image. + + :param data: the data to be written + :type data: str + :param offset: where to start writing data + :type offset: int + :returns: int - the number of bytes written + :raises: :class:`IncompleteWriteError`, :class:`LogicError`, + :class:`InvalidArgument`, :class:`IOError` + """ + if not isinstance(data, str): + raise TypeError('data must be a string') + length = len(data) + ret = self.librbd.rbd_write(self.image, c_uint64(offset), + c_size_t(length), c_char_p(data)) + if ret == length: + return ret + elif ret < 0: + raise make_ex(ret, "error writing to %s" % (self.name,)) + elif ret < length: + raise IncompleteWriteError("Wrote only %ld out of %ld bytes" % (ret, length)) + else: + raise LogicError("logic error: rbd_write(%s) \ +returned %d, but %d was the maximum number of bytes it could have \ +written." % (self.name, ret, length)) + + def discard(self, offset, length): + """ + Trim the range from the image. It will be logically filled + with zeroes. + """ + ret = self.librbd.rbd_discard(self.image, + c_uint64(offset), + c_uint64(length)) + if ret < 0: + msg = 'error discarding region %d~%d' % (offset, length) + raise make_ex(ret, msg) + + def flush(self): + """ + Block until all writes are fully flushed if caching is enabled. + """ + ret = self.librbd.rbd_flush(self.image) + if ret < 0: + raise make_ex(ret, 'error flushing image') + + def stripe_unit(self): + """ + Returns the stripe unit used for the image. + """ + stripe_unit = c_uint64() + ret = self.librbd.rbd_get_stripe_unit(self.image, byref(stripe_unit)) + if ret != 0: + raise make_ex(ret, 'error getting stripe unit for image' % (self.name)) + return stripe_unit.value + + def stripe_count(self): + """ + Returns the stripe count used for the image. + """ + stripe_count = c_uint64() + ret = self.librbd.rbd_get_stripe_count(self.image, byref(stripe_count)) + if ret != 0: + raise make_ex(ret, 'error getting stripe count for image' % (self.name)) + return stripe_count.value + + def flatten(self): + """ + Flatten clone image (copy all blocks from parent to child) + """ + ret = self.librbd.rbd_flatten(self.image) + if (ret < 0): + raise make_ex(ret, "error flattening %s" % self.name) + + def list_children(self): + """ + List children of the currently set snapshot (set via set_snap()). + + :returns: list - a list of (pool name, image name) tuples + """ + pools_size = c_size_t(512) + images_size = c_size_t(512) + while True: + c_pools = create_string_buffer(pools_size.value) + c_images = create_string_buffer(images_size.value) + ret = self.librbd.rbd_list_children(self.image, + byref(c_pools), + byref(pools_size), + byref(c_images), + byref(images_size)) + if ret >= 0: + break + elif ret != -errno.ERANGE: + raise make_ex(ret, 'error listing images') + if ret == 0: + return [] + pools = c_pools.raw[:pools_size.value - 1].split('\0') + images = c_images.raw[:images_size.value - 1].split('\0') + return zip(pools, images) + + def list_lockers(self): + """ + List clients that have locked the image and information + about the lock. + + :returns: dict - contains the following keys: + + * ``tag`` - the tag associated with the lock (every + additional locker must use the same tag) + * ``exclusive`` - boolean indicating whether the + lock is exclusive or shared + * ``lockers`` - a list of (client, cookie, address) + tuples + """ + clients_size = c_size_t(512) + cookies_size = c_size_t(512) + addrs_size = c_size_t(512) + tag_size = c_size_t(512) + exclusive = c_int(0) + + while True: + c_clients = create_string_buffer(clients_size.value) + c_cookies = create_string_buffer(cookies_size.value) + c_addrs = create_string_buffer(addrs_size.value) + c_tag = create_string_buffer(tag_size.value) + ret = self.librbd.rbd_list_lockers(self.image, + byref(exclusive), + byref(c_tag), + byref(tag_size), + byref(c_clients), + byref(clients_size), + byref(c_cookies), + byref(cookies_size), + byref(c_addrs), + byref(addrs_size)) + if ret >= 0: + break + elif ret != -errno.ERANGE: + raise make_ex(ret, 'error listing images') + if ret == 0: + return [] + clients = c_clients.raw[:clients_size.value - 1].split('\0') + cookies = c_cookies.raw[:cookies_size.value - 1].split('\0') + addrs = c_addrs.raw[:addrs_size.value - 1].split('\0') + return { + 'tag' : c_tag.value, + 'exclusive' : exclusive.value == 1, + 'lockers' : zip(clients, cookies, addrs), + } + + def lock_exclusive(self, cookie): + """ + Take an exclusive lock on the image. + + :raises: :class:`ImageBusy` if a different client or cookie locked it + :class:`ImageExists` if the same client and cookie locked it + """ + if not isinstance(cookie, str): + raise TypeError('cookie must be a string') + ret = self.librbd.rbd_lock_exclusive(self.image, c_char_p(cookie)) + if ret < 0: + raise make_ex(ret, 'error acquiring exclusive lock on image') + + def lock_shared(self, cookie, tag): + """ + Take a shared lock on the image. The tag must match + that of the existing lockers, if any. + + :raises: :class:`ImageBusy` if a different client or cookie locked it + :class:`ImageExists` if the same client and cookie locked it + """ + if not isinstance(cookie, str): + raise TypeError('cookie must be a string') + if not isinstance(tag, str): + raise TypeError('tag must be a string') + ret = self.librbd.rbd_lock_shared(self.image, c_char_p(cookie), + c_char_p(tag)) + if ret < 0: + raise make_ex(ret, 'error acquiring shared lock on image') + + def unlock(self, cookie): + """ + Release a lock on the image that was locked by this rados client. + """ + if not isinstance(cookie, str): + raise TypeError('cookie must be a string') + ret = self.librbd.rbd_unlock(self.image, c_char_p(cookie)) + if ret < 0: + raise make_ex(ret, 'error unlocking image') + + def break_lock(self, client, cookie): + """ + Release a lock held by another rados client. + """ + if not isinstance(client, str): + raise TypeError('client must be a string') + if not isinstance(cookie, str): + raise TypeError('cookie must be a string') + ret = self.librbd.rbd_break_lock(self.image, c_char_p(client), + c_char_p(cookie)) + if ret < 0: + raise make_ex(ret, 'error unlocking image') + +class DiffIterateCB(object): + def __init__(self, cb): + self.cb = cb + + def callback(self, offset, length, exists, unused): + self.cb(offset, length, exists == 1) + return 0 + +class SnapIterator(object): + """ + Iterator over snapshot info for an image. + + Yields a dictionary containing information about a snapshot. + + Keys are: + + * ``id`` (int) - numeric identifier of the snapshot + + * ``size`` (int) - size of the image at the time of snapshot (in bytes) + + * ``name`` (str) - name of the snapshot + """ + def __init__(self, image): + self.librbd = image.librbd + num_snaps = c_int(10) + while True: + self.snaps = (rbd_snap_info_t * num_snaps.value)() + ret = self.librbd.rbd_snap_list(image.image, byref(self.snaps), + byref(num_snaps)) + if ret >= 0: + self.num_snaps = ret + break + elif ret != -errno.ERANGE: + raise make_ex(ret, 'error listing snapshots for image %s' % (image.name,)) + + def __iter__(self): + for i in xrange(self.num_snaps): + yield { + 'id' : self.snaps[i].id, + 'size' : self.snaps[i].size, + 'name' : self.snaps[i].name, + } + + def __del__(self): + self.librbd.rbd_snap_list_end(self.snaps) diff --git a/src/pybind/rbd/setup.py b/src/pybind/rbd/setup.py new file mode 100644 index 00000000000..e642cb456b2 --- /dev/null +++ b/src/pybind/rbd/setup.py @@ -0,0 +1,32 @@ +from setuptools import setup, find_packages +import os + + +def long_description(): + readme = os.path.join(os.path.dirname(__file__), 'README.rst') + return open(readme).read() + + +setup( + name = 'rbd', + description = 'Bindings for rbd [ceph]', + packages=find_packages(), + author = 'Inktank', + author_email = 'ceph-devel@vger.kernel.org', + version = '0.67', + license = "LGPL2", + zip_safe = False, + keywords = "ceph, rbd, bindings, api, cli", + long_description = long_description(), + classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)', + 'Topic :: Software Development :: Libraries', + 'Topic :: Utilities', + 'Topic :: System :: Filesystems', + 'Operating System :: POSIX', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], +) diff --git a/src/pybind/rbd/tox.ini b/src/pybind/rbd/tox.ini new file mode 100644 index 00000000000..9c63bb9d884 --- /dev/null +++ b/src/pybind/rbd/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py26, py27 + +[testenv] +deps= +commands= -- cgit v1.2.1 From fd8892b5a253afd7e8f39614b7673984dfdb87d7 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 24 Sep 2013 13:50:11 -0400 Subject: move ceph-rest and creat the package Signed-off-by: Alfredo Deza --- src/pybind/MANIFEST.in | 1 - src/pybind/ceph-rest/argparse.py | 1110 +++++++++++++++++++++++++ src/pybind/ceph-rest/ceph_rest.py | 499 +++++++++++ src/pybind/ceph_argparse.py | 1110 ------------------------- src/pybind/ceph_rest_api.py | 499 ----------- src/pybind/rados.py | 1661 ------------------------------------- src/pybind/rados/rados.py | 13 + src/pybind/setup.py | 32 - src/pybind/tox.ini | 6 - 9 files changed, 1622 insertions(+), 3309 deletions(-) delete mode 100644 src/pybind/MANIFEST.in create mode 100644 src/pybind/ceph-rest/argparse.py create mode 100755 src/pybind/ceph-rest/ceph_rest.py delete mode 100644 src/pybind/ceph_argparse.py delete mode 100755 src/pybind/ceph_rest_api.py delete mode 100644 src/pybind/rados.py delete mode 100644 src/pybind/setup.py delete mode 100644 src/pybind/tox.ini diff --git a/src/pybind/MANIFEST.in b/src/pybind/MANIFEST.in deleted file mode 100644 index f1cb7376052..00000000000 --- a/src/pybind/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include setup.py diff --git a/src/pybind/ceph-rest/argparse.py b/src/pybind/ceph-rest/argparse.py new file mode 100644 index 00000000000..1f6e90b6c1d --- /dev/null +++ b/src/pybind/ceph-rest/argparse.py @@ -0,0 +1,1110 @@ +""" +Types and routines used by the ceph CLI as well as the RESTful +interface. These have to do with querying the daemons for +command-description information, validating user command input against +those descriptions, and submitting the command to the appropriate +daemon. + +Copyright (C) 2013 Inktank Storage, Inc. + +LGPL2. See file COPYING. +""" +import copy +import json +import os +import pprint +import re +import socket +import stat +import sys +import types +import uuid + +class ArgumentError(Exception): + """ + Something wrong with arguments + """ + pass + +class ArgumentNumber(ArgumentError): + """ + Wrong number of a repeated argument + """ + pass + +class ArgumentFormat(ArgumentError): + """ + Argument value has wrong format + """ + pass + +class ArgumentValid(ArgumentError): + """ + Argument value is otherwise invalid (doesn't match choices, for instance) + """ + pass + +class ArgumentTooFew(ArgumentError): + """ + Fewer arguments than descriptors in signature; may mean to continue + the search, so gets a special exception type + """ + +class ArgumentPrefix(ArgumentError): + """ + Special for mismatched prefix; less severe, don't report by default + """ + pass + +class JsonFormat(Exception): + """ + some syntactic or semantic issue with the JSON + """ + pass + +class CephArgtype(object): + """ + Base class for all Ceph argument types + + Instantiating an object sets any validation parameters + (allowable strings, numeric ranges, etc.). The 'valid' + method validates a string against that initialized instance, + throwing ArgumentError if there's a problem. + """ + def __init__(self, **kwargs): + """ + set any per-instance validation parameters here + from kwargs (fixed string sets, integer ranges, etc) + """ + pass + + def valid(self, s, partial=False): + """ + Run validation against given string s (generally one word); + partial means to accept partial string matches (begins-with). + If cool, set self.val to the value that should be returned + (a copy of the input string, or a numeric or boolean interpretation + thereof, for example) + if not, throw ArgumentError(msg-as-to-why) + """ + self.val = s + + def __repr__(self): + """ + return string representation of description of type. Note, + this is not a representation of the actual value. Subclasses + probably also override __str__() to give a more user-friendly + 'name/type' description for use in command format help messages. + """ + a = '' + if hasattr(self, 'typeargs'): + a = self.typeargs + return '{0}(\'{1}\')'.format(self.__class__.__name__, a) + + def __str__(self): + """ + where __repr__ (ideally) returns a string that could be used to + reproduce the object, __str__ returns one you'd like to see in + print messages. Use __str__ to format the argtype descriptor + as it would be useful in a command usage message. + """ + return '<{0}>'.format(self.__class__.__name__) + +class CephInt(CephArgtype): + """ + range-limited integers, [+|-][0-9]+ or 0x[0-9a-f]+ + range: list of 1 or 2 ints, [min] or [min,max] + """ + def __init__(self, range=''): + if range == '': + self.range = list() + else: + self.range = list(range.split('|')) + self.range = map(long, self.range) + + def valid(self, s, partial=False): + try: + val = long(s) + except ValueError: + raise ArgumentValid("{0} doesn't represent an int".format(s)) + if len(self.range) == 2: + if val < self.range[0] or val > self.range[1]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + elif len(self.range) == 1: + if val < self.range[0]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + self.val = val + + def __str__(self): + r = '' + if len(self.range) == 1: + r = '[{0}-]'.format(self.range[0]) + if len(self.range) == 2: + r = '[{0}-{1}]'.format(self.range[0], self.range[1]) + + return ''.format(r) + + +class CephFloat(CephArgtype): + """ + range-limited float type + range: list of 1 or 2 floats, [min] or [min, max] + """ + def __init__(self, range=''): + if range == '': + self.range = list() + else: + self.range = list(range.split('|')) + self.range = map(float, self.range) + + def valid(self, s, partial=False): + try: + val = float(s) + except ValueError: + raise ArgumentValid("{0} doesn't represent a float".format(s)) + if len(self.range) == 2: + if val < self.range[0] or val > self.range[1]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + elif len(self.range) == 1: + if val < self.range[0]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + self.val = val + + def __str__(self): + r = '' + if len(self.range) == 1: + r = '[{0}-]'.format(self.range[0]) + if len(self.range) == 2: + r = '[{0}-{1}]'.format(self.range[0], self.range[1]) + return ''.format(r) + +class CephString(CephArgtype): + """ + String; pretty generic. goodchars is a RE char class of valid chars + """ + def __init__(self, goodchars=''): + from string import printable + try: + re.compile(goodchars) + except: + raise ValueError('CephString(): "{0}" is not a valid RE'.\ + format(goodchars)) + self.goodchars = goodchars + self.goodset = frozenset( + [c for c in printable if re.match(goodchars, c)] + ) + + def valid(self, s, partial=False): + sset = set(s) + if self.goodset and not sset <= self.goodset: + raise ArgumentFormat("invalid chars {0} in {1}".\ + format(''.join(sset - self.goodset), s)) + self.val = s + + def __str__(self): + b = '' + if self.goodchars: + b += '(goodchars {0})'.format(self.goodchars) + return ''.format(b) + +class CephSocketpath(CephArgtype): + """ + Admin socket path; check that it's readable and S_ISSOCK + """ + def valid(self, s, partial=False): + mode = os.stat(s).st_mode + if not stat.S_ISSOCK(mode): + raise ArgumentValid('socket path {0} is not a socket'.format(s)) + self.val = s + + def __str__(self): + return '' + +class CephIPAddr(CephArgtype): + """ + IP address (v4 or v6) with optional port + """ + def valid(self, s, partial=False): + # parse off port, use socket to validate addr + type = 6 + if s.startswith('['): + type = 6 + elif s.find('.') != -1: + type = 4 + if type == 4: + port = s.find(':') + if (port != -1): + a = s[:port] + p = s[port+1:] + if int(p) > 65535: + raise ArgumentValid('{0}: invalid IPv4 port'.format(p)) + else: + a = s + p = None + try: + socket.inet_pton(socket.AF_INET, a) + except: + raise ArgumentValid('{0}: invalid IPv4 address'.format(a)) + else: + # v6 + if s.startswith('['): + end = s.find(']') + if end == -1: + raise ArgumentFormat('{0} missing terminating ]'.format(s)) + if s[end+1] == ':': + try: + p = int(s[end+2]) + except: + raise ArgumentValid('{0}: bad port number'.format(s)) + a = s[1:end] + else: + a = s + p = None + try: + socket.inet_pton(socket.AF_INET6, a) + except: + raise ArgumentValid('{0} not valid IPv6 address'.format(s)) + if p is not None and long(p) > 65535: + raise ArgumentValid("{0} not a valid port number".format(p)) + self.val = s + self.addr = a + self.port = p + + def __str__(self): + return '' + +class CephEntityAddr(CephIPAddr): + """ + EntityAddress, that is, IP address[/nonce] + """ + def valid(self, s, partial=False): + nonce = None + if '/' in s: + ip, nonce = s.split('/') + else: + ip = s + super(self.__class__, self).valid(ip) + if nonce: + nonce_long = None + try: + nonce_long = long(nonce) + except ValueError: + pass + if nonce_long is None or nonce_long < 0: + raise ArgumentValid( + '{0}: invalid entity, nonce {1} not integer > 0'.\ + format(s, nonce) + ) + self.val = s + + def __str__(self): + return '' + +class CephPoolname(CephArgtype): + """ + Pool name; very little utility + """ + def __str__(self): + return '' + +class CephObjectname(CephArgtype): + """ + Object name. Maybe should be combined with Pool name as they're always + present in pairs, and then could be checked for presence + """ + def __str__(self): + return '' + +class CephPgid(CephArgtype): + """ + pgid, in form N.xxx (N = pool number, xxx = hex pgnum) + """ + def valid(self, s, partial=False): + if s.find('.') == -1: + raise ArgumentFormat('pgid has no .') + poolid, pgnum = s.split('.') + if poolid < 0: + raise ArgumentFormat('pool {0} < 0'.format(poolid)) + try: + pgnum = int(pgnum, 16) + except: + raise ArgumentFormat('pgnum {0} not hex integer'.format(pgnum)) + self.val = s + + def __str__(self): + return '' + +class CephName(CephArgtype): + """ + Name (type.id) where: + type is osd|mon|client|mds + id is a base10 int, if type == osd, or a string otherwise + + Also accept '*' + """ + def __init__(self): + self.nametype = None + self.nameid = None + + def valid(self, s, partial=False): + if s == '*': + self.val = s + return + if s.find('.') == -1: + raise ArgumentFormat('CephName: no . in {0}'.format(s)) + else: + t, i = s.split('.') + if not t in ('osd', 'mon', 'client', 'mds'): + raise ArgumentValid('unknown type ' + t) + if t == 'osd': + if i != '*': + try: + i = int(i) + except: + raise ArgumentFormat('osd id ' + i + ' not integer') + self.nametype = t + self.val = s + self.nameid = i + + def __str__(self): + return '' + +class CephOsdName(CephArgtype): + """ + Like CephName, but specific to osds: allow alone + + osd., or , or *, where id is a base10 int + """ + def __init__(self): + self.nametype = None + self.nameid = None + + def valid(self, s, partial=False): + if s == '*': + self.val = s + return + if s.find('.') != -1: + t, i = s.split('.') + if t != 'osd': + raise ArgumentValid('unknown type ' + t) + else: + t = 'osd' + i = s + try: + i = int(i) + except: + raise ArgumentFormat('osd id ' + i + ' not integer') + self.nametype = t + self.nameid = i + self.val = i + + def __str__(self): + return '' + +class CephChoices(CephArgtype): + """ + Set of string literals; init with valid choices + """ + def __init__(self, strings='', **kwargs): + self.strings = strings.split('|') + + def valid(self, s, partial=False): + if not partial: + if not s in self.strings: + # show as __str__ does: {s1|s2..} + raise ArgumentValid("{0} not in {1}".format(s, self)) + self.val = s + return + + # partial + for t in self.strings: + if t.startswith(s): + self.val = s + return + raise ArgumentValid("{0} not in {1}". format(s, self)) + + def __str__(self): + if len(self.strings) == 1: + return '{0}'.format(self.strings[0]) + else: + return '{0}'.format('|'.join(self.strings)) + +class CephFilepath(CephArgtype): + """ + Openable file + """ + def valid(self, s, partial=False): + try: + f = open(s, 'a+') + except Exception as e: + raise ArgumentValid('can\'t open {0}: {1}'.format(s, e)) + f.close() + self.val = s + + def __str__(self): + return '' + +class CephFragment(CephArgtype): + """ + 'Fragment' ??? XXX + """ + def valid(self, s, partial=False): + if s.find('/') == -1: + raise ArgumentFormat('{0}: no /'.format(s)) + val, bits = s.split('/') + # XXX is this right? + if not val.startswith('0x'): + raise ArgumentFormat("{0} not a hex integer".format(val)) + try: + long(val) + except: + raise ArgumentFormat('can\'t convert {0} to integer'.format(val)) + try: + long(bits) + except: + raise ArgumentFormat('can\'t convert {0} to integer'.format(bits)) + self.val = s + + def __str__(self): + return "" + + +class CephUUID(CephArgtype): + """ + CephUUID: pretty self-explanatory + """ + def valid(self, s, partial=False): + try: + uuid.UUID(s) + except Exception as e: + raise ArgumentFormat('invalid UUID {0}: {1}'.format(s, e)) + self.val = s + + def __str__(self): + return '' + + +class CephPrefix(CephArgtype): + """ + CephPrefix: magic type for "all the first n fixed strings" + """ + def __init__(self, prefix=''): + self.prefix = prefix + + def valid(self, s, partial=False): + if partial: + if self.prefix.startswith(s): + self.val = s + return + else: + if (s == self.prefix): + self.val = s + return + + raise ArgumentPrefix("no match for {0}".format(s)) + + def __str__(self): + return self.prefix + + +class argdesc(object): + """ + argdesc(typename, name='name', n=numallowed|N, + req=False, helptext=helptext, **kwargs (type-specific)) + + validation rules: + typename: type(**kwargs) will be constructed + later, type.valid(w) will be called with a word in that position + + name is used for parse errors and for constructing JSON output + n is a numeric literal or 'n|N', meaning "at least one, but maybe more" + req=False means the argument need not be present in the list + helptext is the associated help for the command + anything else are arguments to pass to the type constructor. + + self.instance is an instance of type t constructed with typeargs. + + valid() will later be called with input to validate against it, + and will store the validated value in self.instance.val for extraction. + """ + def __init__(self, t, name=None, n=1, req=True, **kwargs): + if isinstance(t, types.StringTypes): + self.t = CephPrefix + self.typeargs = {'prefix':t} + self.req = True + else: + self.t = t + self.typeargs = kwargs + self.req = bool(req == True or req == 'True') + + self.name = name + self.N = (n in ['n', 'N']) + if self.N: + self.n = 1 + else: + self.n = int(n) + self.instance = self.t(**self.typeargs) + + def __repr__(self): + r = 'argdesc(' + str(self.t) + ', ' + internals = ['N', 'typeargs', 'instance', 't'] + for (k, v) in self.__dict__.iteritems(): + if k.startswith('__') or k in internals: + pass + else: + # undo modification from __init__ + if k == 'n' and self.N: + v = 'N' + r += '{0}={1}, '.format(k, v) + for (k, v) in self.typeargs.iteritems(): + r += '{0}={1}, '.format(k, v) + return r[:-2] + ')' + + def __str__(self): + if ((self.t == CephChoices and len(self.instance.strings) == 1) + or (self.t == CephPrefix)): + s = str(self.instance) + else: + s = '{0}({1})'.format(self.name, str(self.instance)) + if self.N: + s += ' [' + str(self.instance) + '...]' + if not self.req: + s = '{' + s + '}' + return s + + def helpstr(self): + """ + like str(), but omit parameter names (except for CephString, + which really needs them) + """ + if self.t == CephString: + chunk = '<{0}>'.format(self.name) + else: + chunk = str(self.instance) + s = chunk + if self.N: + s += ' [' + chunk + '...]' + if not self.req: + s = '{' + s + '}' + return s + +def concise_sig(sig): + """ + Return string representation of sig useful for syntax reference in help + """ + return ' '.join([d.helpstr() for d in sig]) + +def descsort(sh1, sh2): + """ + sort descriptors by prefixes, defined as the concatenation of all simple + strings in the descriptor; this works out to just the leading strings. + """ + return cmp(concise_sig(sh1['sig']), concise_sig(sh2['sig'])) + +def parse_funcsig(sig): + """ + parse a single descriptor (array of strings or dicts) into a + dict of function descriptor/validators (objects of CephXXX type) + """ + newsig = [] + argnum = 0 + for desc in sig: + argnum += 1 + if isinstance(desc, types.StringTypes): + t = CephPrefix + desc = {'type':t, 'name':'prefix', 'prefix':desc} + else: + # not a simple string, must be dict + if not 'type' in desc: + s = 'JSON descriptor {0} has no type'.format(sig) + raise JsonFormat(s) + # look up type string in our globals() dict; if it's an + # object of type types.TypeType, it must be a + # locally-defined class. otherwise, we haven't a clue. + if desc['type'] in globals(): + t = globals()[desc['type']] + if type(t) != types.TypeType: + s = 'unknown type {0}'.format(desc['type']) + raise JsonFormat(s) + else: + s = 'unknown type {0}'.format(desc['type']) + raise JsonFormat(s) + + kwargs = dict() + for key, val in desc.items(): + if key not in ['type', 'name', 'n', 'req']: + kwargs[key] = val + newsig.append(argdesc(t, + name=desc.get('name', None), + n=desc.get('n', 1), + req=desc.get('req', True), + **kwargs)) + return newsig + + +def parse_json_funcsigs(s, consumer): + """ + A function signature is mostly an array of argdesc; it's represented + in JSON as + { + "cmd001": {"sig":[ "type": type, "name": name, "n": num, "req":true|false ], "help":helptext, "module":modulename, "perm":perms, "avail":availability} + . + . + . + ] + + A set of sigs is in an dict mapped by a unique number: + { + "cmd1": { + "sig": ["type.. ], "help":helptext... + } + "cmd2"{ + "sig": [.. ], "help":helptext... + } + } + + Parse the string s and return a dict of dicts, keyed by opcode; + each dict contains 'sig' with the array of descriptors, and 'help' + with the helptext, 'module' with the module name, 'perm' with a + string representing required permissions in that module to execute + this command (and also whether it is a read or write command from + the cluster state perspective), and 'avail' as a hint for + whether the command should be advertised by CLI, REST, or both. + If avail does not contain 'consumer', don't include the command + in the returned dict. + """ + try: + overall = json.loads(s) + except Exception as e: + print >> sys.stderr, "Couldn't parse JSON {0}: {1}".format(s, e) + raise e + sigdict = {} + for cmdtag, cmd in overall.iteritems(): + if not 'sig' in cmd: + s = "JSON descriptor {0} has no 'sig'".format(cmdtag) + raise JsonFormat(s) + # check 'avail' and possibly ignore this command + if 'avail' in cmd: + if not consumer in cmd['avail']: + continue + # rewrite the 'sig' item with the argdesc-ized version, and... + cmd['sig'] = parse_funcsig(cmd['sig']) + # just take everything else as given + sigdict[cmdtag] = cmd + return sigdict + +def validate_one(word, desc, partial=False): + """ + validate_one(word, desc, partial=False) + + validate word against the constructed instance of the type + in desc. May raise exception. If it returns false (and doesn't + raise an exception), desc.instance.val will + contain the validated value (in the appropriate type). + """ + desc.instance.valid(word, partial) + desc.numseen += 1 + if desc.N: + desc.n = desc.numseen + 1 + +def matchnum(args, signature, partial=False): + """ + matchnum(s, signature, partial=False) + + Returns number of arguments matched in s against signature. + Can be used to determine most-likely command for full or partial + matches (partial applies to string matches). + """ + words = args[:] + mysig = copy.deepcopy(signature) + matchcnt = 0 + for desc in mysig: + setattr(desc, 'numseen', 0) + while desc.numseen < desc.n: + # if there are no more arguments, return + if not words: + return matchcnt + word = words.pop(0) + + try: + validate_one(word, desc, partial) + valid = True + except ArgumentError: + # matchnum doesn't care about type of error + valid = False + + if not valid: + if not desc.req: + # this wasn't required, so word may match the next desc + words.insert(0, word) + break + else: + # it was required, and didn't match, return + return matchcnt + if desc.req: + matchcnt += 1 + return matchcnt + +def get_next_arg(desc, args): + ''' + Get either the value matching key 'desc.name' or the next arg in + the non-dict list. Return None if args are exhausted. Used in + validate() below. + ''' + arg = None + if isinstance(args, dict): + arg = args.pop(desc.name, None) + # allow 'param=param' to be expressed as 'param' + if arg == '': + arg = desc.name + # Hack, or clever? If value is a list, keep the first element, + # push rest back onto myargs for later processing. + # Could process list directly, but nesting here is already bad + if arg and isinstance(arg, list): + args[desc.name] = arg[1:] + arg = arg[0] + elif args: + arg = args.pop(0) + if arg and isinstance(arg, list): + args = arg[1:] + args + arg = arg[0] + return arg + +def store_arg(desc, d): + ''' + Store argument described by, and held in, thanks to valid(), + desc into the dictionary d, keyed by desc.name. Three cases: + + 1) desc.N is set: value in d is a list + 2) prefix: multiple args are joined with ' ' into one d{} item + 3) single prefix or other arg: store as simple value + + Used in validate() below. + ''' + if desc.N: + # value should be a list + if desc.name in d: + d[desc.name] += [desc.instance.val] + else: + d[desc.name] = [desc.instance.val] + elif (desc.t == CephPrefix) and (desc.name in d): + # prefixes' values should be a space-joined concatenation + d[desc.name] += ' ' + desc.instance.val + else: + # if first CephPrefix or any other type, just set it + d[desc.name] = desc.instance.val + +def validate(args, signature, partial=False): + """ + validate(args, signature, partial=False) + + args is a list of either words or k,v pairs representing a possible + command input following format of signature. Runs a validation; no + exception means it's OK. Return a dict containing all arguments keyed + by their descriptor name, with duplicate args per name accumulated + into a list (or space-separated value for CephPrefix). + + Mismatches of prefix are non-fatal, as this probably just means the + search hasn't hit the correct command. Mismatches of non-prefix + arguments are treated as fatal, and an exception raised. + + This matching is modified if partial is set: allow partial matching + (with partial dict returned); in this case, there are no exceptions + raised. + """ + + myargs = copy.deepcopy(args) + mysig = copy.deepcopy(signature) + reqsiglen = len([desc for desc in mysig if desc.req]) + matchcnt = 0 + d = dict() + for desc in mysig: + setattr(desc, 'numseen', 0) + while desc.numseen < desc.n: + myarg = get_next_arg(desc, myargs) + + # no arg, but not required? Continue consuming mysig + # in case there are later required args + if not myarg and not desc.req: + break + + # out of arguments for a required param? + # Either return (if partial validation) or raise + if not myarg and desc.req: + if desc.N and desc.numseen < 1: + # wanted N, didn't even get 1 + if partial: + return d + raise ArgumentNumber( + 'saw {0} of {1}, expected at least 1'.\ + format(desc.numseen, desc) + ) + elif not desc.N and desc.numseen < desc.n: + # wanted n, got too few + if partial: + return d + # special-case the "0 expected 1" case + if desc.numseen == 0 and desc.n == 1: + raise ArgumentNumber( + 'missing required parameter {0}'.format(desc) + ) + raise ArgumentNumber( + 'saw {0} of {1}, expected {2}'.\ + format(desc.numseen, desc, desc.n) + ) + break + + # Have an arg; validate it + try: + validate_one(myarg, desc) + valid = True + except ArgumentError as e: + valid = False + if not valid: + # argument mismatch + if not desc.req: + # if not required, just push back; it might match + # the next arg + print >> sys.stderr, myarg, 'not valid: ', str(e) + myargs.insert(0, myarg) + break + else: + # hm, it was required, so time to return/raise + if partial: + return d + raise e + + # Whew, valid arg acquired. Store in dict + matchcnt += 1 + store_arg(desc, d) + + # Done with entire list of argdescs + if matchcnt < reqsiglen: + raise ArgumentTooFew("not enough arguments given") + + if myargs and not partial: + raise ArgumentError("unused arguments: " + str(myargs)) + + # Finally, success + return d + +def cmdsiglen(sig): + sigdict = sig.values() + assert len(sigdict) == 1 + return len(sig.values()[0]['sig']) + +def validate_command(sigdict, args, verbose=False): + """ + turn args into a valid dictionary ready to be sent off as JSON, + validated against sigdict. + """ + found = [] + valid_dict = {} + if args: + # look for best match, accumulate possibles in bestcmds + # (so we can maybe give a more-useful error message) + best_match_cnt = 0 + bestcmds = [] + for cmdtag, cmd in sigdict.iteritems(): + sig = cmd['sig'] + matched = matchnum(args, sig, partial=True) + if (matched > best_match_cnt): + if verbose: + print >> sys.stderr, \ + "better match: {0} > {1}: {2}:{3} ".format(matched, + best_match_cnt, cmdtag, concise_sig(sig)) + best_match_cnt = matched + bestcmds = [{cmdtag:cmd}] + elif matched == best_match_cnt: + if verbose: + print >> sys.stderr, \ + "equal match: {0} > {1}: {2}:{3} ".format(matched, + best_match_cnt, cmdtag, concise_sig(sig)) + bestcmds.append({cmdtag:cmd}) + + # Sort bestcmds by number of args so we can try shortest first + # (relies on a cmdsig being key,val where val is a list of len 1) + bestcmds_sorted = sorted(bestcmds, + cmp=lambda x,y:cmp(cmdsiglen(x), cmdsiglen(y))) + + if verbose: + print >> sys.stderr, "bestcmds_sorted: " + pprint.PrettyPrinter(stream=sys.stderr).pprint(bestcmds_sorted) + + # for everything in bestcmds, look for a true match + for cmdsig in bestcmds_sorted: + for cmd in cmdsig.itervalues(): + sig = cmd['sig'] + try: + valid_dict = validate(args, sig) + found = cmd + break + except ArgumentPrefix: + # ignore prefix mismatches; we just haven't found + # the right command yet + pass + except ArgumentTooFew: + # It looked like this matched the beginning, but it + # didn't have enough args supplied. If we're out of + # cmdsigs we'll fall out unfound; if we're not, maybe + # the next one matches completely. Whine, but pass. + if verbose: + print >> sys.stderr, 'Not enough args supplied for ', \ + concise_sig(sig) + except ArgumentError as e: + # Solid mismatch on an arg (type, range, etc.) + # Stop now, because we have the right command but + # some other input is invalid + print >> sys.stderr, "Invalid command: ", str(e) + print >> sys.stderr, concise_sig(sig), ': ', cmd['help'] + return {} + if found: + break + + if not found: + print >> sys.stderr, 'no valid command found; 10 closest matches:' + for cmdsig in bestcmds[:10]: + for (cmdtag, cmd) in cmdsig.iteritems(): + print >> sys.stderr, concise_sig(cmd['sig']) + return None + + return valid_dict + +def find_cmd_target(childargs): + """ + Using a minimal validation, figure out whether the command + should be sent to a monitor or an osd. We do this before even + asking for the 'real' set of command signatures, so we can ask the + right daemon. + Returns ('osd', osdid), ('pg', pgid), or ('mon', '') + """ + sig = parse_funcsig(['tell', {'name':'target', 'type':'CephName'}]) + try: + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: + if len(valid_dict) == 2: + # revalidate to isolate type and id + name = CephName() + # if this fails, something is horribly wrong, as it just + # validated successfully above + name.valid(valid_dict['target']) + return name.nametype, name.nameid + + sig = parse_funcsig(['tell', {'name':'pgid', 'type':'CephPgid'}]) + try: + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: + if len(valid_dict) == 2: + # pg doesn't need revalidation; the string is fine + return 'pg', valid_dict['pgid'] + + sig = parse_funcsig(['pg', {'name':'pgid', 'type':'CephPgid'}]) + try: + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: + if len(valid_dict) == 2: + return 'pg', valid_dict['pgid'] + + return 'mon', '' + +def send_command(cluster, target=('mon', ''), cmd=None, inbuf='', timeout=0, + verbose=False): + """ + Send a command to a daemon using librados's + mon_command, osd_command, or pg_command. Any bulk input data + comes in inbuf. + + Returns (ret, outbuf, outs); ret is the return code, outbuf is + the outbl "bulk useful output" buffer, and outs is any status + or error message (intended for stderr). + + If target is osd.N, send command to that osd (except for pgid cmds) + """ + cmd = cmd or [] + try: + if target[0] == 'osd': + osdid = target[1] + + if verbose: + print >> sys.stderr, 'submit {0} to osd.{1}'.\ + format(cmd, osdid) + ret, outbuf, outs = \ + cluster.osd_command(osdid, cmd, inbuf, timeout) + + elif target[0] == 'pg': + pgid = target[1] + # pgid will already be in the command for the pg + # form, but for tell , we need to put it in + if cmd: + cmddict = json.loads(cmd[0]) + cmddict['pgid'] = pgid + else: + cmddict = dict(pgid=pgid) + cmd = [json.dumps(cmddict)] + if verbose: + print >> sys.stderr, 'submit {0} for pgid {1}'.\ + format(cmd, pgid) + ret, outbuf, outs = \ + cluster.pg_command(pgid, cmd, inbuf, timeout) + + elif target[0] == 'mon': + if verbose: + print >> sys.stderr, '{0} to {1}'.\ + format(cmd, target[0]) + if target[1] == '': + ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout) + else: + ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout, target[1]) + + except Exception as e: + raise RuntimeError('"{0}": exception {1}'.format(cmd, e)) + + return ret, outbuf, outs + +def json_command(cluster, target=('mon', ''), prefix=None, argdict=None, + inbuf='', timeout=0, verbose=False): + """ + Format up a JSON command and send it with send_command() above. + Prefix may be supplied separately or in argdict. Any bulk input + data comes in inbuf. + + If target is osd.N, send command to that osd (except for pgid cmds) + """ + cmddict = {} + if prefix: + cmddict.update({'prefix':prefix}) + if argdict: + cmddict.update(argdict) + + # grab prefix for error messages + prefix = cmddict['prefix'] + + try: + if target[0] == 'osd': + osdtarg = CephName() + osdtarget = '{0}.{1}'.format(*target) + # prefer target from cmddict if present and valid + if 'target' in cmddict: + osdtarget = cmddict.pop('target') + try: + osdtarg.valid(osdtarget) + target = ('osd', osdtarg.nameid) + except: + # use the target we were originally given + pass + + ret, outbuf, outs = send_command(cluster, target, [json.dumps(cmddict)], + inbuf, timeout, verbose) + + except Exception as e: + raise RuntimeError('"{0}": exception {1}'.format(prefix, e)) + + return ret, outbuf, outs + + diff --git a/src/pybind/ceph-rest/ceph_rest.py b/src/pybind/ceph-rest/ceph_rest.py new file mode 100755 index 00000000000..75e61060544 --- /dev/null +++ b/src/pybind/ceph-rest/ceph_rest.py @@ -0,0 +1,499 @@ +# vim: ts=4 sw=4 smarttab expandtab + +import errno +import json +import logging +import logging.handlers +import os +import rados +import textwrap +import xml.etree.ElementTree +import xml.sax.saxutils + +import flask +from ceph_argparse import \ + ArgumentError, CephPgid, CephOsdName, CephChoices, CephPrefix, \ + concise_sig, descsort, parse_funcsig, parse_json_funcsigs, \ + validate, json_command + +# +# Globals and defaults +# + +DEFAULT_ADDR = '0.0.0.0' +DEFAULT_PORT = '5000' +DEFAULT_ID = 'restapi' + +DEFAULT_BASEURL = '/api/v0.1' +DEFAULT_LOG_LEVEL = 'warning' +DEFAULT_LOGDIR = '/var/log/ceph' +# default client name will be 'client.' + +# 'app' must be global for decorators, etc. +APPNAME = '__main__' +app = flask.Flask(APPNAME) + +LOGLEVELS = { + 'critical':logging.CRITICAL, + 'error':logging.ERROR, + 'warning':logging.WARNING, + 'info':logging.INFO, + 'debug':logging.DEBUG, +} + +def find_up_osd(app): + ''' + Find an up OSD. Return the last one that's up. + Returns id as an int. + ''' + ret, outbuf, outs = json_command(app.ceph_cluster, prefix="osd dump", + argdict=dict(format='json')) + if ret: + raise EnvironmentError(ret, 'Can\'t get osd dump output') + try: + osddump = json.loads(outbuf) + except: + raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump') + osds = [osd['osd'] for osd in osddump['osds'] if osd['up']] + if not osds: + raise EnvironmentError(errno.ENOENT, 'No up OSDs found') + return int(osds[-1]) + + +METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} + +def api_setup(app, conf, cluster, clientname, clientid, args): + ''' + This is done globally, and cluster connection kept open for + the lifetime of the daemon. librados should assure that even + if the cluster goes away and comes back, our connection remains. + + Initialize the running instance. Open the cluster, get the command + signatures, module, perms, and help; stuff them away in the app.ceph_urls + dict. Also save app.ceph_sigdict for help() handling. + ''' + def get_command_descriptions(cluster, target=('mon','')): + ret, outbuf, outs = json_command(cluster, target, + prefix='get_command_descriptions', + timeout=30) + if ret: + err = "Can't get command descriptions: {0}".format(outs) + app.logger.error(err) + raise EnvironmentError(ret, err) + + try: + sigdict = parse_json_funcsigs(outbuf, 'rest') + except Exception as e: + err = "Can't parse command descriptions: {}".format(e) + app.logger.error(err) + raise EnvironmentError(err) + return sigdict + + app.ceph_cluster = cluster or 'ceph' + app.ceph_urls = {} + app.ceph_sigdict = {} + app.ceph_baseurl = '' + + conf = conf or '' + cluster = cluster or 'ceph' + clientid = clientid or DEFAULT_ID + clientname = clientname or 'client.' + clientid + + app.ceph_cluster = rados.Rados(name=clientname, conffile=conf) + app.ceph_cluster.conf_parse_argv(args) + app.ceph_cluster.connect() + + app.ceph_baseurl = app.ceph_cluster.conf_get('restapi_base_url') \ + or DEFAULT_BASEURL + if app.ceph_baseurl.endswith('/'): + app.ceph_baseurl = app.ceph_baseurl[:-1] + addr = app.ceph_cluster.conf_get('public_addr') or DEFAULT_ADDR + + # remove any nonce from the conf value + addr = addr.split('/')[0] + addr, port = addr.rsplit(':', 1) + addr = addr or DEFAULT_ADDR + port = port or DEFAULT_PORT + port = int(port) + + loglevel = app.ceph_cluster.conf_get('restapi_log_level') \ + or DEFAULT_LOG_LEVEL + # ceph has a default log file for daemons only; clients (like this) + # default to "". Override that for this particular client. + logfile = app.ceph_cluster.conf_get('log_file') + if not logfile: + logfile = os.path.join( + DEFAULT_LOGDIR, + '{cluster}-{clientname}.{pid}.log'.format( + cluster=cluster, + clientname=clientname, + pid=os.getpid() + ) + ) + app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile)) + app.logger.setLevel(LOGLEVELS[loglevel.lower()]) + for h in app.logger.handlers: + h.setFormatter(logging.Formatter( + '%(asctime)s %(name)s %(levelname)s: %(message)s')) + + app.ceph_sigdict = get_command_descriptions(app.ceph_cluster) + + osdid = find_up_osd(app) + if osdid: + osd_sigdict = get_command_descriptions(app.ceph_cluster, + target=('osd', int(osdid))) + + # shift osd_sigdict keys up to fit at the end of the mon's app.ceph_sigdict + maxkey = sorted(app.ceph_sigdict.keys())[-1] + maxkey = int(maxkey.replace('cmd', '')) + osdkey = maxkey + 1 + for k, v in osd_sigdict.iteritems(): + newv = v + newv['flavor'] = 'tell' + globk = 'cmd' + str(osdkey) + app.ceph_sigdict[globk] = newv + osdkey += 1 + + # app.ceph_sigdict maps "cmdNNN" to a dict containing: + # 'sig', an array of argdescs + # 'help', the helptext + # 'module', the Ceph module this command relates to + # 'perm', a 'rwx*' string representing required permissions, and also + # a hint as to whether this is a GET or POST/PUT operation + # 'avail', a comma-separated list of strings of consumers that should + # display this command (filtered by parse_json_funcsigs() above) + app.ceph_urls = {} + for cmdnum, cmddict in app.ceph_sigdict.iteritems(): + cmdsig = cmddict['sig'] + flavor = cmddict.get('flavor', 'mon') + url, params = generate_url_and_params(app, cmdsig, flavor) + perm = cmddict['perm'] + for k in METHOD_DICT.iterkeys(): + if k in perm: + methods = METHOD_DICT[k] + urldict = {'paramsig':params, + 'help':cmddict['help'], + 'module':cmddict['module'], + 'perm':perm, + 'flavor':flavor, + 'methods':methods, + } + + # app.ceph_urls contains a list of urldicts (usually only one long) + if url not in app.ceph_urls: + app.ceph_urls[url] = [urldict] + else: + # If more than one, need to make union of methods of all. + # Method must be checked in handler + methodset = set(methods) + for old_urldict in app.ceph_urls[url]: + methodset |= set(old_urldict['methods']) + methods = list(methodset) + app.ceph_urls[url].append(urldict) + + # add, or re-add, rule with all methods and urldicts + app.add_url_rule(url, url, handler, methods=methods) + url += '.' + app.add_url_rule(url, url, handler, methods=methods) + + app.logger.debug("urls added: %d", len(app.ceph_urls)) + + app.add_url_rule('/', '/', + handler, methods=['GET', 'PUT']) + return addr, port + + +def generate_url_and_params(app, sig, flavor): + ''' + Digest command signature from cluster; generate an absolute + (including app.ceph_baseurl) endpoint from all the prefix words, + and a list of non-prefix param descs + ''' + + url = '' + params = [] + # the OSD command descriptors don't include the 'tell ', so + # tack it onto the front of sig + if flavor == 'tell': + tellsig = parse_funcsig(['tell', + {'name':'target', 'type':'CephOsdName'}]) + sig = tellsig + sig + + for desc in sig: + # prefixes go in the URL path + if desc.t == CephPrefix: + url += '/' + desc.instance.prefix + # CephChoices with 1 required string (not --) do too, unless + # we've already started collecting params, in which case they + # too are params + elif desc.t == CephChoices and \ + len(desc.instance.strings) == 1 and \ + desc.req and \ + not str(desc.instance).startswith('--') and \ + not params: + url += '/' + str(desc.instance) + else: + # tell/ is a weird case; the URL includes what + # would everywhere else be a parameter + if flavor == 'tell' and \ + (desc.t, desc.name) == (CephOsdName, 'target'): + url += '/' + else: + params.append(desc) + + return app.ceph_baseurl + url, params + + +# +# end setup (import-time) functions, begin request-time functions +# + +def concise_sig_for_uri(sig, flavor): + ''' + Return a generic description of how one would send a REST request for sig + ''' + prefix = [] + args = [] + ret = '' + if flavor == 'tell': + ret = 'tell//' + for d in sig: + if d.t == CephPrefix: + prefix.append(d.instance.prefix) + else: + args.append(d.name + '=' + str(d)) + ret += '/'.join(prefix) + if args: + ret += '?' + '&'.join(args) + return ret + +def show_human_help(prefix): + ''' + Dump table showing commands matching prefix + ''' + # XXX There ought to be a better discovery mechanism than an HTML table + s = '' + + permmap = {'r':'GET', 'rw':'PUT'} + line = '' + for cmdsig in sorted(app.ceph_sigdict.itervalues(), cmp=descsort): + concise = concise_sig(cmdsig['sig']) + flavor = cmdsig.get('flavor', 'mon') + if flavor == 'tell': + concise = 'tell//' + concise + if concise.startswith(prefix): + line = ['\n') + s += ''.join(line) + + s += '
Possible commands:MethodDescription
'] + wrapped_sig = textwrap.wrap( + concise_sig_for_uri(cmdsig['sig'], flavor), 40 + ) + for sigline in wrapped_sig: + line.append(flask.escape(sigline) + '\n') + line.append('') + line.append(permmap[cmdsig['perm']]) + line.append('') + line.append(flask.escape(cmdsig['help'])) + line.append('
' + if line: + return s + else: + return '' + +@app.before_request +def log_request(): + ''' + For every request, log it. XXX Probably overkill for production + ''' + app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string) + app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values()) + +@app.route('/') +def root_redir(): + return flask.redirect(app.ceph_baseurl) + +def make_response(fmt, output, statusmsg, errorcode): + ''' + If formatted output, cobble up a response object that contains the + output and status wrapped in enclosing objects; if nonformatted, just + use output+status. Return HTTP status errorcode in any event. + ''' + response = output + if fmt: + if 'json' in fmt: + try: + native_output = json.loads(output or '[]') + response = json.dumps({"output":native_output, + "status":statusmsg}) + except: + return flask.make_response("Error decoding JSON from " + + output, 500) + elif 'xml' in fmt: + # XXX + # one is tempted to do this with xml.etree, but figuring out how + # to 'un-XML' the XML-dumped output so it can be reassembled into + # a piece of the tree here is beyond me right now. + #ET = xml.etree.ElementTree + #resp_elem = ET.Element('response') + #o = ET.SubElement(resp_elem, 'output') + #o.text = output + #s = ET.SubElement(resp_elem, 'status') + #s.text = statusmsg + #response = ET.tostring(resp_elem) + response = ''' + + + {0} + + + {1} + +'''.format(response, xml.sax.saxutils.escape(statusmsg)) + else: + if not 200 <= errorcode < 300: + response = response + '\n' + statusmsg + '\n' + + return flask.make_response(response, errorcode) + +def handler(catchall_path=None, fmt=None, target=None): + ''' + Main endpoint handler; generic for every endpoint, including catchall. + Handles the catchall, anything with <.fmt>, anything with embedded + . Partial match or ?help cause the HTML-table + "show_human_help" output. + ''' + + ep = catchall_path or flask.request.endpoint + ep = ep.replace('.', '') + + if ep[0] != '/': + ep = '/' + ep + + # demand that endpoint begin with app.ceph_baseurl + if not ep.startswith(app.ceph_baseurl): + return make_response(fmt, '', 'Page not found', 404) + + rel_ep = ep[len(app.ceph_baseurl)+1:] + + # Extensions override Accept: headers override defaults + if not fmt: + if 'application/json' in flask.request.accept_mimetypes.values(): + fmt = 'json' + elif 'application/xml' in flask.request.accept_mimetypes.values(): + fmt = 'xml' + + prefix = '' + pgid = None + cmdtarget = 'mon', '' + + if target: + # got tell/; validate osdid or pgid + name = CephOsdName() + pgidobj = CephPgid() + try: + name.valid(target) + except ArgumentError: + # try pgid + try: + pgidobj.valid(target) + except ArgumentError: + return flask.make_response("invalid osdid or pgid", 400) + else: + # it's a pgid + pgid = pgidobj.val + cmdtarget = 'pg', pgid + else: + # it's an osd + cmdtarget = name.nametype, name.nameid + + # prefix does not include tell// + prefix = ' '.join(rel_ep.split('/')[2:]).strip() + else: + # non-target command: prefix is entire path + prefix = ' '.join(rel_ep.split('/')).strip() + + # show "match as much as you gave me" help for unknown endpoints + if not ep in app.ceph_urls: + helptext = show_human_help(prefix) + if helptext: + resp = flask.make_response(helptext, 400) + resp.headers['Content-Type'] = 'text/html' + return resp + else: + return make_response(fmt, '', 'Invalid endpoint ' + ep, 400) + + found = None + exc = '' + for urldict in app.ceph_urls[ep]: + if flask.request.method not in urldict['methods']: + continue + paramsig = urldict['paramsig'] + + # allow '?help' for any specifically-known endpoint + if 'help' in flask.request.args: + response = flask.make_response('{0}: {1}'.\ + format(prefix + concise_sig(paramsig), urldict['help'])) + response.headers['Content-Type'] = 'text/plain' + return response + + # if there are parameters for this endpoint, process them + if paramsig: + args = {} + for k, l in flask.request.args.iterlists(): + if len(l) == 1: + args[k] = l[0] + else: + args[k] = l + + # is this a valid set of params? + try: + argdict = validate(args, paramsig) + found = urldict + break + except Exception as e: + exc += str(e) + continue + else: + if flask.request.args: + continue + found = urldict + argdict = {} + break + + if not found: + return make_response(fmt, '', exc + '\n', 400) + + argdict['format'] = fmt or 'plain' + argdict['module'] = found['module'] + argdict['perm'] = found['perm'] + if pgid: + argdict['pgid'] = pgid + + if not cmdtarget: + cmdtarget = ('mon', '') + + app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) + ret, outbuf, outs = json_command(app.ceph_cluster, prefix=prefix, + target=cmdtarget, + inbuf=flask.request.data, argdict=argdict) + if ret: + return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400) + + response = make_response(fmt, outbuf, outs or 'OK', 200) + if fmt: + contenttype = 'application/' + fmt.replace('-pretty','') + else: + contenttype = 'text/plain' + response.headers['Content-Type'] = contenttype + return response + +# +# Main entry point from wrapper/WSGI server: call with cmdline args, +# get back the WSGI app entry point +# +def generate_app(conf, cluster, clientname, clientid, args): + addr, port = api_setup(app, conf, cluster, clientname, clientid, args) + app.ceph_addr = addr + app.ceph_port = port + return app diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py deleted file mode 100644 index 1f6e90b6c1d..00000000000 --- a/src/pybind/ceph_argparse.py +++ /dev/null @@ -1,1110 +0,0 @@ -""" -Types and routines used by the ceph CLI as well as the RESTful -interface. These have to do with querying the daemons for -command-description information, validating user command input against -those descriptions, and submitting the command to the appropriate -daemon. - -Copyright (C) 2013 Inktank Storage, Inc. - -LGPL2. See file COPYING. -""" -import copy -import json -import os -import pprint -import re -import socket -import stat -import sys -import types -import uuid - -class ArgumentError(Exception): - """ - Something wrong with arguments - """ - pass - -class ArgumentNumber(ArgumentError): - """ - Wrong number of a repeated argument - """ - pass - -class ArgumentFormat(ArgumentError): - """ - Argument value has wrong format - """ - pass - -class ArgumentValid(ArgumentError): - """ - Argument value is otherwise invalid (doesn't match choices, for instance) - """ - pass - -class ArgumentTooFew(ArgumentError): - """ - Fewer arguments than descriptors in signature; may mean to continue - the search, so gets a special exception type - """ - -class ArgumentPrefix(ArgumentError): - """ - Special for mismatched prefix; less severe, don't report by default - """ - pass - -class JsonFormat(Exception): - """ - some syntactic or semantic issue with the JSON - """ - pass - -class CephArgtype(object): - """ - Base class for all Ceph argument types - - Instantiating an object sets any validation parameters - (allowable strings, numeric ranges, etc.). The 'valid' - method validates a string against that initialized instance, - throwing ArgumentError if there's a problem. - """ - def __init__(self, **kwargs): - """ - set any per-instance validation parameters here - from kwargs (fixed string sets, integer ranges, etc) - """ - pass - - def valid(self, s, partial=False): - """ - Run validation against given string s (generally one word); - partial means to accept partial string matches (begins-with). - If cool, set self.val to the value that should be returned - (a copy of the input string, or a numeric or boolean interpretation - thereof, for example) - if not, throw ArgumentError(msg-as-to-why) - """ - self.val = s - - def __repr__(self): - """ - return string representation of description of type. Note, - this is not a representation of the actual value. Subclasses - probably also override __str__() to give a more user-friendly - 'name/type' description for use in command format help messages. - """ - a = '' - if hasattr(self, 'typeargs'): - a = self.typeargs - return '{0}(\'{1}\')'.format(self.__class__.__name__, a) - - def __str__(self): - """ - where __repr__ (ideally) returns a string that could be used to - reproduce the object, __str__ returns one you'd like to see in - print messages. Use __str__ to format the argtype descriptor - as it would be useful in a command usage message. - """ - return '<{0}>'.format(self.__class__.__name__) - -class CephInt(CephArgtype): - """ - range-limited integers, [+|-][0-9]+ or 0x[0-9a-f]+ - range: list of 1 or 2 ints, [min] or [min,max] - """ - def __init__(self, range=''): - if range == '': - self.range = list() - else: - self.range = list(range.split('|')) - self.range = map(long, self.range) - - def valid(self, s, partial=False): - try: - val = long(s) - except ValueError: - raise ArgumentValid("{0} doesn't represent an int".format(s)) - if len(self.range) == 2: - if val < self.range[0] or val > self.range[1]: - raise ArgumentValid("{0} not in range {1}".format(val, self.range)) - elif len(self.range) == 1: - if val < self.range[0]: - raise ArgumentValid("{0} not in range {1}".format(val, self.range)) - self.val = val - - def __str__(self): - r = '' - if len(self.range) == 1: - r = '[{0}-]'.format(self.range[0]) - if len(self.range) == 2: - r = '[{0}-{1}]'.format(self.range[0], self.range[1]) - - return ''.format(r) - - -class CephFloat(CephArgtype): - """ - range-limited float type - range: list of 1 or 2 floats, [min] or [min, max] - """ - def __init__(self, range=''): - if range == '': - self.range = list() - else: - self.range = list(range.split('|')) - self.range = map(float, self.range) - - def valid(self, s, partial=False): - try: - val = float(s) - except ValueError: - raise ArgumentValid("{0} doesn't represent a float".format(s)) - if len(self.range) == 2: - if val < self.range[0] or val > self.range[1]: - raise ArgumentValid("{0} not in range {1}".format(val, self.range)) - elif len(self.range) == 1: - if val < self.range[0]: - raise ArgumentValid("{0} not in range {1}".format(val, self.range)) - self.val = val - - def __str__(self): - r = '' - if len(self.range) == 1: - r = '[{0}-]'.format(self.range[0]) - if len(self.range) == 2: - r = '[{0}-{1}]'.format(self.range[0], self.range[1]) - return ''.format(r) - -class CephString(CephArgtype): - """ - String; pretty generic. goodchars is a RE char class of valid chars - """ - def __init__(self, goodchars=''): - from string import printable - try: - re.compile(goodchars) - except: - raise ValueError('CephString(): "{0}" is not a valid RE'.\ - format(goodchars)) - self.goodchars = goodchars - self.goodset = frozenset( - [c for c in printable if re.match(goodchars, c)] - ) - - def valid(self, s, partial=False): - sset = set(s) - if self.goodset and not sset <= self.goodset: - raise ArgumentFormat("invalid chars {0} in {1}".\ - format(''.join(sset - self.goodset), s)) - self.val = s - - def __str__(self): - b = '' - if self.goodchars: - b += '(goodchars {0})'.format(self.goodchars) - return ''.format(b) - -class CephSocketpath(CephArgtype): - """ - Admin socket path; check that it's readable and S_ISSOCK - """ - def valid(self, s, partial=False): - mode = os.stat(s).st_mode - if not stat.S_ISSOCK(mode): - raise ArgumentValid('socket path {0} is not a socket'.format(s)) - self.val = s - - def __str__(self): - return '' - -class CephIPAddr(CephArgtype): - """ - IP address (v4 or v6) with optional port - """ - def valid(self, s, partial=False): - # parse off port, use socket to validate addr - type = 6 - if s.startswith('['): - type = 6 - elif s.find('.') != -1: - type = 4 - if type == 4: - port = s.find(':') - if (port != -1): - a = s[:port] - p = s[port+1:] - if int(p) > 65535: - raise ArgumentValid('{0}: invalid IPv4 port'.format(p)) - else: - a = s - p = None - try: - socket.inet_pton(socket.AF_INET, a) - except: - raise ArgumentValid('{0}: invalid IPv4 address'.format(a)) - else: - # v6 - if s.startswith('['): - end = s.find(']') - if end == -1: - raise ArgumentFormat('{0} missing terminating ]'.format(s)) - if s[end+1] == ':': - try: - p = int(s[end+2]) - except: - raise ArgumentValid('{0}: bad port number'.format(s)) - a = s[1:end] - else: - a = s - p = None - try: - socket.inet_pton(socket.AF_INET6, a) - except: - raise ArgumentValid('{0} not valid IPv6 address'.format(s)) - if p is not None and long(p) > 65535: - raise ArgumentValid("{0} not a valid port number".format(p)) - self.val = s - self.addr = a - self.port = p - - def __str__(self): - return '' - -class CephEntityAddr(CephIPAddr): - """ - EntityAddress, that is, IP address[/nonce] - """ - def valid(self, s, partial=False): - nonce = None - if '/' in s: - ip, nonce = s.split('/') - else: - ip = s - super(self.__class__, self).valid(ip) - if nonce: - nonce_long = None - try: - nonce_long = long(nonce) - except ValueError: - pass - if nonce_long is None or nonce_long < 0: - raise ArgumentValid( - '{0}: invalid entity, nonce {1} not integer > 0'.\ - format(s, nonce) - ) - self.val = s - - def __str__(self): - return '' - -class CephPoolname(CephArgtype): - """ - Pool name; very little utility - """ - def __str__(self): - return '' - -class CephObjectname(CephArgtype): - """ - Object name. Maybe should be combined with Pool name as they're always - present in pairs, and then could be checked for presence - """ - def __str__(self): - return '' - -class CephPgid(CephArgtype): - """ - pgid, in form N.xxx (N = pool number, xxx = hex pgnum) - """ - def valid(self, s, partial=False): - if s.find('.') == -1: - raise ArgumentFormat('pgid has no .') - poolid, pgnum = s.split('.') - if poolid < 0: - raise ArgumentFormat('pool {0} < 0'.format(poolid)) - try: - pgnum = int(pgnum, 16) - except: - raise ArgumentFormat('pgnum {0} not hex integer'.format(pgnum)) - self.val = s - - def __str__(self): - return '' - -class CephName(CephArgtype): - """ - Name (type.id) where: - type is osd|mon|client|mds - id is a base10 int, if type == osd, or a string otherwise - - Also accept '*' - """ - def __init__(self): - self.nametype = None - self.nameid = None - - def valid(self, s, partial=False): - if s == '*': - self.val = s - return - if s.find('.') == -1: - raise ArgumentFormat('CephName: no . in {0}'.format(s)) - else: - t, i = s.split('.') - if not t in ('osd', 'mon', 'client', 'mds'): - raise ArgumentValid('unknown type ' + t) - if t == 'osd': - if i != '*': - try: - i = int(i) - except: - raise ArgumentFormat('osd id ' + i + ' not integer') - self.nametype = t - self.val = s - self.nameid = i - - def __str__(self): - return '' - -class CephOsdName(CephArgtype): - """ - Like CephName, but specific to osds: allow alone - - osd., or , or *, where id is a base10 int - """ - def __init__(self): - self.nametype = None - self.nameid = None - - def valid(self, s, partial=False): - if s == '*': - self.val = s - return - if s.find('.') != -1: - t, i = s.split('.') - if t != 'osd': - raise ArgumentValid('unknown type ' + t) - else: - t = 'osd' - i = s - try: - i = int(i) - except: - raise ArgumentFormat('osd id ' + i + ' not integer') - self.nametype = t - self.nameid = i - self.val = i - - def __str__(self): - return '' - -class CephChoices(CephArgtype): - """ - Set of string literals; init with valid choices - """ - def __init__(self, strings='', **kwargs): - self.strings = strings.split('|') - - def valid(self, s, partial=False): - if not partial: - if not s in self.strings: - # show as __str__ does: {s1|s2..} - raise ArgumentValid("{0} not in {1}".format(s, self)) - self.val = s - return - - # partial - for t in self.strings: - if t.startswith(s): - self.val = s - return - raise ArgumentValid("{0} not in {1}". format(s, self)) - - def __str__(self): - if len(self.strings) == 1: - return '{0}'.format(self.strings[0]) - else: - return '{0}'.format('|'.join(self.strings)) - -class CephFilepath(CephArgtype): - """ - Openable file - """ - def valid(self, s, partial=False): - try: - f = open(s, 'a+') - except Exception as e: - raise ArgumentValid('can\'t open {0}: {1}'.format(s, e)) - f.close() - self.val = s - - def __str__(self): - return '' - -class CephFragment(CephArgtype): - """ - 'Fragment' ??? XXX - """ - def valid(self, s, partial=False): - if s.find('/') == -1: - raise ArgumentFormat('{0}: no /'.format(s)) - val, bits = s.split('/') - # XXX is this right? - if not val.startswith('0x'): - raise ArgumentFormat("{0} not a hex integer".format(val)) - try: - long(val) - except: - raise ArgumentFormat('can\'t convert {0} to integer'.format(val)) - try: - long(bits) - except: - raise ArgumentFormat('can\'t convert {0} to integer'.format(bits)) - self.val = s - - def __str__(self): - return "" - - -class CephUUID(CephArgtype): - """ - CephUUID: pretty self-explanatory - """ - def valid(self, s, partial=False): - try: - uuid.UUID(s) - except Exception as e: - raise ArgumentFormat('invalid UUID {0}: {1}'.format(s, e)) - self.val = s - - def __str__(self): - return '' - - -class CephPrefix(CephArgtype): - """ - CephPrefix: magic type for "all the first n fixed strings" - """ - def __init__(self, prefix=''): - self.prefix = prefix - - def valid(self, s, partial=False): - if partial: - if self.prefix.startswith(s): - self.val = s - return - else: - if (s == self.prefix): - self.val = s - return - - raise ArgumentPrefix("no match for {0}".format(s)) - - def __str__(self): - return self.prefix - - -class argdesc(object): - """ - argdesc(typename, name='name', n=numallowed|N, - req=False, helptext=helptext, **kwargs (type-specific)) - - validation rules: - typename: type(**kwargs) will be constructed - later, type.valid(w) will be called with a word in that position - - name is used for parse errors and for constructing JSON output - n is a numeric literal or 'n|N', meaning "at least one, but maybe more" - req=False means the argument need not be present in the list - helptext is the associated help for the command - anything else are arguments to pass to the type constructor. - - self.instance is an instance of type t constructed with typeargs. - - valid() will later be called with input to validate against it, - and will store the validated value in self.instance.val for extraction. - """ - def __init__(self, t, name=None, n=1, req=True, **kwargs): - if isinstance(t, types.StringTypes): - self.t = CephPrefix - self.typeargs = {'prefix':t} - self.req = True - else: - self.t = t - self.typeargs = kwargs - self.req = bool(req == True or req == 'True') - - self.name = name - self.N = (n in ['n', 'N']) - if self.N: - self.n = 1 - else: - self.n = int(n) - self.instance = self.t(**self.typeargs) - - def __repr__(self): - r = 'argdesc(' + str(self.t) + ', ' - internals = ['N', 'typeargs', 'instance', 't'] - for (k, v) in self.__dict__.iteritems(): - if k.startswith('__') or k in internals: - pass - else: - # undo modification from __init__ - if k == 'n' and self.N: - v = 'N' - r += '{0}={1}, '.format(k, v) - for (k, v) in self.typeargs.iteritems(): - r += '{0}={1}, '.format(k, v) - return r[:-2] + ')' - - def __str__(self): - if ((self.t == CephChoices and len(self.instance.strings) == 1) - or (self.t == CephPrefix)): - s = str(self.instance) - else: - s = '{0}({1})'.format(self.name, str(self.instance)) - if self.N: - s += ' [' + str(self.instance) + '...]' - if not self.req: - s = '{' + s + '}' - return s - - def helpstr(self): - """ - like str(), but omit parameter names (except for CephString, - which really needs them) - """ - if self.t == CephString: - chunk = '<{0}>'.format(self.name) - else: - chunk = str(self.instance) - s = chunk - if self.N: - s += ' [' + chunk + '...]' - if not self.req: - s = '{' + s + '}' - return s - -def concise_sig(sig): - """ - Return string representation of sig useful for syntax reference in help - """ - return ' '.join([d.helpstr() for d in sig]) - -def descsort(sh1, sh2): - """ - sort descriptors by prefixes, defined as the concatenation of all simple - strings in the descriptor; this works out to just the leading strings. - """ - return cmp(concise_sig(sh1['sig']), concise_sig(sh2['sig'])) - -def parse_funcsig(sig): - """ - parse a single descriptor (array of strings or dicts) into a - dict of function descriptor/validators (objects of CephXXX type) - """ - newsig = [] - argnum = 0 - for desc in sig: - argnum += 1 - if isinstance(desc, types.StringTypes): - t = CephPrefix - desc = {'type':t, 'name':'prefix', 'prefix':desc} - else: - # not a simple string, must be dict - if not 'type' in desc: - s = 'JSON descriptor {0} has no type'.format(sig) - raise JsonFormat(s) - # look up type string in our globals() dict; if it's an - # object of type types.TypeType, it must be a - # locally-defined class. otherwise, we haven't a clue. - if desc['type'] in globals(): - t = globals()[desc['type']] - if type(t) != types.TypeType: - s = 'unknown type {0}'.format(desc['type']) - raise JsonFormat(s) - else: - s = 'unknown type {0}'.format(desc['type']) - raise JsonFormat(s) - - kwargs = dict() - for key, val in desc.items(): - if key not in ['type', 'name', 'n', 'req']: - kwargs[key] = val - newsig.append(argdesc(t, - name=desc.get('name', None), - n=desc.get('n', 1), - req=desc.get('req', True), - **kwargs)) - return newsig - - -def parse_json_funcsigs(s, consumer): - """ - A function signature is mostly an array of argdesc; it's represented - in JSON as - { - "cmd001": {"sig":[ "type": type, "name": name, "n": num, "req":true|false ], "help":helptext, "module":modulename, "perm":perms, "avail":availability} - . - . - . - ] - - A set of sigs is in an dict mapped by a unique number: - { - "cmd1": { - "sig": ["type.. ], "help":helptext... - } - "cmd2"{ - "sig": [.. ], "help":helptext... - } - } - - Parse the string s and return a dict of dicts, keyed by opcode; - each dict contains 'sig' with the array of descriptors, and 'help' - with the helptext, 'module' with the module name, 'perm' with a - string representing required permissions in that module to execute - this command (and also whether it is a read or write command from - the cluster state perspective), and 'avail' as a hint for - whether the command should be advertised by CLI, REST, or both. - If avail does not contain 'consumer', don't include the command - in the returned dict. - """ - try: - overall = json.loads(s) - except Exception as e: - print >> sys.stderr, "Couldn't parse JSON {0}: {1}".format(s, e) - raise e - sigdict = {} - for cmdtag, cmd in overall.iteritems(): - if not 'sig' in cmd: - s = "JSON descriptor {0} has no 'sig'".format(cmdtag) - raise JsonFormat(s) - # check 'avail' and possibly ignore this command - if 'avail' in cmd: - if not consumer in cmd['avail']: - continue - # rewrite the 'sig' item with the argdesc-ized version, and... - cmd['sig'] = parse_funcsig(cmd['sig']) - # just take everything else as given - sigdict[cmdtag] = cmd - return sigdict - -def validate_one(word, desc, partial=False): - """ - validate_one(word, desc, partial=False) - - validate word against the constructed instance of the type - in desc. May raise exception. If it returns false (and doesn't - raise an exception), desc.instance.val will - contain the validated value (in the appropriate type). - """ - desc.instance.valid(word, partial) - desc.numseen += 1 - if desc.N: - desc.n = desc.numseen + 1 - -def matchnum(args, signature, partial=False): - """ - matchnum(s, signature, partial=False) - - Returns number of arguments matched in s against signature. - Can be used to determine most-likely command for full or partial - matches (partial applies to string matches). - """ - words = args[:] - mysig = copy.deepcopy(signature) - matchcnt = 0 - for desc in mysig: - setattr(desc, 'numseen', 0) - while desc.numseen < desc.n: - # if there are no more arguments, return - if not words: - return matchcnt - word = words.pop(0) - - try: - validate_one(word, desc, partial) - valid = True - except ArgumentError: - # matchnum doesn't care about type of error - valid = False - - if not valid: - if not desc.req: - # this wasn't required, so word may match the next desc - words.insert(0, word) - break - else: - # it was required, and didn't match, return - return matchcnt - if desc.req: - matchcnt += 1 - return matchcnt - -def get_next_arg(desc, args): - ''' - Get either the value matching key 'desc.name' or the next arg in - the non-dict list. Return None if args are exhausted. Used in - validate() below. - ''' - arg = None - if isinstance(args, dict): - arg = args.pop(desc.name, None) - # allow 'param=param' to be expressed as 'param' - if arg == '': - arg = desc.name - # Hack, or clever? If value is a list, keep the first element, - # push rest back onto myargs for later processing. - # Could process list directly, but nesting here is already bad - if arg and isinstance(arg, list): - args[desc.name] = arg[1:] - arg = arg[0] - elif args: - arg = args.pop(0) - if arg and isinstance(arg, list): - args = arg[1:] + args - arg = arg[0] - return arg - -def store_arg(desc, d): - ''' - Store argument described by, and held in, thanks to valid(), - desc into the dictionary d, keyed by desc.name. Three cases: - - 1) desc.N is set: value in d is a list - 2) prefix: multiple args are joined with ' ' into one d{} item - 3) single prefix or other arg: store as simple value - - Used in validate() below. - ''' - if desc.N: - # value should be a list - if desc.name in d: - d[desc.name] += [desc.instance.val] - else: - d[desc.name] = [desc.instance.val] - elif (desc.t == CephPrefix) and (desc.name in d): - # prefixes' values should be a space-joined concatenation - d[desc.name] += ' ' + desc.instance.val - else: - # if first CephPrefix or any other type, just set it - d[desc.name] = desc.instance.val - -def validate(args, signature, partial=False): - """ - validate(args, signature, partial=False) - - args is a list of either words or k,v pairs representing a possible - command input following format of signature. Runs a validation; no - exception means it's OK. Return a dict containing all arguments keyed - by their descriptor name, with duplicate args per name accumulated - into a list (or space-separated value for CephPrefix). - - Mismatches of prefix are non-fatal, as this probably just means the - search hasn't hit the correct command. Mismatches of non-prefix - arguments are treated as fatal, and an exception raised. - - This matching is modified if partial is set: allow partial matching - (with partial dict returned); in this case, there are no exceptions - raised. - """ - - myargs = copy.deepcopy(args) - mysig = copy.deepcopy(signature) - reqsiglen = len([desc for desc in mysig if desc.req]) - matchcnt = 0 - d = dict() - for desc in mysig: - setattr(desc, 'numseen', 0) - while desc.numseen < desc.n: - myarg = get_next_arg(desc, myargs) - - # no arg, but not required? Continue consuming mysig - # in case there are later required args - if not myarg and not desc.req: - break - - # out of arguments for a required param? - # Either return (if partial validation) or raise - if not myarg and desc.req: - if desc.N and desc.numseen < 1: - # wanted N, didn't even get 1 - if partial: - return d - raise ArgumentNumber( - 'saw {0} of {1}, expected at least 1'.\ - format(desc.numseen, desc) - ) - elif not desc.N and desc.numseen < desc.n: - # wanted n, got too few - if partial: - return d - # special-case the "0 expected 1" case - if desc.numseen == 0 and desc.n == 1: - raise ArgumentNumber( - 'missing required parameter {0}'.format(desc) - ) - raise ArgumentNumber( - 'saw {0} of {1}, expected {2}'.\ - format(desc.numseen, desc, desc.n) - ) - break - - # Have an arg; validate it - try: - validate_one(myarg, desc) - valid = True - except ArgumentError as e: - valid = False - if not valid: - # argument mismatch - if not desc.req: - # if not required, just push back; it might match - # the next arg - print >> sys.stderr, myarg, 'not valid: ', str(e) - myargs.insert(0, myarg) - break - else: - # hm, it was required, so time to return/raise - if partial: - return d - raise e - - # Whew, valid arg acquired. Store in dict - matchcnt += 1 - store_arg(desc, d) - - # Done with entire list of argdescs - if matchcnt < reqsiglen: - raise ArgumentTooFew("not enough arguments given") - - if myargs and not partial: - raise ArgumentError("unused arguments: " + str(myargs)) - - # Finally, success - return d - -def cmdsiglen(sig): - sigdict = sig.values() - assert len(sigdict) == 1 - return len(sig.values()[0]['sig']) - -def validate_command(sigdict, args, verbose=False): - """ - turn args into a valid dictionary ready to be sent off as JSON, - validated against sigdict. - """ - found = [] - valid_dict = {} - if args: - # look for best match, accumulate possibles in bestcmds - # (so we can maybe give a more-useful error message) - best_match_cnt = 0 - bestcmds = [] - for cmdtag, cmd in sigdict.iteritems(): - sig = cmd['sig'] - matched = matchnum(args, sig, partial=True) - if (matched > best_match_cnt): - if verbose: - print >> sys.stderr, \ - "better match: {0} > {1}: {2}:{3} ".format(matched, - best_match_cnt, cmdtag, concise_sig(sig)) - best_match_cnt = matched - bestcmds = [{cmdtag:cmd}] - elif matched == best_match_cnt: - if verbose: - print >> sys.stderr, \ - "equal match: {0} > {1}: {2}:{3} ".format(matched, - best_match_cnt, cmdtag, concise_sig(sig)) - bestcmds.append({cmdtag:cmd}) - - # Sort bestcmds by number of args so we can try shortest first - # (relies on a cmdsig being key,val where val is a list of len 1) - bestcmds_sorted = sorted(bestcmds, - cmp=lambda x,y:cmp(cmdsiglen(x), cmdsiglen(y))) - - if verbose: - print >> sys.stderr, "bestcmds_sorted: " - pprint.PrettyPrinter(stream=sys.stderr).pprint(bestcmds_sorted) - - # for everything in bestcmds, look for a true match - for cmdsig in bestcmds_sorted: - for cmd in cmdsig.itervalues(): - sig = cmd['sig'] - try: - valid_dict = validate(args, sig) - found = cmd - break - except ArgumentPrefix: - # ignore prefix mismatches; we just haven't found - # the right command yet - pass - except ArgumentTooFew: - # It looked like this matched the beginning, but it - # didn't have enough args supplied. If we're out of - # cmdsigs we'll fall out unfound; if we're not, maybe - # the next one matches completely. Whine, but pass. - if verbose: - print >> sys.stderr, 'Not enough args supplied for ', \ - concise_sig(sig) - except ArgumentError as e: - # Solid mismatch on an arg (type, range, etc.) - # Stop now, because we have the right command but - # some other input is invalid - print >> sys.stderr, "Invalid command: ", str(e) - print >> sys.stderr, concise_sig(sig), ': ', cmd['help'] - return {} - if found: - break - - if not found: - print >> sys.stderr, 'no valid command found; 10 closest matches:' - for cmdsig in bestcmds[:10]: - for (cmdtag, cmd) in cmdsig.iteritems(): - print >> sys.stderr, concise_sig(cmd['sig']) - return None - - return valid_dict - -def find_cmd_target(childargs): - """ - Using a minimal validation, figure out whether the command - should be sent to a monitor or an osd. We do this before even - asking for the 'real' set of command signatures, so we can ask the - right daemon. - Returns ('osd', osdid), ('pg', pgid), or ('mon', '') - """ - sig = parse_funcsig(['tell', {'name':'target', 'type':'CephName'}]) - try: - valid_dict = validate(childargs, sig, partial=True) - except ArgumentError: - pass - else: - if len(valid_dict) == 2: - # revalidate to isolate type and id - name = CephName() - # if this fails, something is horribly wrong, as it just - # validated successfully above - name.valid(valid_dict['target']) - return name.nametype, name.nameid - - sig = parse_funcsig(['tell', {'name':'pgid', 'type':'CephPgid'}]) - try: - valid_dict = validate(childargs, sig, partial=True) - except ArgumentError: - pass - else: - if len(valid_dict) == 2: - # pg doesn't need revalidation; the string is fine - return 'pg', valid_dict['pgid'] - - sig = parse_funcsig(['pg', {'name':'pgid', 'type':'CephPgid'}]) - try: - valid_dict = validate(childargs, sig, partial=True) - except ArgumentError: - pass - else: - if len(valid_dict) == 2: - return 'pg', valid_dict['pgid'] - - return 'mon', '' - -def send_command(cluster, target=('mon', ''), cmd=None, inbuf='', timeout=0, - verbose=False): - """ - Send a command to a daemon using librados's - mon_command, osd_command, or pg_command. Any bulk input data - comes in inbuf. - - Returns (ret, outbuf, outs); ret is the return code, outbuf is - the outbl "bulk useful output" buffer, and outs is any status - or error message (intended for stderr). - - If target is osd.N, send command to that osd (except for pgid cmds) - """ - cmd = cmd or [] - try: - if target[0] == 'osd': - osdid = target[1] - - if verbose: - print >> sys.stderr, 'submit {0} to osd.{1}'.\ - format(cmd, osdid) - ret, outbuf, outs = \ - cluster.osd_command(osdid, cmd, inbuf, timeout) - - elif target[0] == 'pg': - pgid = target[1] - # pgid will already be in the command for the pg - # form, but for tell , we need to put it in - if cmd: - cmddict = json.loads(cmd[0]) - cmddict['pgid'] = pgid - else: - cmddict = dict(pgid=pgid) - cmd = [json.dumps(cmddict)] - if verbose: - print >> sys.stderr, 'submit {0} for pgid {1}'.\ - format(cmd, pgid) - ret, outbuf, outs = \ - cluster.pg_command(pgid, cmd, inbuf, timeout) - - elif target[0] == 'mon': - if verbose: - print >> sys.stderr, '{0} to {1}'.\ - format(cmd, target[0]) - if target[1] == '': - ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout) - else: - ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout, target[1]) - - except Exception as e: - raise RuntimeError('"{0}": exception {1}'.format(cmd, e)) - - return ret, outbuf, outs - -def json_command(cluster, target=('mon', ''), prefix=None, argdict=None, - inbuf='', timeout=0, verbose=False): - """ - Format up a JSON command and send it with send_command() above. - Prefix may be supplied separately or in argdict. Any bulk input - data comes in inbuf. - - If target is osd.N, send command to that osd (except for pgid cmds) - """ - cmddict = {} - if prefix: - cmddict.update({'prefix':prefix}) - if argdict: - cmddict.update(argdict) - - # grab prefix for error messages - prefix = cmddict['prefix'] - - try: - if target[0] == 'osd': - osdtarg = CephName() - osdtarget = '{0}.{1}'.format(*target) - # prefer target from cmddict if present and valid - if 'target' in cmddict: - osdtarget = cmddict.pop('target') - try: - osdtarg.valid(osdtarget) - target = ('osd', osdtarg.nameid) - except: - # use the target we were originally given - pass - - ret, outbuf, outs = send_command(cluster, target, [json.dumps(cmddict)], - inbuf, timeout, verbose) - - except Exception as e: - raise RuntimeError('"{0}": exception {1}'.format(prefix, e)) - - return ret, outbuf, outs - - diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py deleted file mode 100755 index 75e61060544..00000000000 --- a/src/pybind/ceph_rest_api.py +++ /dev/null @@ -1,499 +0,0 @@ -# vim: ts=4 sw=4 smarttab expandtab - -import errno -import json -import logging -import logging.handlers -import os -import rados -import textwrap -import xml.etree.ElementTree -import xml.sax.saxutils - -import flask -from ceph_argparse import \ - ArgumentError, CephPgid, CephOsdName, CephChoices, CephPrefix, \ - concise_sig, descsort, parse_funcsig, parse_json_funcsigs, \ - validate, json_command - -# -# Globals and defaults -# - -DEFAULT_ADDR = '0.0.0.0' -DEFAULT_PORT = '5000' -DEFAULT_ID = 'restapi' - -DEFAULT_BASEURL = '/api/v0.1' -DEFAULT_LOG_LEVEL = 'warning' -DEFAULT_LOGDIR = '/var/log/ceph' -# default client name will be 'client.' - -# 'app' must be global for decorators, etc. -APPNAME = '__main__' -app = flask.Flask(APPNAME) - -LOGLEVELS = { - 'critical':logging.CRITICAL, - 'error':logging.ERROR, - 'warning':logging.WARNING, - 'info':logging.INFO, - 'debug':logging.DEBUG, -} - -def find_up_osd(app): - ''' - Find an up OSD. Return the last one that's up. - Returns id as an int. - ''' - ret, outbuf, outs = json_command(app.ceph_cluster, prefix="osd dump", - argdict=dict(format='json')) - if ret: - raise EnvironmentError(ret, 'Can\'t get osd dump output') - try: - osddump = json.loads(outbuf) - except: - raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump') - osds = [osd['osd'] for osd in osddump['osds'] if osd['up']] - if not osds: - raise EnvironmentError(errno.ENOENT, 'No up OSDs found') - return int(osds[-1]) - - -METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} - -def api_setup(app, conf, cluster, clientname, clientid, args): - ''' - This is done globally, and cluster connection kept open for - the lifetime of the daemon. librados should assure that even - if the cluster goes away and comes back, our connection remains. - - Initialize the running instance. Open the cluster, get the command - signatures, module, perms, and help; stuff them away in the app.ceph_urls - dict. Also save app.ceph_sigdict for help() handling. - ''' - def get_command_descriptions(cluster, target=('mon','')): - ret, outbuf, outs = json_command(cluster, target, - prefix='get_command_descriptions', - timeout=30) - if ret: - err = "Can't get command descriptions: {0}".format(outs) - app.logger.error(err) - raise EnvironmentError(ret, err) - - try: - sigdict = parse_json_funcsigs(outbuf, 'rest') - except Exception as e: - err = "Can't parse command descriptions: {}".format(e) - app.logger.error(err) - raise EnvironmentError(err) - return sigdict - - app.ceph_cluster = cluster or 'ceph' - app.ceph_urls = {} - app.ceph_sigdict = {} - app.ceph_baseurl = '' - - conf = conf or '' - cluster = cluster or 'ceph' - clientid = clientid or DEFAULT_ID - clientname = clientname or 'client.' + clientid - - app.ceph_cluster = rados.Rados(name=clientname, conffile=conf) - app.ceph_cluster.conf_parse_argv(args) - app.ceph_cluster.connect() - - app.ceph_baseurl = app.ceph_cluster.conf_get('restapi_base_url') \ - or DEFAULT_BASEURL - if app.ceph_baseurl.endswith('/'): - app.ceph_baseurl = app.ceph_baseurl[:-1] - addr = app.ceph_cluster.conf_get('public_addr') or DEFAULT_ADDR - - # remove any nonce from the conf value - addr = addr.split('/')[0] - addr, port = addr.rsplit(':', 1) - addr = addr or DEFAULT_ADDR - port = port or DEFAULT_PORT - port = int(port) - - loglevel = app.ceph_cluster.conf_get('restapi_log_level') \ - or DEFAULT_LOG_LEVEL - # ceph has a default log file for daemons only; clients (like this) - # default to "". Override that for this particular client. - logfile = app.ceph_cluster.conf_get('log_file') - if not logfile: - logfile = os.path.join( - DEFAULT_LOGDIR, - '{cluster}-{clientname}.{pid}.log'.format( - cluster=cluster, - clientname=clientname, - pid=os.getpid() - ) - ) - app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile)) - app.logger.setLevel(LOGLEVELS[loglevel.lower()]) - for h in app.logger.handlers: - h.setFormatter(logging.Formatter( - '%(asctime)s %(name)s %(levelname)s: %(message)s')) - - app.ceph_sigdict = get_command_descriptions(app.ceph_cluster) - - osdid = find_up_osd(app) - if osdid: - osd_sigdict = get_command_descriptions(app.ceph_cluster, - target=('osd', int(osdid))) - - # shift osd_sigdict keys up to fit at the end of the mon's app.ceph_sigdict - maxkey = sorted(app.ceph_sigdict.keys())[-1] - maxkey = int(maxkey.replace('cmd', '')) - osdkey = maxkey + 1 - for k, v in osd_sigdict.iteritems(): - newv = v - newv['flavor'] = 'tell' - globk = 'cmd' + str(osdkey) - app.ceph_sigdict[globk] = newv - osdkey += 1 - - # app.ceph_sigdict maps "cmdNNN" to a dict containing: - # 'sig', an array of argdescs - # 'help', the helptext - # 'module', the Ceph module this command relates to - # 'perm', a 'rwx*' string representing required permissions, and also - # a hint as to whether this is a GET or POST/PUT operation - # 'avail', a comma-separated list of strings of consumers that should - # display this command (filtered by parse_json_funcsigs() above) - app.ceph_urls = {} - for cmdnum, cmddict in app.ceph_sigdict.iteritems(): - cmdsig = cmddict['sig'] - flavor = cmddict.get('flavor', 'mon') - url, params = generate_url_and_params(app, cmdsig, flavor) - perm = cmddict['perm'] - for k in METHOD_DICT.iterkeys(): - if k in perm: - methods = METHOD_DICT[k] - urldict = {'paramsig':params, - 'help':cmddict['help'], - 'module':cmddict['module'], - 'perm':perm, - 'flavor':flavor, - 'methods':methods, - } - - # app.ceph_urls contains a list of urldicts (usually only one long) - if url not in app.ceph_urls: - app.ceph_urls[url] = [urldict] - else: - # If more than one, need to make union of methods of all. - # Method must be checked in handler - methodset = set(methods) - for old_urldict in app.ceph_urls[url]: - methodset |= set(old_urldict['methods']) - methods = list(methodset) - app.ceph_urls[url].append(urldict) - - # add, or re-add, rule with all methods and urldicts - app.add_url_rule(url, url, handler, methods=methods) - url += '.' - app.add_url_rule(url, url, handler, methods=methods) - - app.logger.debug("urls added: %d", len(app.ceph_urls)) - - app.add_url_rule('/', '/', - handler, methods=['GET', 'PUT']) - return addr, port - - -def generate_url_and_params(app, sig, flavor): - ''' - Digest command signature from cluster; generate an absolute - (including app.ceph_baseurl) endpoint from all the prefix words, - and a list of non-prefix param descs - ''' - - url = '' - params = [] - # the OSD command descriptors don't include the 'tell ', so - # tack it onto the front of sig - if flavor == 'tell': - tellsig = parse_funcsig(['tell', - {'name':'target', 'type':'CephOsdName'}]) - sig = tellsig + sig - - for desc in sig: - # prefixes go in the URL path - if desc.t == CephPrefix: - url += '/' + desc.instance.prefix - # CephChoices with 1 required string (not --) do too, unless - # we've already started collecting params, in which case they - # too are params - elif desc.t == CephChoices and \ - len(desc.instance.strings) == 1 and \ - desc.req and \ - not str(desc.instance).startswith('--') and \ - not params: - url += '/' + str(desc.instance) - else: - # tell/ is a weird case; the URL includes what - # would everywhere else be a parameter - if flavor == 'tell' and \ - (desc.t, desc.name) == (CephOsdName, 'target'): - url += '/' - else: - params.append(desc) - - return app.ceph_baseurl + url, params - - -# -# end setup (import-time) functions, begin request-time functions -# - -def concise_sig_for_uri(sig, flavor): - ''' - Return a generic description of how one would send a REST request for sig - ''' - prefix = [] - args = [] - ret = '' - if flavor == 'tell': - ret = 'tell//' - for d in sig: - if d.t == CephPrefix: - prefix.append(d.instance.prefix) - else: - args.append(d.name + '=' + str(d)) - ret += '/'.join(prefix) - if args: - ret += '?' + '&'.join(args) - return ret - -def show_human_help(prefix): - ''' - Dump table showing commands matching prefix - ''' - # XXX There ought to be a better discovery mechanism than an HTML table - s = '' - - permmap = {'r':'GET', 'rw':'PUT'} - line = '' - for cmdsig in sorted(app.ceph_sigdict.itervalues(), cmp=descsort): - concise = concise_sig(cmdsig['sig']) - flavor = cmdsig.get('flavor', 'mon') - if flavor == 'tell': - concise = 'tell//' + concise - if concise.startswith(prefix): - line = ['\n') - s += ''.join(line) - - s += '
Possible commands:MethodDescription
'] - wrapped_sig = textwrap.wrap( - concise_sig_for_uri(cmdsig['sig'], flavor), 40 - ) - for sigline in wrapped_sig: - line.append(flask.escape(sigline) + '\n') - line.append('') - line.append(permmap[cmdsig['perm']]) - line.append('') - line.append(flask.escape(cmdsig['help'])) - line.append('
' - if line: - return s - else: - return '' - -@app.before_request -def log_request(): - ''' - For every request, log it. XXX Probably overkill for production - ''' - app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string) - app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values()) - -@app.route('/') -def root_redir(): - return flask.redirect(app.ceph_baseurl) - -def make_response(fmt, output, statusmsg, errorcode): - ''' - If formatted output, cobble up a response object that contains the - output and status wrapped in enclosing objects; if nonformatted, just - use output+status. Return HTTP status errorcode in any event. - ''' - response = output - if fmt: - if 'json' in fmt: - try: - native_output = json.loads(output or '[]') - response = json.dumps({"output":native_output, - "status":statusmsg}) - except: - return flask.make_response("Error decoding JSON from " + - output, 500) - elif 'xml' in fmt: - # XXX - # one is tempted to do this with xml.etree, but figuring out how - # to 'un-XML' the XML-dumped output so it can be reassembled into - # a piece of the tree here is beyond me right now. - #ET = xml.etree.ElementTree - #resp_elem = ET.Element('response') - #o = ET.SubElement(resp_elem, 'output') - #o.text = output - #s = ET.SubElement(resp_elem, 'status') - #s.text = statusmsg - #response = ET.tostring(resp_elem) - response = ''' - - - {0} - - - {1} - -'''.format(response, xml.sax.saxutils.escape(statusmsg)) - else: - if not 200 <= errorcode < 300: - response = response + '\n' + statusmsg + '\n' - - return flask.make_response(response, errorcode) - -def handler(catchall_path=None, fmt=None, target=None): - ''' - Main endpoint handler; generic for every endpoint, including catchall. - Handles the catchall, anything with <.fmt>, anything with embedded - . Partial match or ?help cause the HTML-table - "show_human_help" output. - ''' - - ep = catchall_path or flask.request.endpoint - ep = ep.replace('.', '') - - if ep[0] != '/': - ep = '/' + ep - - # demand that endpoint begin with app.ceph_baseurl - if not ep.startswith(app.ceph_baseurl): - return make_response(fmt, '', 'Page not found', 404) - - rel_ep = ep[len(app.ceph_baseurl)+1:] - - # Extensions override Accept: headers override defaults - if not fmt: - if 'application/json' in flask.request.accept_mimetypes.values(): - fmt = 'json' - elif 'application/xml' in flask.request.accept_mimetypes.values(): - fmt = 'xml' - - prefix = '' - pgid = None - cmdtarget = 'mon', '' - - if target: - # got tell/; validate osdid or pgid - name = CephOsdName() - pgidobj = CephPgid() - try: - name.valid(target) - except ArgumentError: - # try pgid - try: - pgidobj.valid(target) - except ArgumentError: - return flask.make_response("invalid osdid or pgid", 400) - else: - # it's a pgid - pgid = pgidobj.val - cmdtarget = 'pg', pgid - else: - # it's an osd - cmdtarget = name.nametype, name.nameid - - # prefix does not include tell// - prefix = ' '.join(rel_ep.split('/')[2:]).strip() - else: - # non-target command: prefix is entire path - prefix = ' '.join(rel_ep.split('/')).strip() - - # show "match as much as you gave me" help for unknown endpoints - if not ep in app.ceph_urls: - helptext = show_human_help(prefix) - if helptext: - resp = flask.make_response(helptext, 400) - resp.headers['Content-Type'] = 'text/html' - return resp - else: - return make_response(fmt, '', 'Invalid endpoint ' + ep, 400) - - found = None - exc = '' - for urldict in app.ceph_urls[ep]: - if flask.request.method not in urldict['methods']: - continue - paramsig = urldict['paramsig'] - - # allow '?help' for any specifically-known endpoint - if 'help' in flask.request.args: - response = flask.make_response('{0}: {1}'.\ - format(prefix + concise_sig(paramsig), urldict['help'])) - response.headers['Content-Type'] = 'text/plain' - return response - - # if there are parameters for this endpoint, process them - if paramsig: - args = {} - for k, l in flask.request.args.iterlists(): - if len(l) == 1: - args[k] = l[0] - else: - args[k] = l - - # is this a valid set of params? - try: - argdict = validate(args, paramsig) - found = urldict - break - except Exception as e: - exc += str(e) - continue - else: - if flask.request.args: - continue - found = urldict - argdict = {} - break - - if not found: - return make_response(fmt, '', exc + '\n', 400) - - argdict['format'] = fmt or 'plain' - argdict['module'] = found['module'] - argdict['perm'] = found['perm'] - if pgid: - argdict['pgid'] = pgid - - if not cmdtarget: - cmdtarget = ('mon', '') - - app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) - ret, outbuf, outs = json_command(app.ceph_cluster, prefix=prefix, - target=cmdtarget, - inbuf=flask.request.data, argdict=argdict) - if ret: - return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400) - - response = make_response(fmt, outbuf, outs or 'OK', 200) - if fmt: - contenttype = 'application/' + fmt.replace('-pretty','') - else: - contenttype = 'text/plain' - response.headers['Content-Type'] = contenttype - return response - -# -# Main entry point from wrapper/WSGI server: call with cmdline args, -# get back the WSGI app entry point -# -def generate_app(conf, cluster, clientname, clientid, args): - addr, port = api_setup(app, conf, cluster, clientname, clientid, args) - app.ceph_addr = addr - app.ceph_port = port - return app diff --git a/src/pybind/rados.py b/src/pybind/rados.py deleted file mode 100644 index a0e5bf42ba9..00000000000 --- a/src/pybind/rados.py +++ /dev/null @@ -1,1661 +0,0 @@ -""" -This module is a thin wrapper around librados. - -Copyright 2011, Hannu Valtonen -""" -from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_char, c_int, c_long, \ - c_ulong, create_string_buffer, byref, Structure, c_uint64, c_ubyte, \ - pointer, CFUNCTYPE -import ctypes -import errno -import threading -import time -from datetime import datetime - -ANONYMOUS_AUID = 0xffffffffffffffff -ADMIN_AUID = 0 - -class Error(Exception): - """ `Error` class, derived from `Exception` """ - pass - -class PermissionError(Error): - """ `PermissionError` class, derived from `Error` """ - pass - -class ObjectNotFound(Error): - """ `ObjectNotFound` class, derived from `Error` """ - pass - -class NoData(Error): - """ `NoData` class, derived from `Error` """ - pass - -class ObjectExists(Error): - """ `ObjectExists` class, derived from `Error` """ - pass - -class IOError(Error): - """ `IOError` class, derived from `Error` """ - pass - -class NoSpace(Error): - """ `NoSpace` class, derived from `Error` """ - pass - -class IncompleteWriteError(Error): - """ `IncompleteWriteError` class, derived from `Error` """ - pass - -class RadosStateError(Error): - """ `RadosStateError` class, derived from `Error` """ - pass - -class IoctxStateError(Error): - """ `IoctxStateError` class, derived from `Error` """ - pass - -class ObjectStateError(Error): - """ `ObjectStateError` class, derived from `Error` """ - pass - -class LogicError(Error): - """ `` class, derived from `Error` """ - pass - -def make_ex(ret, msg): - """ - Translate a librados return code into an exception. - - :param ret: the return code - :type ret: int - :param msg: the error message to use - :type msg: str - :returns: a subclass of :class:`Error` - """ - - errors = { - errno.EPERM : PermissionError, - errno.ENOENT : ObjectNotFound, - errno.EIO : IOError, - errno.ENOSPC : NoSpace, - errno.EEXIST : ObjectExists, - errno.ENODATA : NoData - } - ret = abs(ret) - if ret in errors: - return errors[ret](msg) - else: - return Error(msg + (": errno %s" % errno.errorcode[ret])) - -class rados_pool_stat_t(Structure): - """ Usage information for a pool """ - _fields_ = [("num_bytes", c_uint64), - ("num_kb", c_uint64), - ("num_objects", c_uint64), - ("num_object_clones", c_uint64), - ("num_object_copies", c_uint64), - ("num_objects_missing_on_primary", c_uint64), - ("num_objects_unfound", c_uint64), - ("num_objects_degraded", c_uint64), - ("num_rd", c_uint64), - ("num_rd_kb", c_uint64), - ("num_wr", c_uint64), - ("num_wr_kb", c_uint64)] - -class rados_cluster_stat_t(Structure): - """ Cluster-wide usage information """ - _fields_ = [("kb", c_uint64), - ("kb_used", c_uint64), - ("kb_avail", c_uint64), - ("num_objects", c_uint64)] - -class Version(object): - """ Version information """ - def __init__(self, major, minor, extra): - self.major = major - self.minor = minor - self.extra = extra - - def __str__(self): - return "%d.%d.%d" % (self.major, self.minor, self.extra) - -class RadosThread(threading.Thread): - def __init__(self, target, args=None): - self.args = args - self.target = target - threading.Thread.__init__(self) - - def run(self): - self.retval = self.target(*self.args) - -# time in seconds between each call to t.join() for child thread -POLL_TIME_INCR = 0.5 - -def run_in_thread(target, args, timeout=0): - import sys - interrupt = False - - countdown = timeout - t = RadosThread(target, args) - - # allow the main thread to exit (presumably, avoid a join() on this - # subthread) before this thread terminates. This allows SIGINT - # exit of a blocked call. See below. - t.daemon = True - - t.start() - try: - # poll for thread exit - while t.is_alive(): - t.join(POLL_TIME_INCR) - if timeout: - countdown = countdown - POLL_TIME_INCR - if countdown <= 0: - raise KeyboardInterrupt - - t.join() # in case t exits before reaching the join() above - except KeyboardInterrupt: - # ..but allow SIGINT to terminate the waiting. Note: this - # relies on the Linux kernel behavior of delivering the signal - # to the main thread in preference to any subthread (all that's - # strictly guaranteed is that *some* thread that has the signal - # unblocked will receive it). But there doesn't seem to be - # any interface to create t with SIGINT blocked. - interrupt = True - - if interrupt: - t.retval = -errno.EINTR - return t.retval - -class Rados(object): - """librados python wrapper""" - def require_state(self, *args): - """ - Checks if the Rados object is in a special state - - :raises: RadosStateError - """ - for a in args: - if self.state == a: - return - raise RadosStateError("You cannot perform that operation on a \ -Rados object in state %s." % (self.state)) - - def __init__(self, rados_id=None, name=None, clustername=None, - conf_defaults=None, conffile=None, conf=None, flags=0): - self.librados = CDLL('librados.so.2') - self.cluster = c_void_p() - self.rados_id = rados_id - if rados_id is not None and not isinstance(rados_id, str): - raise TypeError('rados_id must be a string or None') - if conffile is not None and not isinstance(conffile, str): - raise TypeError('conffile must be a string or None') - if name is not None and not isinstance(name, str): - raise TypeError('name must be a string or None') - if clustername is not None and not isinstance(clustername, str): - raise TypeError('clustername must be a string or None') - if rados_id and name: - raise Error("Rados(): can't supply both rados_id and name") - if rados_id: - name = 'client.' + rados_id - if name is None: - name = 'client.admin' - if clustername is None: - clustername = 'ceph' - ret = run_in_thread(self.librados.rados_create2, - (byref(self.cluster), c_char_p(clustername), - c_char_p(name), c_uint64(flags))) - - if ret != 0: - raise Error("rados_initialize failed with error code: %d" % ret) - self.state = "configuring" - # order is important: conf_defaults, then conffile, then conf - if conf_defaults: - for key, value in conf_defaults.iteritems(): - self.conf_set(key, value) - if conffile is not None: - # read the default conf file when '' is given - if conffile == '': - conffile = None - self.conf_read_file(conffile) - if conf: - for key, value in conf.iteritems(): - self.conf_set(key, value) - - def shutdown(self): - """ - Disconnects from the cluster. - """ - if (self.__dict__.has_key("state") and self.state != "shutdown"): - run_in_thread(self.librados.rados_shutdown, (self.cluster,)) - self.state = "shutdown" - - def __enter__(self): - self.connect() - return self - - def __exit__(self, type_, value, traceback): - self.shutdown() - return False - - def __del__(self): - self.shutdown() - - def version(self): - """ - Get the version number of the ``librados`` C library. - - :returns: a tuple of ``(major, minor, extra)`` components of the - librados version - """ - major = c_int(0) - minor = c_int(0) - extra = c_int(0) - run_in_thread(self.librados.rados_version, - (byref(major), byref(minor), byref(extra))) - return Version(major.value, minor.value, extra.value) - - def conf_read_file(self, path=None): - """ - Configure the cluster handle using a Ceph config file. - - :param path: path to the config file - :type path: str - """ - self.require_state("configuring", "connected") - if path is not None and not isinstance(path, str): - raise TypeError('path must be a string') - ret = run_in_thread(self.librados.rados_conf_read_file, - (self.cluster, c_char_p(path))) - if (ret != 0): - raise make_ex(ret, "error calling conf_read_file") - - def conf_parse_argv(self, args): - """ - Parse known arguments from args, and remove; returned - args contain only those unknown to ceph - """ - self.require_state("configuring", "connected") - if not args: - return - # create instances of arrays of c_char_p's, both len(args) long - # cretargs will always be a subset of cargs (perhaps identical) - cargs = (c_char_p * len(args))(*args) - cretargs = (c_char_p * len(args))() - ret = run_in_thread(self.librados.rados_conf_parse_argv_remainder, - (self.cluster, len(args), cargs, cretargs)) - if ret: - raise make_ex(ret, "error calling conf_parse_argv_remainder") - - # cretargs was allocated with fixed length; collapse return - # list to eliminate any missing args - - retargs = [a for a in cretargs if a is not None] - return retargs - - def conf_parse_env(self, var='CEPH_ARGS'): - """ - Parse known arguments from an environment variable, normally - CEPH_ARGS. - """ - self.require_state("configuring", "connected") - if not var: - return - ret = run_in_thread(self.librados.rados_conf_parse_env, - (self.cluster, c_char_p(var))) - if (ret != 0): - raise make_ex(ret, "error calling conf_parse_env") - - def conf_get(self, option): - """ - Get the value of a configuration option - - :param option: which option to read - :type option: str - - :returns: str - value of the option or None - :raises: :class:`TypeError` - """ - self.require_state("configuring", "connected") - if not isinstance(option, str): - raise TypeError('option must be a string') - length = 20 - while True: - ret_buf = create_string_buffer(length) - ret = run_in_thread(self.librados.rados_conf_get, - (self.cluster, c_char_p(option), ret_buf, - c_size_t(length))) - if (ret == 0): - return ret_buf.value - elif (ret == -errno.ENAMETOOLONG): - length = length * 2 - elif (ret == -errno.ENOENT): - return None - else: - raise make_ex(ret, "error calling conf_get") - - def conf_set(self, option, val): - """ - Set the value of a configuration option - - :param option: which option to set - :type option: str - :param option: value of the option - :type option: str - - :raises: :class:`TypeError`, :class:`ObjectNotFound` - """ - self.require_state("configuring", "connected") - if not isinstance(option, str): - raise TypeError('option must be a string') - if not isinstance(val, str): - raise TypeError('val must be a string') - ret = run_in_thread(self.librados.rados_conf_set, - (self.cluster, c_char_p(option), c_char_p(val))) - if (ret != 0): - raise make_ex(ret, "error calling conf_set") - - def connect(self, timeout=0): - """ - Connect to the cluster. - """ - self.require_state("configuring") - ret = run_in_thread(self.librados.rados_connect, (self.cluster,), - timeout) - if (ret != 0): - raise make_ex(ret, "error calling connect") - self.state = "connected" - - def get_cluster_stats(self): - """ - Read usage info about the cluster - - This tells you total space, space used, space available, and number - of objects. These are not updated immediately when data is written, - they are eventually consistent. - - :returns: dict - contains the following keys: - - *``kb`` (int) - total space - - *``kb_used`` (int) - space used - - *``kb_avail`` (int) - free space available - - *``num_objects`` (int) - number of objects - - """ - stats = rados_cluster_stat_t() - ret = run_in_thread(self.librados.rados_cluster_stat, - (self.cluster, byref(stats))) - if ret < 0: - raise make_ex( - ret, "Rados.get_cluster_stats(%s): get_stats failed" % self.rados_id) - return {'kb': stats.kb, - 'kb_used': stats.kb_used, - 'kb_avail': stats.kb_avail, - 'num_objects': stats.num_objects} - - def pool_exists(self, pool_name): - """ - Checks if a given pool exists. - - :param pool_name: name of the pool to check - :type pool_name: str - - :raises: :class:`TypeError`, :class:`Error` - :returns: bool - whether the pool exists, false otherwise. - """ - self.require_state("connected") - if not isinstance(pool_name, str): - raise TypeError('pool_name must be a string') - ret = run_in_thread(self.librados.rados_pool_lookup, - (self.cluster, c_char_p(pool_name))) - if (ret >= 0): - return True - elif (ret == -errno.ENOENT): - return False - else: - raise make_ex(ret, "error looking up pool '%s'" % pool_name) - - def create_pool(self, pool_name, auid=None, crush_rule=None): - """ - Create a pool: - - with default settings: if auid=None and crush_rule=None - - owned by a specific auid: auid given and crush_rule=None - - with a specific CRUSH rule: if auid=None and crush_rule given - - with a specific CRUSH rule and auid: if auid and crush_rule given - - :param pool_name: name of the pool to create - :type pool_name: str - :param auid: the id of the owner of the new pool - :type auid: int - :param crush_rule: rule to use for placement in the new pool - :type crush_rule: str - - :raises: :class:`TypeError`, :class:`Error` - """ - self.require_state("connected") - if not isinstance(pool_name, str): - raise TypeError('pool_name must be a string') - if crush_rule is not None and not isinstance(crush_rule, str): - raise TypeError('cruse_rule must be a string') - if (auid == None): - if (crush_rule == None): - ret = run_in_thread(self.librados.rados_pool_create, - (self.cluster, c_char_p(pool_name))) - else: - ret = run_in_thread(self.librados.\ - rados_pool_create_with_crush_rule, - (self.cluster, c_char_p(pool_name), - c_ubyte(crush_rule))) - - elif (crush_rule == None): - ret = run_in_thread(self.librados.rados_pool_create_with_auid, - (self.cluster, c_char_p(pool_name), - c_uint64(auid))) - else: - ret = run_in_thread(self.librados.rados_pool_create_with_all, - (self.cluster, c_char_p(pool_name), - c_uint64(auid), c_ubyte(crush_rule))) - if ret < 0: - raise make_ex(ret, "error creating pool '%s'" % pool_name) - - def delete_pool(self, pool_name): - """ - Delete a pool and all data inside it. - - The pool is removed from the cluster immediately, - but the actual data is deleted in the background. - - :param pool_name: name of the pool to delete - :type pool_name: str - - :raises: :class:`TypeError`, :class:`Error` - """ - self.require_state("connected") - if not isinstance(pool_name, str): - raise TypeError('pool_name must be a string') - ret = run_in_thread(self.librados.rados_pool_delete, - (self.cluster, c_char_p(pool_name))) - if ret < 0: - raise make_ex(ret, "error deleting pool '%s'" % pool_name) - - def list_pools(self): - """ - Gets a list of pool names. - - :returns: list - of pool names. - """ - self.require_state("connected") - size = c_size_t(512) - while True: - c_names = create_string_buffer(size.value) - ret = run_in_thread(self.librados.rados_pool_list, - (self.cluster, byref(c_names), size)) - if ret > size.value: - size = c_size_t(ret) - else: - break - return filter(lambda name: name != '', c_names.raw.split('\0')) - - def get_fsid(self): - """ - Get the fsid of the cluster as a hexadecimal string. - - :raises: :class:`Error` - :returns: str - cluster fsid - """ - self.require_state("connected") - buf_len = 37 - fsid = create_string_buffer(buf_len) - ret = run_in_thread(self.librados.rados_cluster_fsid, - (self.cluster, byref(fsid), c_size_t(buf_len))) - if ret < 0: - raise make_ex(ret, "error getting cluster fsid") - return fsid.value - - def open_ioctx(self, ioctx_name): - """ - Create an io context - - The io context allows you to perform operations within a particular - pool. - - :param ioctx_name: name of the pool - :type ioctx_name: str - - :raises: :class:`TypeError`, :class:`Error` - :returns: Ioctx - Rados Ioctx object - """ - self.require_state("connected") - if not isinstance(ioctx_name, str): - raise TypeError('ioctx_name must be a string') - ioctx = c_void_p() - ret = run_in_thread(self.librados.rados_ioctx_create, - (self.cluster, c_char_p(ioctx_name), byref(ioctx))) - if ret < 0: - raise make_ex(ret, "error opening ioctx '%s'" % ioctx_name) - return Ioctx(ioctx_name, self.librados, ioctx) - - def mon_command(self, cmd, inbuf, timeout=0, target=None): - """ - mon_command[_target](cmd, inbuf, outbuf, outbuflen, outs, outslen) - returns (int ret, string outbuf, string outs) - """ - import sys - self.require_state("connected") - outbufp = pointer(pointer(c_char())) - outbuflen = c_long() - outsp = pointer(pointer(c_char())) - outslen = c_long() - cmdarr = (c_char_p * len(cmd))(*cmd) - - if target: - ret = run_in_thread(self.librados.rados_mon_command_target, - (self.cluster, c_char_p(target), cmdarr, - len(cmd), c_char_p(inbuf), len(inbuf), - outbufp, byref(outbuflen), outsp, - byref(outslen)), timeout) - else: - ret = run_in_thread(self.librados.rados_mon_command, - (self.cluster, cmdarr, len(cmd), - c_char_p(inbuf), len(inbuf), - outbufp, byref(outbuflen), outsp, byref(outslen)), - timeout) - - # copy returned memory (ctypes makes a copy, not a reference) - my_outbuf = outbufp.contents[:(outbuflen.value)] - my_outs = outsp.contents[:(outslen.value)] - - # free callee's allocations - if outbuflen.value: - run_in_thread(self.librados.rados_buffer_free, (outbufp.contents,)) - if outslen.value: - run_in_thread(self.librados.rados_buffer_free, (outsp.contents,)) - - return (ret, my_outbuf, my_outs) - - def osd_command(self, osdid, cmd, inbuf, timeout=0): - """ - osd_command(osdid, cmd, inbuf, outbuf, outbuflen, outs, outslen) - returns (int ret, string outbuf, string outs) - """ - import sys - self.require_state("connected") - outbufp = pointer(pointer(c_char())) - outbuflen = c_long() - outsp = pointer(pointer(c_char())) - outslen = c_long() - cmdarr = (c_char_p * len(cmd))(*cmd) - ret = run_in_thread(self.librados.rados_osd_command, - (self.cluster, osdid, cmdarr, len(cmd), - c_char_p(inbuf), len(inbuf), - outbufp, byref(outbuflen), outsp, byref(outslen)), - timeout) - - # copy returned memory (ctypes makes a copy, not a reference) - my_outbuf = outbufp.contents[:(outbuflen.value)] - my_outs = outsp.contents[:(outslen.value)] - - # free callee's allocations - if outbuflen.value: - run_in_thread(self.librados.rados_buffer_free, (outbufp.contents,)) - if outslen.value: - run_in_thread(self.librados.rados_buffer_free, (outsp.contents,)) - - return (ret, my_outbuf, my_outs) - - def pg_command(self, pgid, cmd, inbuf, timeout=0): - """ - pg_command(pgid, cmd, inbuf, outbuf, outbuflen, outs, outslen) - returns (int ret, string outbuf, string outs) - """ - import sys - self.require_state("connected") - outbufp = pointer(pointer(c_char())) - outbuflen = c_long() - outsp = pointer(pointer(c_char())) - outslen = c_long() - cmdarr = (c_char_p * len(cmd))(*cmd) - ret = run_in_thread(self.librados.rados_pg_command, - (self.cluster, c_char_p(pgid), cmdarr, len(cmd), - c_char_p(inbuf), len(inbuf), - outbufp, byref(outbuflen), outsp, byref(outslen)), - timeout) - - # copy returned memory (ctypes makes a copy, not a reference) - my_outbuf = outbufp.contents[:(outbuflen.value)] - my_outs = outsp.contents[:(outslen.value)] - - # free callee's allocations - if outbuflen.value: - run_in_thread(self.librados.rados_buffer_free, (outbufp.contents,)) - if outslen.value: - run_in_thread(self.librados.rados_buffer_free, (outsp.contents,)) - - return (ret, my_outbuf, my_outs) - -class ObjectIterator(object): - """rados.Ioctx Object iterator""" - def __init__(self, ioctx): - self.ioctx = ioctx - self.ctx = c_void_p() - ret = run_in_thread(self.ioctx.librados.rados_objects_list_open, - (self.ioctx.io, byref(self.ctx))) - if ret < 0: - raise make_ex(ret, "error iterating over the objects in ioctx '%s'" \ - % self.ioctx.name) - - def __iter__(self): - return self - - def next(self): - """ - Get the next object name and locator in the pool - - :raises: StopIteration - :returns: next rados.Ioctx Object - """ - key = c_char_p() - locator = c_char_p() - ret = run_in_thread(self.ioctx.librados.rados_objects_list_next, - (self.ctx, byref(key), byref(locator))) - if ret < 0: - raise StopIteration() - return Object(self.ioctx, key.value, locator.value) - - def __del__(self): - run_in_thread(self.ioctx.librados.rados_objects_list_close, (self.ctx,)) - -class XattrIterator(object): - """Extended attribute iterator""" - def __init__(self, ioctx, it, oid): - self.ioctx = ioctx - self.it = it - self.oid = oid - - def __iter__(self): - return self - - def next(self): - """ - Get the next xattr on the object - - :raises: StopIteration - :returns: pair - of name and value of the next Xattr - """ - name_ = c_char_p(0) - val_ = c_char_p(0) - len_ = c_int(0) - ret = run_in_thread(self.ioctx.librados.rados_getxattrs_next, - (self.it, byref(name_), byref(val_), byref(len_))) - if (ret != 0): - raise make_ex(ret, "error iterating over the extended attributes \ -in '%s'" % self.oid) - if name_.value == None: - raise StopIteration() - name = ctypes.string_at(name_) - val = ctypes.string_at(val_, len_) - return (name, val) - - def __del__(self): - run_in_thread(self.ioctx.librados.rados_getxattrs_end, (self.it,)) - -class SnapIterator(object): - """Snapshot iterator""" - def __init__(self, ioctx): - self.ioctx = ioctx - # We don't know how big a buffer we need until we've called the - # function. So use the exponential doubling strategy. - num_snaps = 10 - while True: - self.snaps = (ctypes.c_uint64 * num_snaps)() - ret = run_in_thread(self.ioctx.librados.rados_ioctx_snap_list, - (self.ioctx.io, self.snaps, c_int(num_snaps))) - if (ret >= 0): - self.max_snap = ret - break - elif (ret != -errno.ERANGE): - raise make_ex(ret, "error calling rados_snap_list for \ -ioctx '%s'" % self.ioctx.name) - num_snaps = num_snaps * 2 - self.cur_snap = 0 - - def __iter__(self): - return self - - def next(self): - """ - Get the next Snapshot - - :raises: :class:`Error`, StopIteration - :returns: Snap - next snapshot - """ - if (self.cur_snap >= self.max_snap): - raise StopIteration - snap_id = self.snaps[self.cur_snap] - name_len = 10 - while True: - name = create_string_buffer(name_len) - ret = run_in_thread(self.ioctx.librados.rados_ioctx_snap_get_name, - (self.ioctx.io, c_uint64(snap_id), byref(name), - c_int(name_len))) - if (ret == 0): - name_len = ret - break - elif (ret != -errno.ERANGE): - raise make_ex(ret, "rados_snap_get_name error") - name_len = name_len * 2 - snap = Snap(self.ioctx, name.value, snap_id) - self.cur_snap = self.cur_snap + 1 - return snap - -class Snap(object): - """Snapshot object""" - def __init__(self, ioctx, name, snap_id): - self.ioctx = ioctx - self.name = name - self.snap_id = snap_id - - def __str__(self): - return "rados.Snap(ioctx=%s,name=%s,snap_id=%d)" \ - % (str(self.ioctx), self.name, self.snap_id) - - def get_timestamp(self): - """ - Find when a snapshot in the current pool occurred - - :raises: :class:`Error` - :returns: datetime - the data and time the snapshot was created - """ - snap_time = c_long(0) - ret = run_in_thread(self.ioctx.librados.rados_ioctx_snap_get_stamp, - (self.ioctx.io, self.snap_id, byref(snap_time))) - if (ret != 0): - raise make_ex(ret, "rados_ioctx_snap_get_stamp error") - return datetime.fromtimestamp(snap_time.value) - -class Completion(object): - """completion object""" - def __init__(self, ioctx, rados_comp, oncomplete, onsafe): - self.rados_comp = rados_comp - self.oncomplete = oncomplete - self.onsafe = onsafe - self.ioctx = ioctx - - def wait_for_safe(self): - """ - Is an asynchronous operation safe? - - This does not imply that the safe callback has finished. - - :returns: whether the operation is safe - """ - return run_in_thread(self.ioctx.librados.rados_aio_is_safe, - (self.rados_comp,)) - - def wait_for_complete(self): - """ - Has an asynchronous operation completed? - - This does not imply that the safe callback has finished. - - :returns: whether the operation is completed - """ - return run_in_thread(self.ioctx.librados.rados_aio_is_complete, - (self.rados_comp,)) - - def get_return_value(self): - """ - Get the return value of an asychronous operation - - The return value is set when the operation is complete or safe, - whichever comes first. - - :returns: int - return value of the operation - """ - return run_in_thread(self.ioctx.librados.rados_aio_get_return_value, - (self.rados_comp,)) - - def __del__(self): - """ - Release a completion - - Call this when you no longer need the completion. It may not be - freed immediately if the operation is not acked and committed. - """ - run_in_thread(self.ioctx.librados.rados_aio_release, - (self.rados_comp,)) - -class Ioctx(object): - """rados.Ioctx object""" - def __init__(self, name, librados, io): - self.name = name - self.librados = librados - self.io = io - self.state = "open" - self.locator_key = "" - self.safe_cbs = {} - self.complete_cbs = {} - RADOS_CB = CFUNCTYPE(c_int, c_void_p, c_void_p) - self.__aio_safe_cb_c = RADOS_CB(self.__aio_safe_cb) - self.__aio_complete_cb_c = RADOS_CB(self.__aio_complete_cb) - self.lock = threading.Lock() - - def __enter__(self): - return self - - def __exit__(self, type_, value, traceback): - self.close() - return False - - def __del__(self): - self.close() - - def __aio_safe_cb(self, completion, _): - """ - Callback to onsafe() for asynchronous operations - """ - cb = None - with self.lock: - cb = self.safe_cbs[completion] - del self.safe_cbs[completion] - cb.onsafe(cb) - return 0 - - def __aio_complete_cb(self, completion, _): - """ - Callback to oncomplete() for asynchronous operations - """ - cb = None - with self.lock: - cb = self.complete_cbs[completion] - del self.complete_cbs[completion] - cb.oncomplete(cb) - return 0 - - def __get_completion(self, oncomplete, onsafe): - """ - Constructs a completion to use with asynchronous operations - - :param oncomplete: what to do when the write is safe and complete in memory - on all replicas - :type oncomplete: completion - :param onsafe: what to do when the write is safe and complete on storage - on all replicas - :type onsafe: completion - - :raises: :class:`Error` - :returns: completion object - """ - completion = c_void_p(0) - complete_cb = None - safe_cb = None - if oncomplete: - complete_cb = self.__aio_complete_cb_c - if onsafe: - safe_cb = self.__aio_safe_cb_c - ret = run_in_thread(self.librados.rados_aio_create_completion, - (c_void_p(0), complete_cb, safe_cb, - byref(completion))) - if ret < 0: - raise make_ex(ret, "error getting a completion") - with self.lock: - completion_obj = Completion(self, completion, oncomplete, onsafe) - if oncomplete: - self.complete_cbs[completion.value] = completion_obj - if onsafe: - self.safe_cbs[completion.value] = completion_obj - return completion_obj - - def aio_write(self, object_name, to_write, offset=0, - oncomplete=None, onsafe=None): - """ - Write data to an object asynchronously - - Queues the write and returns. - - :param object_name: name of the object - :type object_name: str - :param to_write: data to write - :type to_write: str - :param offset: byte offset in the object to begin writing at - :type offset: int - :param oncomplete: what to do when the write is safe and complete in memory - on all replicas - :type oncomplete: completion - :param onsafe: what to do when the write is safe and complete on storage - on all replicas - :type onsafe: completion - - :raises: :class:`Error` - :returns: completion object - """ - completion = self.__get_completion(oncomplete, onsafe) - ret = run_in_thread(self.librados.rados_aio_write, - (self.io, c_char_p(object_name), - completion.rados_comp, c_char_p(to_write), - c_size_t(len(to_write)), c_uint64(offset))) - if ret < 0: - raise make_ex(ret, "error writing object %s" % object_name) - return completion - - def aio_write_full(self, object_name, to_write, - oncomplete=None, onsafe=None): - """ - Asychronously write an entire object - - The object is filled with the provided data. If the object exists, - it is atomically truncated and then written. - Queues the write and returns. - - :param object_name: name of the object - :type object_name: str - :param to_write: data to write - :type to_write: str - :param oncomplete: what to do when the write is safe and complete in memory - on all replicas - :type oncomplete: completion - :param onsafe: what to do when the write is safe and complete on storage - on all replicas - :type onsafe: completion - - :raises: :class:`Error` - :returns: completion object - """ - completion = self.__get_completion(oncomplete, onsafe) - ret = run_in_thread(self.librados.rados_aio_write_full, - (self.io, c_char_p(object_name), - completion.rados_comp, c_char_p(to_write), - c_size_t(len(to_write)))) - if ret < 0: - raise make_ex(ret, "error writing object %s" % object_name) - return completion - - def aio_append(self, object_name, to_append, oncomplete=None, onsafe=None): - """ - Asychronously append data to an object - - Queues the write and returns. - - :param object_name: name of the object - :type object_name: str - :param to_append: data to append - :type to_append: str - :param offset: byte offset in the object to begin writing at - :type offset: int - :param oncomplete: what to do when the write is safe and complete in memory - on all replicas - :type oncomplete: completion - :param onsafe: what to do when the write is safe and complete on storage - on all replicas - :type onsafe: completion - - :raises: :class:`Error` - :returns: completion object - """ - completion = self.__get_completion(oncomplete, onsafe) - ret = run_in_thread(self.librados.rados_aio_append, - (self.io, c_char_p(object_name), - completion.rados_comp, c_char_p(to_append), - c_size_t(len(to_append)))) - if ret < 0: - raise make_ex(ret, "error appending to object %s" % object_name) - return completion - - def aio_flush(self): - """ - Block until all pending writes in an io context are safe - - :raises: :class:`Error` - """ - ret = run_in_thread(self.librados.rados_aio_flush, (self.io,)) - if ret < 0: - raise make_ex(ret, "error flushing") - - def aio_read(self, object_name, length, offset, oncomplete): - """ - Asychronously read data from an object - - oncomplete will be called with the returned read value as - well as the completion: - - oncomplete(completion, data_read) - - :param object_name: name of the object to read from - :type object_name: str - :param length: the number of bytes to read - :type length: int - :param offset: byte offset in the object to begin reading from - :type offset: int - :param oncomplete: what to do when the read is complete - :type oncomplete: completion - - :raises: :class:`Error` - :returns: completion object - """ - buf = create_string_buffer(length) - def oncomplete_(completion): - return oncomplete(completion, buf.value) - completion = self.__get_completion(oncomplete_, None) - ret = run_in_thread(self.librados.rados_aio_read, - (self.io, c_char_p(object_name), - completion.rados_comp, buf, c_size_t(length), - c_uint64(offset))) - if ret < 0: - raise make_ex(ret, "error reading %s" % object_name) - return completion - - def require_ioctx_open(self): - """ - Checks if the rados.Ioctx object state is 'open' - - :raises: IoctxStateError - """ - if self.state != "open": - raise IoctxStateError("The pool is %s" % self.state) - - def change_auid(self, auid): - """ - Attempt to change an io context's associated auid "owner." - - Requires that you have write permission on both the current and new - auid. - - :raises: :class:`Error` - """ - self.require_ioctx_open() - ret = run_in_thread(self.librados.rados_ioctx_pool_set_auid, - (self.io, ctypes.c_uint64(auid))) - if ret < 0: - raise make_ex(ret, "error changing auid of '%s' to %d" %\ - (self.name, auid)) - - def set_locator_key(self, loc_key): - """ - Set the key for mapping objects to pgs within an io context. - - The key is used instead of the object name to determine which - placement groups an object is put in. This affects all subsequent - operations of the io context - until a different locator key is - set, all objects in this io context will be placed in the same pg. - - :param loc_key: the key to use as the object locator, or NULL to discard - any previously set key - :type loc_key: str - - :raises: :class:`TypeError` - """ - self.require_ioctx_open() - if not isinstance(loc_key, str): - raise TypeError('loc_key must be a string') - run_in_thread(self.librados.rados_ioctx_locator_set_key, - (self.io, c_char_p(loc_key))) - self.locator_key = loc_key - - def get_locator_key(self): - """ - Get the locator_key of context - - :returns: locator_key - """ - return self.locator_key - - def close(self): - """ - Close a rados.Ioctx object. - - This just tells librados that you no longer need to use the io context. - It may not be freed immediately if there are pending asynchronous - requests on it, but you should not use an io context again after - calling this function on it. - """ - if self.state == "open": - self.require_ioctx_open() - run_in_thread(self.librados.rados_ioctx_destroy, (self.io,)) - self.state = "closed" - - def write(self, key, data, offset=0): - """ - Write data to an object synchronously - - :param key: name of the object - :type key: str - :param data: data to write - :type data: str - :param offset: byte offset in the object to begin writing at - :type offset: int - - :raises: :class:`TypeError` - :raises: :class:`IncompleteWriteError` - :raises: :class:`LogicError` - :returns: int - number of bytes written - """ - self.require_ioctx_open() - if not isinstance(data, str): - raise TypeError('data must be a string') - length = len(data) - ret = run_in_thread(self.librados.rados_write, - (self.io, c_char_p(key), c_char_p(data), - c_size_t(length), c_uint64(offset))) - if ret == length: - return ret - elif ret < 0: - raise make_ex(ret, "Ioctx.write(%s): failed to write %s" % \ - (self.name, key)) - elif ret < length: - raise IncompleteWriteError("Wrote only %d out of %d bytes" % \ - (ret, length)) - else: - raise LogicError("Ioctx.write(%s): rados_write \ -returned %d, but %d was the maximum number of bytes it could have \ -written." % (self.name, ret, length)) - - def write_full(self, key, data): - """ - Write an entire object synchronously. - - The object is filled with the provided data. If the object exists, - it is atomically truncated and then written. - - :param key: name of the object - :type key: str - :param data: data to write - :type data: str - - :raises: :class:`TypeError` - :raises: :class:`Error` - :returns: int - 0 on success - """ - self.require_ioctx_open() - if not isinstance(key, str): - raise TypeError('key must be a string') - if not isinstance(data, str): - raise TypeError('data must be a string') - length = len(data) - ret = run_in_thread(self.librados.rados_write_full, - (self.io, c_char_p(key), c_char_p(data), - c_size_t(length))) - if ret == 0: - return ret - else: - raise make_ex(ret, "Ioctx.write(%s): failed to write_full %s" % \ - (self.name, key)) - - def read(self, key, length=8192, offset=0): - """ - Write data to an object synchronously - - :param key: name of the object - :type key: str - :param length: the number of bytes to read (default=8192) - :type length: int - :param offset: byte offset in the object to begin reading at - :type offset: int - - :raises: :class:`TypeError` - :raises: :class:`Error` - :returns: str - data read from object - """ - self.require_ioctx_open() - if not isinstance(key, str): - raise TypeError('key must be a string') - ret_buf = create_string_buffer(length) - ret = run_in_thread(self.librados.rados_read, - (self.io, c_char_p(key), ret_buf, c_size_t(length), - c_uint64(offset))) - if ret < 0: - raise make_ex(ret, "Ioctx.read(%s): failed to read %s" % (self.name, key)) - return ctypes.string_at(ret_buf, ret) - - def get_stats(self): - """ - Get pool usage statistics - - :returns: dict - contains the following keys: - - *``num_bytes`` (int) - size of pool in bytes - - *``num_kb`` (int) - size of pool in kbytes - - *``num_objects`` (int) - number of objects in the pool - - *``num_object_clones`` (int) - number of object clones - - *``num_object_copies`` (int) - number of object copies - - *``num_objects_missing_on_primary`` (int) - number of objets - missing on primary - - *``num_objects_unfound`` (int) - number of unfound objects - - *``num_objects_degraded`` (int) - number of degraded objects - - *``num_rd`` (int) - bytes read - - *``num_rd_kb`` (int) - kbytes read - - *``num_wr`` (int) - bytes written - - *``num_wr_kb`` (int) - kbytes written - """ - self.require_ioctx_open() - stats = rados_pool_stat_t() - ret = run_in_thread(self.librados.rados_ioctx_pool_stat, - (self.io, byref(stats))) - if ret < 0: - raise make_ex(ret, "Ioctx.get_stats(%s): get_stats failed" % self.name) - return {'num_bytes': stats.num_bytes, - 'num_kb': stats.num_kb, - 'num_objects': stats.num_objects, - 'num_object_clones': stats.num_object_clones, - 'num_object_copies': stats.num_object_copies, - "num_objects_missing_on_primary": stats.num_objects_missing_on_primary, - "num_objects_unfound": stats.num_objects_unfound, - "num_objects_degraded": stats.num_objects_degraded, - "num_rd": stats.num_rd, - "num_rd_kb": stats.num_rd_kb, - "num_wr": stats.num_wr, - "num_wr_kb": stats.num_wr_kb } - - def remove_object(self, key): - """ - Delete an object - - This does not delete any snapshots of the object. - - :param key: the name of the object to delete - :type key: str - - :raises: :class:`TypeError` - :raises: :class:`Error` - :returns: bool - True on success - """ - self.require_ioctx_open() - if not isinstance(key, str): - raise TypeError('key must be a string') - ret = run_in_thread(self.librados.rados_remove, - (self.io, c_char_p(key))) - if ret < 0: - raise make_ex(ret, "Failed to remove '%s'" % key) - return True - - def trunc(self, key, size): - """ - Resize an object - - If this enlarges the object, the new area is logically filled with - zeroes. If this shrinks the object, the excess data is removed. - - :param key: the name of the object to resize - :type key: str - :param size: the new size of the object in bytes - :type size: int - - :raises: :class:`TypeError` - :raises: :class:`Error` - :returns: int - 0 on success, otherwise raises error - """ - - self.require_ioctx_open() - if not isinstance(key, str): - raise TypeError('key must be a string') - ret = run_in_thread(self.librados.rados_trunc, - (self.io, c_char_p(key), c_uint64(size))) - if ret < 0: - raise make_ex(ret, "Ioctx.trunc(%s): failed to truncate %s" % (self.name, key)) - return ret - - def stat(self, key): - """ - Get object stats (size/mtime) - - :param key: the name of the object to get stats from - :type key: str - - :raises: :class:`TypeError` - :raises: :class:`Error` - :returns: (size,timestamp) - """ - self.require_ioctx_open() - if not isinstance(key, str): - raise TypeError('key must be a string') - psize = c_uint64() - pmtime = c_uint64() - - ret = run_in_thread(self.librados.rados_stat, - (self.io, c_char_p(key), pointer(psize), - pointer(pmtime))) - if ret < 0: - raise make_ex(ret, "Failed to stat %r" % key) - return psize.value, time.localtime(pmtime.value) - - def get_xattr(self, key, xattr_name): - """ - Get the value of an extended attribute on an object. - - :param key: the name of the object to get xattr from - :type key: str - :param xattr_name: which extended attribute to read - :type xattr_name: str - - :raises: :class:`TypeError` - :raises: :class:`Error` - :returns: str - value of the xattr - """ - self.require_ioctx_open() - if not isinstance(xattr_name, str): - raise TypeError('xattr_name must be a string') - ret_length = 4096 - while ret_length < 4096 * 1024 * 1024: - ret_buf = create_string_buffer(ret_length) - ret = run_in_thread(self.librados.rados_getxattr, - (self.io, c_char_p(key), c_char_p(xattr_name), - ret_buf, c_size_t(ret_length))) - if (ret == -errno.ERANGE): - ret_length *= 2 - elif ret < 0: - raise make_ex(ret, "Failed to get xattr %r" % xattr_name) - else: - break - return ctypes.string_at(ret_buf, ret) - - def get_xattrs(self, oid): - """ - Start iterating over xattrs on an object. - - :param oid: the name of the object to get xattrs from - :type key: str - - :raises: :class:`TypeError` - :raises: :class:`Error` - :returns: XattrIterator - """ - self.require_ioctx_open() - if not isinstance(oid, str): - raise TypeError('oid must be a string') - it = c_void_p(0) - ret = run_in_thread(self.librados.rados_getxattrs, - (self.io, oid, byref(it))) - if ret != 0: - raise make_ex(ret, "Failed to get rados xattrs for object %r" % oid) - return XattrIterator(self, it, oid) - - def set_xattr(self, key, xattr_name, xattr_value): - """ - Set an extended attribute on an object. - - :param key: the name of the object to set xattr to - :type key: str - :param xattr_name: which extended attribute to set - :type xattr_name: str - :param xattr_value: the value of the extended attribute - :type xattr_value: str - - :raises: :class:`TypeError` - :raises: :class:`Error` - :returns: bool - True on success, otherwise raise an error - """ - self.require_ioctx_open() - if not isinstance(key, str): - raise TypeError('key must be a string') - if not isinstance(xattr_name, str): - raise TypeError('xattr_name must be a string') - if not isinstance(xattr_value, str): - raise TypeError('xattr_value must be a string') - ret = run_in_thread(self.librados.rados_setxattr, - (self.io, c_char_p(key), c_char_p(xattr_name), - c_char_p(xattr_value), c_size_t(len(xattr_value)))) - if ret < 0: - raise make_ex(ret, "Failed to set xattr %r" % xattr_name) - return True - - def rm_xattr(self, key, xattr_name): - """ - Removes an extended attribute on from an object. - - :param key: the name of the object to remove xattr from - :type key: str - :param xattr_name: which extended attribute to remove - :type xattr_name: str - - :raises: :class:`TypeError` - :raises: :class:`Error` - :returns: bool - True on success, otherwise raise an error - """ - self.require_ioctx_open() - if not isinstance(key, str): - raise TypeError('key must be a string') - if not isinstance(xattr_name, str): - raise TypeError('xattr_name must be a string') - ret = run_in_thread(self.librados.rados_rmxattr, - (self.io, c_char_p(key), c_char_p(xattr_name))) - if ret < 0: - raise make_ex(ret, "Failed to delete key %r xattr %r" % - (key, xattr_name)) - return True - - def list_objects(self): - """ - Get ObjectIterator on rados.Ioctx object. - - :returns: ObjectIterator - """ - self.require_ioctx_open() - return ObjectIterator(self) - - def list_snaps(self): - """ - Get SnapIterator on rados.Ioctx object. - - :returns: SnapIterator - """ - self.require_ioctx_open() - return SnapIterator(self) - - def create_snap(self, snap_name): - """ - Create a pool-wide snapshot - - :param snap_name: the name of the snapshot - :type snap_name: str - - :raises: :class:`TypeError` - :raises: :class:`Error` - """ - self.require_ioctx_open() - if not isinstance(snap_name, str): - raise TypeError('snap_name must be a string') - ret = run_in_thread(self.librados.rados_ioctx_snap_create, - (self.io, c_char_p(snap_name))) - if (ret != 0): - raise make_ex(ret, "Failed to create snap %s" % snap_name) - - def remove_snap(self, snap_name): - """ - Removes a pool-wide snapshot - - :param snap_name: the name of the snapshot - :type snap_name: str - - :raises: :class:`TypeError` - :raises: :class:`Error` - """ - self.require_ioctx_open() - if not isinstance(snap_name, str): - raise TypeError('snap_name must be a string') - ret = run_in_thread(self.librados.rados_ioctx_snap_remove, - (self.io, c_char_p(snap_name))) - if (ret != 0): - raise make_ex(ret, "Failed to remove snap %s" % snap_name) - - def lookup_snap(self, snap_name): - """ - Get the id of a pool snapshot - - :param snap_name: the name of the snapshot to lookop - :type snap_name: str - - :raises: :class:`TypeError` - :raises: :class:`Error` - :returns: Snap - on success - """ - self.require_ioctx_open() - if not isinstance(snap_name, str): - raise TypeError('snap_name must be a string') - snap_id = c_uint64() - ret = run_in_thread(self.librados.rados_ioctx_snap_lookup, - (self.io, c_char_p(snap_name), byref(snap_id))) - if (ret != 0): - raise make_ex(ret, "Failed to lookup snap %s" % snap_name) - return Snap(self, snap_name, snap_id) - - def get_last_version(self): - """ - Return the version of the last object read or written to. - - This exposes the internal version number of the last object read or - written via this io context - - :returns: version of the last object used - """ - self.require_ioctx_open() - return run_in_thread(self.librados.rados_get_last_version, (self.io,)) - -def set_object_locator(func): - def retfunc(self, *args, **kwargs): - if self.locator_key is not None: - old_locator = self.ioctx.get_locator_key() - self.ioctx.set_locator_key(self.locator_key) - retval = func(self, *args, **kwargs) - self.ioctx.set_locator_key(old_locator) - return retval - else: - return func(self, *args, **kwargs) - return retfunc - -class Object(object): - """Rados object wrapper, makes the object look like a file""" - def __init__(self, ioctx, key, locator_key=None): - self.key = key - self.ioctx = ioctx - self.offset = 0 - self.state = "exists" - self.locator_key = locator_key - - def __str__(self): - return "rados.Object(ioctx=%s,key=%s)" % (str(self.ioctx), self.key) - - def require_object_exists(self): - if self.state != "exists": - raise ObjectStateError("The object is %s" % self.state) - - @set_object_locator - def read(self, length = 1024*1024): - self.require_object_exists() - ret = self.ioctx.read(self.key, length, self.offset) - self.offset += len(ret) - return ret - - @set_object_locator - def write(self, string_to_write): - self.require_object_exists() - ret = self.ioctx.write(self.key, string_to_write, self.offset) - self.offset += ret - return ret - - @set_object_locator - def remove(self): - self.require_object_exists() - self.ioctx.remove_object(self.key) - self.state = "removed" - - @set_object_locator - def stat(self): - self.require_object_exists() - return self.ioctx.stat(self.key) - - def seek(self, position): - self.require_object_exists() - self.offset = position - - @set_object_locator - def get_xattr(self, xattr_name): - self.require_object_exists() - return self.ioctx.get_xattr(self.key, xattr_name) - - @set_object_locator - def get_xattrs(self): - self.require_object_exists() - return self.ioctx.get_xattrs(self.key) - - @set_object_locator - def set_xattr(self, xattr_name, xattr_value): - self.require_object_exists() - return self.ioctx.set_xattr(self.key, xattr_name, xattr_value) - - @set_object_locator - def rm_xattr(self, xattr_name): - self.require_object_exists() - return self.ioctx.rm_xattr(self.key, xattr_name) - -MONITOR_LEVELS = [ - "debug", - "info", - "warn", "warning", - "err", "error", - "sec", - ] - - -class MonitorLog(object): - """ - For watching cluster log messages. Instantiate an object and keep - it around while callback is periodically called. Construct with - 'level' to monitor 'level' messages (one of MONITOR_LEVELS). - arg will be passed to the callback. - - callback will be called with: - arg (given to __init__) - line (the full line, including timestamp, who, level, msg) - who (which entity issued the log message) - timestamp_sec (sec of a struct timespec) - timestamp_nsec (sec of a struct timespec) - seq (sequence number) - level (string representing the level of the log message) - msg (the message itself) - callback's return value is ignored - """ - - def monitor_log_callback(self, arg, line, who, sec, nsec, seq, level, msg): - """ - Local callback wrapper, in case we decide to do something - """ - self.callback(arg, line, who, sec, nsec, seq, level, msg) - return 0 - - def __init__(self, cluster, level, callback, arg): - if level not in MONITOR_LEVELS: - raise LogicError("invalid monitor level " + level) - if not callable(callback): - raise LogicError("callback must be a callable function") - self.level = level - self.callback = callback - self.arg = arg - callback_factory = CFUNCTYPE(c_int, # return type (really void) - c_void_p, # arg - c_char_p, # line - c_char_p, # who - c_uint64, # timestamp_sec - c_uint64, # timestamp_nsec - c_ulong, # seq - c_char_p, # level - c_char_p) # msg - self.internal_callback = callback_factory(self.monitor_log_callback) - - r = run_in_thread(cluster.librados.rados_monitor_log, - (cluster.cluster, level, self.internal_callback, arg)) - if r: - raise make_ex(r, 'error calling rados_monitor_log') diff --git a/src/pybind/rados/rados.py b/src/pybind/rados/rados.py index 7768f8c39d3..a0e5bf42ba9 100644 --- a/src/pybind/rados/rados.py +++ b/src/pybind/rados/rados.py @@ -294,6 +294,19 @@ Rados object in state %s." % (self.state)) retargs = [a for a in cretargs if a is not None] return retargs + def conf_parse_env(self, var='CEPH_ARGS'): + """ + Parse known arguments from an environment variable, normally + CEPH_ARGS. + """ + self.require_state("configuring", "connected") + if not var: + return + ret = run_in_thread(self.librados.rados_conf_parse_env, + (self.cluster, c_char_p(var))) + if (ret != 0): + raise make_ex(ret, "error calling conf_parse_env") + def conf_get(self, option): """ Get the value of a configuration option diff --git a/src/pybind/setup.py b/src/pybind/setup.py deleted file mode 100644 index 6814ba09d87..00000000000 --- a/src/pybind/setup.py +++ /dev/null @@ -1,32 +0,0 @@ -from setuptools import setup -import os - - -def long_description(): - readme = os.path.join(os.path.dirname(__file__), 'README.rst') - return open(readme).read() - - -setup( - name = 'ceph', - description = 'Bindings for Ceph', - packages = ['ceph'], - author = 'Inktank', - author_email = 'ceph-devel@vger.kernel.org', - version = '0.0.1', #XXX Fix version - license = "GPLv2", - zip_safe = False, - keywords = "ceph, bindings, api, cli", - long_description = "", #XXX Long description should come from the README.rst - classifiers = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', - 'Topic :: Software Development :: Libraries', - 'Topic :: Utilities', - 'Topic :: System :: Filesystems', - 'Operating System :: POSIX', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - ], -) diff --git a/src/pybind/tox.ini b/src/pybind/tox.ini deleted file mode 100644 index 9c63bb9d884..00000000000 --- a/src/pybind/tox.ini +++ /dev/null @@ -1,6 +0,0 @@ -[tox] -envlist = py26, py27 - -[testenv] -deps= -commands= -- cgit v1.2.1 From 5851bfacc070a987f1327d57886f445f8fc8e3af Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 16 Aug 2013 14:05:28 -0400 Subject: make a ceph-rest module Signed-off-by: Alfredo Deza --- src/pybind/ceph-rest/MANIFEST.in | 2 + src/pybind/ceph-rest/README.rst | 0 src/pybind/ceph-rest/argparse.py | 1110 ---------------------------- src/pybind/ceph-rest/ceph_rest.py | 499 ------------- src/pybind/ceph-rest/ceph_rest/__init__.py | 499 +++++++++++++ src/pybind/ceph-rest/ceph_rest/argparse.py | 1110 ++++++++++++++++++++++++++++ src/pybind/ceph-rest/setup.py | 33 + src/pybind/ceph-rest/tox.ini | 6 + 8 files changed, 1650 insertions(+), 1609 deletions(-) create mode 100644 src/pybind/ceph-rest/MANIFEST.in create mode 100644 src/pybind/ceph-rest/README.rst delete mode 100644 src/pybind/ceph-rest/argparse.py delete mode 100755 src/pybind/ceph-rest/ceph_rest.py create mode 100755 src/pybind/ceph-rest/ceph_rest/__init__.py create mode 100644 src/pybind/ceph-rest/ceph_rest/argparse.py create mode 100644 src/pybind/ceph-rest/setup.py create mode 100644 src/pybind/ceph-rest/tox.ini diff --git a/src/pybind/ceph-rest/MANIFEST.in b/src/pybind/ceph-rest/MANIFEST.in new file mode 100644 index 00000000000..3c01b12a5a5 --- /dev/null +++ b/src/pybind/ceph-rest/MANIFEST.in @@ -0,0 +1,2 @@ +include setup.py +include README.rst diff --git a/src/pybind/ceph-rest/README.rst b/src/pybind/ceph-rest/README.rst new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/ceph-rest/argparse.py b/src/pybind/ceph-rest/argparse.py deleted file mode 100644 index 1f6e90b6c1d..00000000000 --- a/src/pybind/ceph-rest/argparse.py +++ /dev/null @@ -1,1110 +0,0 @@ -""" -Types and routines used by the ceph CLI as well as the RESTful -interface. These have to do with querying the daemons for -command-description information, validating user command input against -those descriptions, and submitting the command to the appropriate -daemon. - -Copyright (C) 2013 Inktank Storage, Inc. - -LGPL2. See file COPYING. -""" -import copy -import json -import os -import pprint -import re -import socket -import stat -import sys -import types -import uuid - -class ArgumentError(Exception): - """ - Something wrong with arguments - """ - pass - -class ArgumentNumber(ArgumentError): - """ - Wrong number of a repeated argument - """ - pass - -class ArgumentFormat(ArgumentError): - """ - Argument value has wrong format - """ - pass - -class ArgumentValid(ArgumentError): - """ - Argument value is otherwise invalid (doesn't match choices, for instance) - """ - pass - -class ArgumentTooFew(ArgumentError): - """ - Fewer arguments than descriptors in signature; may mean to continue - the search, so gets a special exception type - """ - -class ArgumentPrefix(ArgumentError): - """ - Special for mismatched prefix; less severe, don't report by default - """ - pass - -class JsonFormat(Exception): - """ - some syntactic or semantic issue with the JSON - """ - pass - -class CephArgtype(object): - """ - Base class for all Ceph argument types - - Instantiating an object sets any validation parameters - (allowable strings, numeric ranges, etc.). The 'valid' - method validates a string against that initialized instance, - throwing ArgumentError if there's a problem. - """ - def __init__(self, **kwargs): - """ - set any per-instance validation parameters here - from kwargs (fixed string sets, integer ranges, etc) - """ - pass - - def valid(self, s, partial=False): - """ - Run validation against given string s (generally one word); - partial means to accept partial string matches (begins-with). - If cool, set self.val to the value that should be returned - (a copy of the input string, or a numeric or boolean interpretation - thereof, for example) - if not, throw ArgumentError(msg-as-to-why) - """ - self.val = s - - def __repr__(self): - """ - return string representation of description of type. Note, - this is not a representation of the actual value. Subclasses - probably also override __str__() to give a more user-friendly - 'name/type' description for use in command format help messages. - """ - a = '' - if hasattr(self, 'typeargs'): - a = self.typeargs - return '{0}(\'{1}\')'.format(self.__class__.__name__, a) - - def __str__(self): - """ - where __repr__ (ideally) returns a string that could be used to - reproduce the object, __str__ returns one you'd like to see in - print messages. Use __str__ to format the argtype descriptor - as it would be useful in a command usage message. - """ - return '<{0}>'.format(self.__class__.__name__) - -class CephInt(CephArgtype): - """ - range-limited integers, [+|-][0-9]+ or 0x[0-9a-f]+ - range: list of 1 or 2 ints, [min] or [min,max] - """ - def __init__(self, range=''): - if range == '': - self.range = list() - else: - self.range = list(range.split('|')) - self.range = map(long, self.range) - - def valid(self, s, partial=False): - try: - val = long(s) - except ValueError: - raise ArgumentValid("{0} doesn't represent an int".format(s)) - if len(self.range) == 2: - if val < self.range[0] or val > self.range[1]: - raise ArgumentValid("{0} not in range {1}".format(val, self.range)) - elif len(self.range) == 1: - if val < self.range[0]: - raise ArgumentValid("{0} not in range {1}".format(val, self.range)) - self.val = val - - def __str__(self): - r = '' - if len(self.range) == 1: - r = '[{0}-]'.format(self.range[0]) - if len(self.range) == 2: - r = '[{0}-{1}]'.format(self.range[0], self.range[1]) - - return ''.format(r) - - -class CephFloat(CephArgtype): - """ - range-limited float type - range: list of 1 or 2 floats, [min] or [min, max] - """ - def __init__(self, range=''): - if range == '': - self.range = list() - else: - self.range = list(range.split('|')) - self.range = map(float, self.range) - - def valid(self, s, partial=False): - try: - val = float(s) - except ValueError: - raise ArgumentValid("{0} doesn't represent a float".format(s)) - if len(self.range) == 2: - if val < self.range[0] or val > self.range[1]: - raise ArgumentValid("{0} not in range {1}".format(val, self.range)) - elif len(self.range) == 1: - if val < self.range[0]: - raise ArgumentValid("{0} not in range {1}".format(val, self.range)) - self.val = val - - def __str__(self): - r = '' - if len(self.range) == 1: - r = '[{0}-]'.format(self.range[0]) - if len(self.range) == 2: - r = '[{0}-{1}]'.format(self.range[0], self.range[1]) - return ''.format(r) - -class CephString(CephArgtype): - """ - String; pretty generic. goodchars is a RE char class of valid chars - """ - def __init__(self, goodchars=''): - from string import printable - try: - re.compile(goodchars) - except: - raise ValueError('CephString(): "{0}" is not a valid RE'.\ - format(goodchars)) - self.goodchars = goodchars - self.goodset = frozenset( - [c for c in printable if re.match(goodchars, c)] - ) - - def valid(self, s, partial=False): - sset = set(s) - if self.goodset and not sset <= self.goodset: - raise ArgumentFormat("invalid chars {0} in {1}".\ - format(''.join(sset - self.goodset), s)) - self.val = s - - def __str__(self): - b = '' - if self.goodchars: - b += '(goodchars {0})'.format(self.goodchars) - return ''.format(b) - -class CephSocketpath(CephArgtype): - """ - Admin socket path; check that it's readable and S_ISSOCK - """ - def valid(self, s, partial=False): - mode = os.stat(s).st_mode - if not stat.S_ISSOCK(mode): - raise ArgumentValid('socket path {0} is not a socket'.format(s)) - self.val = s - - def __str__(self): - return '' - -class CephIPAddr(CephArgtype): - """ - IP address (v4 or v6) with optional port - """ - def valid(self, s, partial=False): - # parse off port, use socket to validate addr - type = 6 - if s.startswith('['): - type = 6 - elif s.find('.') != -1: - type = 4 - if type == 4: - port = s.find(':') - if (port != -1): - a = s[:port] - p = s[port+1:] - if int(p) > 65535: - raise ArgumentValid('{0}: invalid IPv4 port'.format(p)) - else: - a = s - p = None - try: - socket.inet_pton(socket.AF_INET, a) - except: - raise ArgumentValid('{0}: invalid IPv4 address'.format(a)) - else: - # v6 - if s.startswith('['): - end = s.find(']') - if end == -1: - raise ArgumentFormat('{0} missing terminating ]'.format(s)) - if s[end+1] == ':': - try: - p = int(s[end+2]) - except: - raise ArgumentValid('{0}: bad port number'.format(s)) - a = s[1:end] - else: - a = s - p = None - try: - socket.inet_pton(socket.AF_INET6, a) - except: - raise ArgumentValid('{0} not valid IPv6 address'.format(s)) - if p is not None and long(p) > 65535: - raise ArgumentValid("{0} not a valid port number".format(p)) - self.val = s - self.addr = a - self.port = p - - def __str__(self): - return '' - -class CephEntityAddr(CephIPAddr): - """ - EntityAddress, that is, IP address[/nonce] - """ - def valid(self, s, partial=False): - nonce = None - if '/' in s: - ip, nonce = s.split('/') - else: - ip = s - super(self.__class__, self).valid(ip) - if nonce: - nonce_long = None - try: - nonce_long = long(nonce) - except ValueError: - pass - if nonce_long is None or nonce_long < 0: - raise ArgumentValid( - '{0}: invalid entity, nonce {1} not integer > 0'.\ - format(s, nonce) - ) - self.val = s - - def __str__(self): - return '' - -class CephPoolname(CephArgtype): - """ - Pool name; very little utility - """ - def __str__(self): - return '' - -class CephObjectname(CephArgtype): - """ - Object name. Maybe should be combined with Pool name as they're always - present in pairs, and then could be checked for presence - """ - def __str__(self): - return '' - -class CephPgid(CephArgtype): - """ - pgid, in form N.xxx (N = pool number, xxx = hex pgnum) - """ - def valid(self, s, partial=False): - if s.find('.') == -1: - raise ArgumentFormat('pgid has no .') - poolid, pgnum = s.split('.') - if poolid < 0: - raise ArgumentFormat('pool {0} < 0'.format(poolid)) - try: - pgnum = int(pgnum, 16) - except: - raise ArgumentFormat('pgnum {0} not hex integer'.format(pgnum)) - self.val = s - - def __str__(self): - return '' - -class CephName(CephArgtype): - """ - Name (type.id) where: - type is osd|mon|client|mds - id is a base10 int, if type == osd, or a string otherwise - - Also accept '*' - """ - def __init__(self): - self.nametype = None - self.nameid = None - - def valid(self, s, partial=False): - if s == '*': - self.val = s - return - if s.find('.') == -1: - raise ArgumentFormat('CephName: no . in {0}'.format(s)) - else: - t, i = s.split('.') - if not t in ('osd', 'mon', 'client', 'mds'): - raise ArgumentValid('unknown type ' + t) - if t == 'osd': - if i != '*': - try: - i = int(i) - except: - raise ArgumentFormat('osd id ' + i + ' not integer') - self.nametype = t - self.val = s - self.nameid = i - - def __str__(self): - return '' - -class CephOsdName(CephArgtype): - """ - Like CephName, but specific to osds: allow alone - - osd., or , or *, where id is a base10 int - """ - def __init__(self): - self.nametype = None - self.nameid = None - - def valid(self, s, partial=False): - if s == '*': - self.val = s - return - if s.find('.') != -1: - t, i = s.split('.') - if t != 'osd': - raise ArgumentValid('unknown type ' + t) - else: - t = 'osd' - i = s - try: - i = int(i) - except: - raise ArgumentFormat('osd id ' + i + ' not integer') - self.nametype = t - self.nameid = i - self.val = i - - def __str__(self): - return '' - -class CephChoices(CephArgtype): - """ - Set of string literals; init with valid choices - """ - def __init__(self, strings='', **kwargs): - self.strings = strings.split('|') - - def valid(self, s, partial=False): - if not partial: - if not s in self.strings: - # show as __str__ does: {s1|s2..} - raise ArgumentValid("{0} not in {1}".format(s, self)) - self.val = s - return - - # partial - for t in self.strings: - if t.startswith(s): - self.val = s - return - raise ArgumentValid("{0} not in {1}". format(s, self)) - - def __str__(self): - if len(self.strings) == 1: - return '{0}'.format(self.strings[0]) - else: - return '{0}'.format('|'.join(self.strings)) - -class CephFilepath(CephArgtype): - """ - Openable file - """ - def valid(self, s, partial=False): - try: - f = open(s, 'a+') - except Exception as e: - raise ArgumentValid('can\'t open {0}: {1}'.format(s, e)) - f.close() - self.val = s - - def __str__(self): - return '' - -class CephFragment(CephArgtype): - """ - 'Fragment' ??? XXX - """ - def valid(self, s, partial=False): - if s.find('/') == -1: - raise ArgumentFormat('{0}: no /'.format(s)) - val, bits = s.split('/') - # XXX is this right? - if not val.startswith('0x'): - raise ArgumentFormat("{0} not a hex integer".format(val)) - try: - long(val) - except: - raise ArgumentFormat('can\'t convert {0} to integer'.format(val)) - try: - long(bits) - except: - raise ArgumentFormat('can\'t convert {0} to integer'.format(bits)) - self.val = s - - def __str__(self): - return "" - - -class CephUUID(CephArgtype): - """ - CephUUID: pretty self-explanatory - """ - def valid(self, s, partial=False): - try: - uuid.UUID(s) - except Exception as e: - raise ArgumentFormat('invalid UUID {0}: {1}'.format(s, e)) - self.val = s - - def __str__(self): - return '' - - -class CephPrefix(CephArgtype): - """ - CephPrefix: magic type for "all the first n fixed strings" - """ - def __init__(self, prefix=''): - self.prefix = prefix - - def valid(self, s, partial=False): - if partial: - if self.prefix.startswith(s): - self.val = s - return - else: - if (s == self.prefix): - self.val = s - return - - raise ArgumentPrefix("no match for {0}".format(s)) - - def __str__(self): - return self.prefix - - -class argdesc(object): - """ - argdesc(typename, name='name', n=numallowed|N, - req=False, helptext=helptext, **kwargs (type-specific)) - - validation rules: - typename: type(**kwargs) will be constructed - later, type.valid(w) will be called with a word in that position - - name is used for parse errors and for constructing JSON output - n is a numeric literal or 'n|N', meaning "at least one, but maybe more" - req=False means the argument need not be present in the list - helptext is the associated help for the command - anything else are arguments to pass to the type constructor. - - self.instance is an instance of type t constructed with typeargs. - - valid() will later be called with input to validate against it, - and will store the validated value in self.instance.val for extraction. - """ - def __init__(self, t, name=None, n=1, req=True, **kwargs): - if isinstance(t, types.StringTypes): - self.t = CephPrefix - self.typeargs = {'prefix':t} - self.req = True - else: - self.t = t - self.typeargs = kwargs - self.req = bool(req == True or req == 'True') - - self.name = name - self.N = (n in ['n', 'N']) - if self.N: - self.n = 1 - else: - self.n = int(n) - self.instance = self.t(**self.typeargs) - - def __repr__(self): - r = 'argdesc(' + str(self.t) + ', ' - internals = ['N', 'typeargs', 'instance', 't'] - for (k, v) in self.__dict__.iteritems(): - if k.startswith('__') or k in internals: - pass - else: - # undo modification from __init__ - if k == 'n' and self.N: - v = 'N' - r += '{0}={1}, '.format(k, v) - for (k, v) in self.typeargs.iteritems(): - r += '{0}={1}, '.format(k, v) - return r[:-2] + ')' - - def __str__(self): - if ((self.t == CephChoices and len(self.instance.strings) == 1) - or (self.t == CephPrefix)): - s = str(self.instance) - else: - s = '{0}({1})'.format(self.name, str(self.instance)) - if self.N: - s += ' [' + str(self.instance) + '...]' - if not self.req: - s = '{' + s + '}' - return s - - def helpstr(self): - """ - like str(), but omit parameter names (except for CephString, - which really needs them) - """ - if self.t == CephString: - chunk = '<{0}>'.format(self.name) - else: - chunk = str(self.instance) - s = chunk - if self.N: - s += ' [' + chunk + '...]' - if not self.req: - s = '{' + s + '}' - return s - -def concise_sig(sig): - """ - Return string representation of sig useful for syntax reference in help - """ - return ' '.join([d.helpstr() for d in sig]) - -def descsort(sh1, sh2): - """ - sort descriptors by prefixes, defined as the concatenation of all simple - strings in the descriptor; this works out to just the leading strings. - """ - return cmp(concise_sig(sh1['sig']), concise_sig(sh2['sig'])) - -def parse_funcsig(sig): - """ - parse a single descriptor (array of strings or dicts) into a - dict of function descriptor/validators (objects of CephXXX type) - """ - newsig = [] - argnum = 0 - for desc in sig: - argnum += 1 - if isinstance(desc, types.StringTypes): - t = CephPrefix - desc = {'type':t, 'name':'prefix', 'prefix':desc} - else: - # not a simple string, must be dict - if not 'type' in desc: - s = 'JSON descriptor {0} has no type'.format(sig) - raise JsonFormat(s) - # look up type string in our globals() dict; if it's an - # object of type types.TypeType, it must be a - # locally-defined class. otherwise, we haven't a clue. - if desc['type'] in globals(): - t = globals()[desc['type']] - if type(t) != types.TypeType: - s = 'unknown type {0}'.format(desc['type']) - raise JsonFormat(s) - else: - s = 'unknown type {0}'.format(desc['type']) - raise JsonFormat(s) - - kwargs = dict() - for key, val in desc.items(): - if key not in ['type', 'name', 'n', 'req']: - kwargs[key] = val - newsig.append(argdesc(t, - name=desc.get('name', None), - n=desc.get('n', 1), - req=desc.get('req', True), - **kwargs)) - return newsig - - -def parse_json_funcsigs(s, consumer): - """ - A function signature is mostly an array of argdesc; it's represented - in JSON as - { - "cmd001": {"sig":[ "type": type, "name": name, "n": num, "req":true|false ], "help":helptext, "module":modulename, "perm":perms, "avail":availability} - . - . - . - ] - - A set of sigs is in an dict mapped by a unique number: - { - "cmd1": { - "sig": ["type.. ], "help":helptext... - } - "cmd2"{ - "sig": [.. ], "help":helptext... - } - } - - Parse the string s and return a dict of dicts, keyed by opcode; - each dict contains 'sig' with the array of descriptors, and 'help' - with the helptext, 'module' with the module name, 'perm' with a - string representing required permissions in that module to execute - this command (and also whether it is a read or write command from - the cluster state perspective), and 'avail' as a hint for - whether the command should be advertised by CLI, REST, or both. - If avail does not contain 'consumer', don't include the command - in the returned dict. - """ - try: - overall = json.loads(s) - except Exception as e: - print >> sys.stderr, "Couldn't parse JSON {0}: {1}".format(s, e) - raise e - sigdict = {} - for cmdtag, cmd in overall.iteritems(): - if not 'sig' in cmd: - s = "JSON descriptor {0} has no 'sig'".format(cmdtag) - raise JsonFormat(s) - # check 'avail' and possibly ignore this command - if 'avail' in cmd: - if not consumer in cmd['avail']: - continue - # rewrite the 'sig' item with the argdesc-ized version, and... - cmd['sig'] = parse_funcsig(cmd['sig']) - # just take everything else as given - sigdict[cmdtag] = cmd - return sigdict - -def validate_one(word, desc, partial=False): - """ - validate_one(word, desc, partial=False) - - validate word against the constructed instance of the type - in desc. May raise exception. If it returns false (and doesn't - raise an exception), desc.instance.val will - contain the validated value (in the appropriate type). - """ - desc.instance.valid(word, partial) - desc.numseen += 1 - if desc.N: - desc.n = desc.numseen + 1 - -def matchnum(args, signature, partial=False): - """ - matchnum(s, signature, partial=False) - - Returns number of arguments matched in s against signature. - Can be used to determine most-likely command for full or partial - matches (partial applies to string matches). - """ - words = args[:] - mysig = copy.deepcopy(signature) - matchcnt = 0 - for desc in mysig: - setattr(desc, 'numseen', 0) - while desc.numseen < desc.n: - # if there are no more arguments, return - if not words: - return matchcnt - word = words.pop(0) - - try: - validate_one(word, desc, partial) - valid = True - except ArgumentError: - # matchnum doesn't care about type of error - valid = False - - if not valid: - if not desc.req: - # this wasn't required, so word may match the next desc - words.insert(0, word) - break - else: - # it was required, and didn't match, return - return matchcnt - if desc.req: - matchcnt += 1 - return matchcnt - -def get_next_arg(desc, args): - ''' - Get either the value matching key 'desc.name' or the next arg in - the non-dict list. Return None if args are exhausted. Used in - validate() below. - ''' - arg = None - if isinstance(args, dict): - arg = args.pop(desc.name, None) - # allow 'param=param' to be expressed as 'param' - if arg == '': - arg = desc.name - # Hack, or clever? If value is a list, keep the first element, - # push rest back onto myargs for later processing. - # Could process list directly, but nesting here is already bad - if arg and isinstance(arg, list): - args[desc.name] = arg[1:] - arg = arg[0] - elif args: - arg = args.pop(0) - if arg and isinstance(arg, list): - args = arg[1:] + args - arg = arg[0] - return arg - -def store_arg(desc, d): - ''' - Store argument described by, and held in, thanks to valid(), - desc into the dictionary d, keyed by desc.name. Three cases: - - 1) desc.N is set: value in d is a list - 2) prefix: multiple args are joined with ' ' into one d{} item - 3) single prefix or other arg: store as simple value - - Used in validate() below. - ''' - if desc.N: - # value should be a list - if desc.name in d: - d[desc.name] += [desc.instance.val] - else: - d[desc.name] = [desc.instance.val] - elif (desc.t == CephPrefix) and (desc.name in d): - # prefixes' values should be a space-joined concatenation - d[desc.name] += ' ' + desc.instance.val - else: - # if first CephPrefix or any other type, just set it - d[desc.name] = desc.instance.val - -def validate(args, signature, partial=False): - """ - validate(args, signature, partial=False) - - args is a list of either words or k,v pairs representing a possible - command input following format of signature. Runs a validation; no - exception means it's OK. Return a dict containing all arguments keyed - by their descriptor name, with duplicate args per name accumulated - into a list (or space-separated value for CephPrefix). - - Mismatches of prefix are non-fatal, as this probably just means the - search hasn't hit the correct command. Mismatches of non-prefix - arguments are treated as fatal, and an exception raised. - - This matching is modified if partial is set: allow partial matching - (with partial dict returned); in this case, there are no exceptions - raised. - """ - - myargs = copy.deepcopy(args) - mysig = copy.deepcopy(signature) - reqsiglen = len([desc for desc in mysig if desc.req]) - matchcnt = 0 - d = dict() - for desc in mysig: - setattr(desc, 'numseen', 0) - while desc.numseen < desc.n: - myarg = get_next_arg(desc, myargs) - - # no arg, but not required? Continue consuming mysig - # in case there are later required args - if not myarg and not desc.req: - break - - # out of arguments for a required param? - # Either return (if partial validation) or raise - if not myarg and desc.req: - if desc.N and desc.numseen < 1: - # wanted N, didn't even get 1 - if partial: - return d - raise ArgumentNumber( - 'saw {0} of {1}, expected at least 1'.\ - format(desc.numseen, desc) - ) - elif not desc.N and desc.numseen < desc.n: - # wanted n, got too few - if partial: - return d - # special-case the "0 expected 1" case - if desc.numseen == 0 and desc.n == 1: - raise ArgumentNumber( - 'missing required parameter {0}'.format(desc) - ) - raise ArgumentNumber( - 'saw {0} of {1}, expected {2}'.\ - format(desc.numseen, desc, desc.n) - ) - break - - # Have an arg; validate it - try: - validate_one(myarg, desc) - valid = True - except ArgumentError as e: - valid = False - if not valid: - # argument mismatch - if not desc.req: - # if not required, just push back; it might match - # the next arg - print >> sys.stderr, myarg, 'not valid: ', str(e) - myargs.insert(0, myarg) - break - else: - # hm, it was required, so time to return/raise - if partial: - return d - raise e - - # Whew, valid arg acquired. Store in dict - matchcnt += 1 - store_arg(desc, d) - - # Done with entire list of argdescs - if matchcnt < reqsiglen: - raise ArgumentTooFew("not enough arguments given") - - if myargs and not partial: - raise ArgumentError("unused arguments: " + str(myargs)) - - # Finally, success - return d - -def cmdsiglen(sig): - sigdict = sig.values() - assert len(sigdict) == 1 - return len(sig.values()[0]['sig']) - -def validate_command(sigdict, args, verbose=False): - """ - turn args into a valid dictionary ready to be sent off as JSON, - validated against sigdict. - """ - found = [] - valid_dict = {} - if args: - # look for best match, accumulate possibles in bestcmds - # (so we can maybe give a more-useful error message) - best_match_cnt = 0 - bestcmds = [] - for cmdtag, cmd in sigdict.iteritems(): - sig = cmd['sig'] - matched = matchnum(args, sig, partial=True) - if (matched > best_match_cnt): - if verbose: - print >> sys.stderr, \ - "better match: {0} > {1}: {2}:{3} ".format(matched, - best_match_cnt, cmdtag, concise_sig(sig)) - best_match_cnt = matched - bestcmds = [{cmdtag:cmd}] - elif matched == best_match_cnt: - if verbose: - print >> sys.stderr, \ - "equal match: {0} > {1}: {2}:{3} ".format(matched, - best_match_cnt, cmdtag, concise_sig(sig)) - bestcmds.append({cmdtag:cmd}) - - # Sort bestcmds by number of args so we can try shortest first - # (relies on a cmdsig being key,val where val is a list of len 1) - bestcmds_sorted = sorted(bestcmds, - cmp=lambda x,y:cmp(cmdsiglen(x), cmdsiglen(y))) - - if verbose: - print >> sys.stderr, "bestcmds_sorted: " - pprint.PrettyPrinter(stream=sys.stderr).pprint(bestcmds_sorted) - - # for everything in bestcmds, look for a true match - for cmdsig in bestcmds_sorted: - for cmd in cmdsig.itervalues(): - sig = cmd['sig'] - try: - valid_dict = validate(args, sig) - found = cmd - break - except ArgumentPrefix: - # ignore prefix mismatches; we just haven't found - # the right command yet - pass - except ArgumentTooFew: - # It looked like this matched the beginning, but it - # didn't have enough args supplied. If we're out of - # cmdsigs we'll fall out unfound; if we're not, maybe - # the next one matches completely. Whine, but pass. - if verbose: - print >> sys.stderr, 'Not enough args supplied for ', \ - concise_sig(sig) - except ArgumentError as e: - # Solid mismatch on an arg (type, range, etc.) - # Stop now, because we have the right command but - # some other input is invalid - print >> sys.stderr, "Invalid command: ", str(e) - print >> sys.stderr, concise_sig(sig), ': ', cmd['help'] - return {} - if found: - break - - if not found: - print >> sys.stderr, 'no valid command found; 10 closest matches:' - for cmdsig in bestcmds[:10]: - for (cmdtag, cmd) in cmdsig.iteritems(): - print >> sys.stderr, concise_sig(cmd['sig']) - return None - - return valid_dict - -def find_cmd_target(childargs): - """ - Using a minimal validation, figure out whether the command - should be sent to a monitor or an osd. We do this before even - asking for the 'real' set of command signatures, so we can ask the - right daemon. - Returns ('osd', osdid), ('pg', pgid), or ('mon', '') - """ - sig = parse_funcsig(['tell', {'name':'target', 'type':'CephName'}]) - try: - valid_dict = validate(childargs, sig, partial=True) - except ArgumentError: - pass - else: - if len(valid_dict) == 2: - # revalidate to isolate type and id - name = CephName() - # if this fails, something is horribly wrong, as it just - # validated successfully above - name.valid(valid_dict['target']) - return name.nametype, name.nameid - - sig = parse_funcsig(['tell', {'name':'pgid', 'type':'CephPgid'}]) - try: - valid_dict = validate(childargs, sig, partial=True) - except ArgumentError: - pass - else: - if len(valid_dict) == 2: - # pg doesn't need revalidation; the string is fine - return 'pg', valid_dict['pgid'] - - sig = parse_funcsig(['pg', {'name':'pgid', 'type':'CephPgid'}]) - try: - valid_dict = validate(childargs, sig, partial=True) - except ArgumentError: - pass - else: - if len(valid_dict) == 2: - return 'pg', valid_dict['pgid'] - - return 'mon', '' - -def send_command(cluster, target=('mon', ''), cmd=None, inbuf='', timeout=0, - verbose=False): - """ - Send a command to a daemon using librados's - mon_command, osd_command, or pg_command. Any bulk input data - comes in inbuf. - - Returns (ret, outbuf, outs); ret is the return code, outbuf is - the outbl "bulk useful output" buffer, and outs is any status - or error message (intended for stderr). - - If target is osd.N, send command to that osd (except for pgid cmds) - """ - cmd = cmd or [] - try: - if target[0] == 'osd': - osdid = target[1] - - if verbose: - print >> sys.stderr, 'submit {0} to osd.{1}'.\ - format(cmd, osdid) - ret, outbuf, outs = \ - cluster.osd_command(osdid, cmd, inbuf, timeout) - - elif target[0] == 'pg': - pgid = target[1] - # pgid will already be in the command for the pg - # form, but for tell , we need to put it in - if cmd: - cmddict = json.loads(cmd[0]) - cmddict['pgid'] = pgid - else: - cmddict = dict(pgid=pgid) - cmd = [json.dumps(cmddict)] - if verbose: - print >> sys.stderr, 'submit {0} for pgid {1}'.\ - format(cmd, pgid) - ret, outbuf, outs = \ - cluster.pg_command(pgid, cmd, inbuf, timeout) - - elif target[0] == 'mon': - if verbose: - print >> sys.stderr, '{0} to {1}'.\ - format(cmd, target[0]) - if target[1] == '': - ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout) - else: - ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout, target[1]) - - except Exception as e: - raise RuntimeError('"{0}": exception {1}'.format(cmd, e)) - - return ret, outbuf, outs - -def json_command(cluster, target=('mon', ''), prefix=None, argdict=None, - inbuf='', timeout=0, verbose=False): - """ - Format up a JSON command and send it with send_command() above. - Prefix may be supplied separately or in argdict. Any bulk input - data comes in inbuf. - - If target is osd.N, send command to that osd (except for pgid cmds) - """ - cmddict = {} - if prefix: - cmddict.update({'prefix':prefix}) - if argdict: - cmddict.update(argdict) - - # grab prefix for error messages - prefix = cmddict['prefix'] - - try: - if target[0] == 'osd': - osdtarg = CephName() - osdtarget = '{0}.{1}'.format(*target) - # prefer target from cmddict if present and valid - if 'target' in cmddict: - osdtarget = cmddict.pop('target') - try: - osdtarg.valid(osdtarget) - target = ('osd', osdtarg.nameid) - except: - # use the target we were originally given - pass - - ret, outbuf, outs = send_command(cluster, target, [json.dumps(cmddict)], - inbuf, timeout, verbose) - - except Exception as e: - raise RuntimeError('"{0}": exception {1}'.format(prefix, e)) - - return ret, outbuf, outs - - diff --git a/src/pybind/ceph-rest/ceph_rest.py b/src/pybind/ceph-rest/ceph_rest.py deleted file mode 100755 index 75e61060544..00000000000 --- a/src/pybind/ceph-rest/ceph_rest.py +++ /dev/null @@ -1,499 +0,0 @@ -# vim: ts=4 sw=4 smarttab expandtab - -import errno -import json -import logging -import logging.handlers -import os -import rados -import textwrap -import xml.etree.ElementTree -import xml.sax.saxutils - -import flask -from ceph_argparse import \ - ArgumentError, CephPgid, CephOsdName, CephChoices, CephPrefix, \ - concise_sig, descsort, parse_funcsig, parse_json_funcsigs, \ - validate, json_command - -# -# Globals and defaults -# - -DEFAULT_ADDR = '0.0.0.0' -DEFAULT_PORT = '5000' -DEFAULT_ID = 'restapi' - -DEFAULT_BASEURL = '/api/v0.1' -DEFAULT_LOG_LEVEL = 'warning' -DEFAULT_LOGDIR = '/var/log/ceph' -# default client name will be 'client.' - -# 'app' must be global for decorators, etc. -APPNAME = '__main__' -app = flask.Flask(APPNAME) - -LOGLEVELS = { - 'critical':logging.CRITICAL, - 'error':logging.ERROR, - 'warning':logging.WARNING, - 'info':logging.INFO, - 'debug':logging.DEBUG, -} - -def find_up_osd(app): - ''' - Find an up OSD. Return the last one that's up. - Returns id as an int. - ''' - ret, outbuf, outs = json_command(app.ceph_cluster, prefix="osd dump", - argdict=dict(format='json')) - if ret: - raise EnvironmentError(ret, 'Can\'t get osd dump output') - try: - osddump = json.loads(outbuf) - except: - raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump') - osds = [osd['osd'] for osd in osddump['osds'] if osd['up']] - if not osds: - raise EnvironmentError(errno.ENOENT, 'No up OSDs found') - return int(osds[-1]) - - -METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} - -def api_setup(app, conf, cluster, clientname, clientid, args): - ''' - This is done globally, and cluster connection kept open for - the lifetime of the daemon. librados should assure that even - if the cluster goes away and comes back, our connection remains. - - Initialize the running instance. Open the cluster, get the command - signatures, module, perms, and help; stuff them away in the app.ceph_urls - dict. Also save app.ceph_sigdict for help() handling. - ''' - def get_command_descriptions(cluster, target=('mon','')): - ret, outbuf, outs = json_command(cluster, target, - prefix='get_command_descriptions', - timeout=30) - if ret: - err = "Can't get command descriptions: {0}".format(outs) - app.logger.error(err) - raise EnvironmentError(ret, err) - - try: - sigdict = parse_json_funcsigs(outbuf, 'rest') - except Exception as e: - err = "Can't parse command descriptions: {}".format(e) - app.logger.error(err) - raise EnvironmentError(err) - return sigdict - - app.ceph_cluster = cluster or 'ceph' - app.ceph_urls = {} - app.ceph_sigdict = {} - app.ceph_baseurl = '' - - conf = conf or '' - cluster = cluster or 'ceph' - clientid = clientid or DEFAULT_ID - clientname = clientname or 'client.' + clientid - - app.ceph_cluster = rados.Rados(name=clientname, conffile=conf) - app.ceph_cluster.conf_parse_argv(args) - app.ceph_cluster.connect() - - app.ceph_baseurl = app.ceph_cluster.conf_get('restapi_base_url') \ - or DEFAULT_BASEURL - if app.ceph_baseurl.endswith('/'): - app.ceph_baseurl = app.ceph_baseurl[:-1] - addr = app.ceph_cluster.conf_get('public_addr') or DEFAULT_ADDR - - # remove any nonce from the conf value - addr = addr.split('/')[0] - addr, port = addr.rsplit(':', 1) - addr = addr or DEFAULT_ADDR - port = port or DEFAULT_PORT - port = int(port) - - loglevel = app.ceph_cluster.conf_get('restapi_log_level') \ - or DEFAULT_LOG_LEVEL - # ceph has a default log file for daemons only; clients (like this) - # default to "". Override that for this particular client. - logfile = app.ceph_cluster.conf_get('log_file') - if not logfile: - logfile = os.path.join( - DEFAULT_LOGDIR, - '{cluster}-{clientname}.{pid}.log'.format( - cluster=cluster, - clientname=clientname, - pid=os.getpid() - ) - ) - app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile)) - app.logger.setLevel(LOGLEVELS[loglevel.lower()]) - for h in app.logger.handlers: - h.setFormatter(logging.Formatter( - '%(asctime)s %(name)s %(levelname)s: %(message)s')) - - app.ceph_sigdict = get_command_descriptions(app.ceph_cluster) - - osdid = find_up_osd(app) - if osdid: - osd_sigdict = get_command_descriptions(app.ceph_cluster, - target=('osd', int(osdid))) - - # shift osd_sigdict keys up to fit at the end of the mon's app.ceph_sigdict - maxkey = sorted(app.ceph_sigdict.keys())[-1] - maxkey = int(maxkey.replace('cmd', '')) - osdkey = maxkey + 1 - for k, v in osd_sigdict.iteritems(): - newv = v - newv['flavor'] = 'tell' - globk = 'cmd' + str(osdkey) - app.ceph_sigdict[globk] = newv - osdkey += 1 - - # app.ceph_sigdict maps "cmdNNN" to a dict containing: - # 'sig', an array of argdescs - # 'help', the helptext - # 'module', the Ceph module this command relates to - # 'perm', a 'rwx*' string representing required permissions, and also - # a hint as to whether this is a GET or POST/PUT operation - # 'avail', a comma-separated list of strings of consumers that should - # display this command (filtered by parse_json_funcsigs() above) - app.ceph_urls = {} - for cmdnum, cmddict in app.ceph_sigdict.iteritems(): - cmdsig = cmddict['sig'] - flavor = cmddict.get('flavor', 'mon') - url, params = generate_url_and_params(app, cmdsig, flavor) - perm = cmddict['perm'] - for k in METHOD_DICT.iterkeys(): - if k in perm: - methods = METHOD_DICT[k] - urldict = {'paramsig':params, - 'help':cmddict['help'], - 'module':cmddict['module'], - 'perm':perm, - 'flavor':flavor, - 'methods':methods, - } - - # app.ceph_urls contains a list of urldicts (usually only one long) - if url not in app.ceph_urls: - app.ceph_urls[url] = [urldict] - else: - # If more than one, need to make union of methods of all. - # Method must be checked in handler - methodset = set(methods) - for old_urldict in app.ceph_urls[url]: - methodset |= set(old_urldict['methods']) - methods = list(methodset) - app.ceph_urls[url].append(urldict) - - # add, or re-add, rule with all methods and urldicts - app.add_url_rule(url, url, handler, methods=methods) - url += '.' - app.add_url_rule(url, url, handler, methods=methods) - - app.logger.debug("urls added: %d", len(app.ceph_urls)) - - app.add_url_rule('/', '/', - handler, methods=['GET', 'PUT']) - return addr, port - - -def generate_url_and_params(app, sig, flavor): - ''' - Digest command signature from cluster; generate an absolute - (including app.ceph_baseurl) endpoint from all the prefix words, - and a list of non-prefix param descs - ''' - - url = '' - params = [] - # the OSD command descriptors don't include the 'tell ', so - # tack it onto the front of sig - if flavor == 'tell': - tellsig = parse_funcsig(['tell', - {'name':'target', 'type':'CephOsdName'}]) - sig = tellsig + sig - - for desc in sig: - # prefixes go in the URL path - if desc.t == CephPrefix: - url += '/' + desc.instance.prefix - # CephChoices with 1 required string (not --) do too, unless - # we've already started collecting params, in which case they - # too are params - elif desc.t == CephChoices and \ - len(desc.instance.strings) == 1 and \ - desc.req and \ - not str(desc.instance).startswith('--') and \ - not params: - url += '/' + str(desc.instance) - else: - # tell/ is a weird case; the URL includes what - # would everywhere else be a parameter - if flavor == 'tell' and \ - (desc.t, desc.name) == (CephOsdName, 'target'): - url += '/' - else: - params.append(desc) - - return app.ceph_baseurl + url, params - - -# -# end setup (import-time) functions, begin request-time functions -# - -def concise_sig_for_uri(sig, flavor): - ''' - Return a generic description of how one would send a REST request for sig - ''' - prefix = [] - args = [] - ret = '' - if flavor == 'tell': - ret = 'tell//' - for d in sig: - if d.t == CephPrefix: - prefix.append(d.instance.prefix) - else: - args.append(d.name + '=' + str(d)) - ret += '/'.join(prefix) - if args: - ret += '?' + '&'.join(args) - return ret - -def show_human_help(prefix): - ''' - Dump table showing commands matching prefix - ''' - # XXX There ought to be a better discovery mechanism than an HTML table - s = '' - - permmap = {'r':'GET', 'rw':'PUT'} - line = '' - for cmdsig in sorted(app.ceph_sigdict.itervalues(), cmp=descsort): - concise = concise_sig(cmdsig['sig']) - flavor = cmdsig.get('flavor', 'mon') - if flavor == 'tell': - concise = 'tell//' + concise - if concise.startswith(prefix): - line = ['\n') - s += ''.join(line) - - s += '
Possible commands:MethodDescription
'] - wrapped_sig = textwrap.wrap( - concise_sig_for_uri(cmdsig['sig'], flavor), 40 - ) - for sigline in wrapped_sig: - line.append(flask.escape(sigline) + '\n') - line.append('') - line.append(permmap[cmdsig['perm']]) - line.append('') - line.append(flask.escape(cmdsig['help'])) - line.append('
' - if line: - return s - else: - return '' - -@app.before_request -def log_request(): - ''' - For every request, log it. XXX Probably overkill for production - ''' - app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string) - app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values()) - -@app.route('/') -def root_redir(): - return flask.redirect(app.ceph_baseurl) - -def make_response(fmt, output, statusmsg, errorcode): - ''' - If formatted output, cobble up a response object that contains the - output and status wrapped in enclosing objects; if nonformatted, just - use output+status. Return HTTP status errorcode in any event. - ''' - response = output - if fmt: - if 'json' in fmt: - try: - native_output = json.loads(output or '[]') - response = json.dumps({"output":native_output, - "status":statusmsg}) - except: - return flask.make_response("Error decoding JSON from " + - output, 500) - elif 'xml' in fmt: - # XXX - # one is tempted to do this with xml.etree, but figuring out how - # to 'un-XML' the XML-dumped output so it can be reassembled into - # a piece of the tree here is beyond me right now. - #ET = xml.etree.ElementTree - #resp_elem = ET.Element('response') - #o = ET.SubElement(resp_elem, 'output') - #o.text = output - #s = ET.SubElement(resp_elem, 'status') - #s.text = statusmsg - #response = ET.tostring(resp_elem) - response = ''' - - - {0} - - - {1} - -'''.format(response, xml.sax.saxutils.escape(statusmsg)) - else: - if not 200 <= errorcode < 300: - response = response + '\n' + statusmsg + '\n' - - return flask.make_response(response, errorcode) - -def handler(catchall_path=None, fmt=None, target=None): - ''' - Main endpoint handler; generic for every endpoint, including catchall. - Handles the catchall, anything with <.fmt>, anything with embedded - . Partial match or ?help cause the HTML-table - "show_human_help" output. - ''' - - ep = catchall_path or flask.request.endpoint - ep = ep.replace('.', '') - - if ep[0] != '/': - ep = '/' + ep - - # demand that endpoint begin with app.ceph_baseurl - if not ep.startswith(app.ceph_baseurl): - return make_response(fmt, '', 'Page not found', 404) - - rel_ep = ep[len(app.ceph_baseurl)+1:] - - # Extensions override Accept: headers override defaults - if not fmt: - if 'application/json' in flask.request.accept_mimetypes.values(): - fmt = 'json' - elif 'application/xml' in flask.request.accept_mimetypes.values(): - fmt = 'xml' - - prefix = '' - pgid = None - cmdtarget = 'mon', '' - - if target: - # got tell/; validate osdid or pgid - name = CephOsdName() - pgidobj = CephPgid() - try: - name.valid(target) - except ArgumentError: - # try pgid - try: - pgidobj.valid(target) - except ArgumentError: - return flask.make_response("invalid osdid or pgid", 400) - else: - # it's a pgid - pgid = pgidobj.val - cmdtarget = 'pg', pgid - else: - # it's an osd - cmdtarget = name.nametype, name.nameid - - # prefix does not include tell// - prefix = ' '.join(rel_ep.split('/')[2:]).strip() - else: - # non-target command: prefix is entire path - prefix = ' '.join(rel_ep.split('/')).strip() - - # show "match as much as you gave me" help for unknown endpoints - if not ep in app.ceph_urls: - helptext = show_human_help(prefix) - if helptext: - resp = flask.make_response(helptext, 400) - resp.headers['Content-Type'] = 'text/html' - return resp - else: - return make_response(fmt, '', 'Invalid endpoint ' + ep, 400) - - found = None - exc = '' - for urldict in app.ceph_urls[ep]: - if flask.request.method not in urldict['methods']: - continue - paramsig = urldict['paramsig'] - - # allow '?help' for any specifically-known endpoint - if 'help' in flask.request.args: - response = flask.make_response('{0}: {1}'.\ - format(prefix + concise_sig(paramsig), urldict['help'])) - response.headers['Content-Type'] = 'text/plain' - return response - - # if there are parameters for this endpoint, process them - if paramsig: - args = {} - for k, l in flask.request.args.iterlists(): - if len(l) == 1: - args[k] = l[0] - else: - args[k] = l - - # is this a valid set of params? - try: - argdict = validate(args, paramsig) - found = urldict - break - except Exception as e: - exc += str(e) - continue - else: - if flask.request.args: - continue - found = urldict - argdict = {} - break - - if not found: - return make_response(fmt, '', exc + '\n', 400) - - argdict['format'] = fmt or 'plain' - argdict['module'] = found['module'] - argdict['perm'] = found['perm'] - if pgid: - argdict['pgid'] = pgid - - if not cmdtarget: - cmdtarget = ('mon', '') - - app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) - ret, outbuf, outs = json_command(app.ceph_cluster, prefix=prefix, - target=cmdtarget, - inbuf=flask.request.data, argdict=argdict) - if ret: - return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400) - - response = make_response(fmt, outbuf, outs or 'OK', 200) - if fmt: - contenttype = 'application/' + fmt.replace('-pretty','') - else: - contenttype = 'text/plain' - response.headers['Content-Type'] = contenttype - return response - -# -# Main entry point from wrapper/WSGI server: call with cmdline args, -# get back the WSGI app entry point -# -def generate_app(conf, cluster, clientname, clientid, args): - addr, port = api_setup(app, conf, cluster, clientname, clientid, args) - app.ceph_addr = addr - app.ceph_port = port - return app diff --git a/src/pybind/ceph-rest/ceph_rest/__init__.py b/src/pybind/ceph-rest/ceph_rest/__init__.py new file mode 100755 index 00000000000..a1a138d6d95 --- /dev/null +++ b/src/pybind/ceph-rest/ceph_rest/__init__.py @@ -0,0 +1,499 @@ +# vim: ts=4 sw=4 smarttab expandtab + +import errno +import json +import logging +import logging.handlers +import os +import rados +import textwrap +import xml.etree.ElementTree +import xml.sax.saxutils + +import flask +from ceph_rest.argparse import \ + ArgumentError, CephPgid, CephOsdName, CephChoices, CephPrefix, \ + concise_sig, descsort, parse_funcsig, parse_json_funcsigs, \ + validate, json_command + +# +# Globals and defaults +# + +DEFAULT_ADDR = '0.0.0.0' +DEFAULT_PORT = '5000' +DEFAULT_ID = 'restapi' + +DEFAULT_BASEURL = '/api/v0.1' +DEFAULT_LOG_LEVEL = 'warning' +DEFAULT_LOGDIR = '/var/log/ceph' +# default client name will be 'client.' + +# 'app' must be global for decorators, etc. +APPNAME = '__main__' +app = flask.Flask(APPNAME) + +LOGLEVELS = { + 'critical':logging.CRITICAL, + 'error':logging.ERROR, + 'warning':logging.WARNING, + 'info':logging.INFO, + 'debug':logging.DEBUG, +} + +def find_up_osd(app): + ''' + Find an up OSD. Return the last one that's up. + Returns id as an int. + ''' + ret, outbuf, outs = json_command(app.ceph_cluster, prefix="osd dump", + argdict=dict(format='json')) + if ret: + raise EnvironmentError(ret, 'Can\'t get osd dump output') + try: + osddump = json.loads(outbuf) + except: + raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump') + osds = [osd['osd'] for osd in osddump['osds'] if osd['up']] + if not osds: + raise EnvironmentError(errno.ENOENT, 'No up OSDs found') + return int(osds[-1]) + + +METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} + +def api_setup(app, conf, cluster, clientname, clientid, args): + ''' + This is done globally, and cluster connection kept open for + the lifetime of the daemon. librados should assure that even + if the cluster goes away and comes back, our connection remains. + + Initialize the running instance. Open the cluster, get the command + signatures, module, perms, and help; stuff them away in the app.ceph_urls + dict. Also save app.ceph_sigdict for help() handling. + ''' + def get_command_descriptions(cluster, target=('mon','')): + ret, outbuf, outs = json_command(cluster, target, + prefix='get_command_descriptions', + timeout=30) + if ret: + err = "Can't get command descriptions: {0}".format(outs) + app.logger.error(err) + raise EnvironmentError(ret, err) + + try: + sigdict = parse_json_funcsigs(outbuf, 'rest') + except Exception as e: + err = "Can't parse command descriptions: {}".format(e) + app.logger.error(err) + raise EnvironmentError(err) + return sigdict + + app.ceph_cluster = cluster or 'ceph' + app.ceph_urls = {} + app.ceph_sigdict = {} + app.ceph_baseurl = '' + + conf = conf or '' + cluster = cluster or 'ceph' + clientid = clientid or DEFAULT_ID + clientname = clientname or 'client.' + clientid + + app.ceph_cluster = rados.Rados(name=clientname, conffile=conf) + app.ceph_cluster.conf_parse_argv(args) + app.ceph_cluster.connect() + + app.ceph_baseurl = app.ceph_cluster.conf_get('restapi_base_url') \ + or DEFAULT_BASEURL + if app.ceph_baseurl.endswith('/'): + app.ceph_baseurl = app.ceph_baseurl[:-1] + addr = app.ceph_cluster.conf_get('public_addr') or DEFAULT_ADDR + + # remove any nonce from the conf value + addr = addr.split('/')[0] + addr, port = addr.rsplit(':', 1) + addr = addr or DEFAULT_ADDR + port = port or DEFAULT_PORT + port = int(port) + + loglevel = app.ceph_cluster.conf_get('restapi_log_level') \ + or DEFAULT_LOG_LEVEL + # ceph has a default log file for daemons only; clients (like this) + # default to "". Override that for this particular client. + logfile = app.ceph_cluster.conf_get('log_file') + if not logfile: + logfile = os.path.join( + DEFAULT_LOGDIR, + '{cluster}-{clientname}.{pid}.log'.format( + cluster=cluster, + clientname=clientname, + pid=os.getpid() + ) + ) + app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile)) + app.logger.setLevel(LOGLEVELS[loglevel.lower()]) + for h in app.logger.handlers: + h.setFormatter(logging.Formatter( + '%(asctime)s %(name)s %(levelname)s: %(message)s')) + + app.ceph_sigdict = get_command_descriptions(app.ceph_cluster) + + osdid = find_up_osd(app) + if osdid: + osd_sigdict = get_command_descriptions(app.ceph_cluster, + target=('osd', int(osdid))) + + # shift osd_sigdict keys up to fit at the end of the mon's app.ceph_sigdict + maxkey = sorted(app.ceph_sigdict.keys())[-1] + maxkey = int(maxkey.replace('cmd', '')) + osdkey = maxkey + 1 + for k, v in osd_sigdict.iteritems(): + newv = v + newv['flavor'] = 'tell' + globk = 'cmd' + str(osdkey) + app.ceph_sigdict[globk] = newv + osdkey += 1 + + # app.ceph_sigdict maps "cmdNNN" to a dict containing: + # 'sig', an array of argdescs + # 'help', the helptext + # 'module', the Ceph module this command relates to + # 'perm', a 'rwx*' string representing required permissions, and also + # a hint as to whether this is a GET or POST/PUT operation + # 'avail', a comma-separated list of strings of consumers that should + # display this command (filtered by parse_json_funcsigs() above) + app.ceph_urls = {} + for cmdnum, cmddict in app.ceph_sigdict.iteritems(): + cmdsig = cmddict['sig'] + flavor = cmddict.get('flavor', 'mon') + url, params = generate_url_and_params(app, cmdsig, flavor) + perm = cmddict['perm'] + for k in METHOD_DICT.iterkeys(): + if k in perm: + methods = METHOD_DICT[k] + urldict = {'paramsig':params, + 'help':cmddict['help'], + 'module':cmddict['module'], + 'perm':perm, + 'flavor':flavor, + 'methods':methods, + } + + # app.ceph_urls contains a list of urldicts (usually only one long) + if url not in app.ceph_urls: + app.ceph_urls[url] = [urldict] + else: + # If more than one, need to make union of methods of all. + # Method must be checked in handler + methodset = set(methods) + for old_urldict in app.ceph_urls[url]: + methodset |= set(old_urldict['methods']) + methods = list(methodset) + app.ceph_urls[url].append(urldict) + + # add, or re-add, rule with all methods and urldicts + app.add_url_rule(url, url, handler, methods=methods) + url += '.' + app.add_url_rule(url, url, handler, methods=methods) + + app.logger.debug("urls added: %d", len(app.ceph_urls)) + + app.add_url_rule('/', '/', + handler, methods=['GET', 'PUT']) + return addr, port + + +def generate_url_and_params(app, sig, flavor): + ''' + Digest command signature from cluster; generate an absolute + (including app.ceph_baseurl) endpoint from all the prefix words, + and a list of non-prefix param descs + ''' + + url = '' + params = [] + # the OSD command descriptors don't include the 'tell ', so + # tack it onto the front of sig + if flavor == 'tell': + tellsig = parse_funcsig(['tell', + {'name':'target', 'type':'CephOsdName'}]) + sig = tellsig + sig + + for desc in sig: + # prefixes go in the URL path + if desc.t == CephPrefix: + url += '/' + desc.instance.prefix + # CephChoices with 1 required string (not --) do too, unless + # we've already started collecting params, in which case they + # too are params + elif desc.t == CephChoices and \ + len(desc.instance.strings) == 1 and \ + desc.req and \ + not str(desc.instance).startswith('--') and \ + not params: + url += '/' + str(desc.instance) + else: + # tell/ is a weird case; the URL includes what + # would everywhere else be a parameter + if flavor == 'tell' and \ + (desc.t, desc.name) == (CephOsdName, 'target'): + url += '/' + else: + params.append(desc) + + return app.ceph_baseurl + url, params + + +# +# end setup (import-time) functions, begin request-time functions +# + +def concise_sig_for_uri(sig, flavor): + ''' + Return a generic description of how one would send a REST request for sig + ''' + prefix = [] + args = [] + ret = '' + if flavor == 'tell': + ret = 'tell//' + for d in sig: + if d.t == CephPrefix: + prefix.append(d.instance.prefix) + else: + args.append(d.name + '=' + str(d)) + ret += '/'.join(prefix) + if args: + ret += '?' + '&'.join(args) + return ret + +def show_human_help(prefix): + ''' + Dump table showing commands matching prefix + ''' + # XXX There ought to be a better discovery mechanism than an HTML table + s = '' + + permmap = {'r':'GET', 'rw':'PUT'} + line = '' + for cmdsig in sorted(app.ceph_sigdict.itervalues(), cmp=descsort): + concise = concise_sig(cmdsig['sig']) + flavor = cmdsig.get('flavor', 'mon') + if flavor == 'tell': + concise = 'tell//' + concise + if concise.startswith(prefix): + line = ['\n') + s += ''.join(line) + + s += '
Possible commands:MethodDescription
'] + wrapped_sig = textwrap.wrap( + concise_sig_for_uri(cmdsig['sig'], flavor), 40 + ) + for sigline in wrapped_sig: + line.append(flask.escape(sigline) + '\n') + line.append('') + line.append(permmap[cmdsig['perm']]) + line.append('') + line.append(flask.escape(cmdsig['help'])) + line.append('
' + if line: + return s + else: + return '' + +@app.before_request +def log_request(): + ''' + For every request, log it. XXX Probably overkill for production + ''' + app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string) + app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values()) + +@app.route('/') +def root_redir(): + return flask.redirect(app.ceph_baseurl) + +def make_response(fmt, output, statusmsg, errorcode): + ''' + If formatted output, cobble up a response object that contains the + output and status wrapped in enclosing objects; if nonformatted, just + use output+status. Return HTTP status errorcode in any event. + ''' + response = output + if fmt: + if 'json' in fmt: + try: + native_output = json.loads(output or '[]') + response = json.dumps({"output":native_output, + "status":statusmsg}) + except: + return flask.make_response("Error decoding JSON from " + + output, 500) + elif 'xml' in fmt: + # XXX + # one is tempted to do this with xml.etree, but figuring out how + # to 'un-XML' the XML-dumped output so it can be reassembled into + # a piece of the tree here is beyond me right now. + #ET = xml.etree.ElementTree + #resp_elem = ET.Element('response') + #o = ET.SubElement(resp_elem, 'output') + #o.text = output + #s = ET.SubElement(resp_elem, 'status') + #s.text = statusmsg + #response = ET.tostring(resp_elem) + response = ''' + + + {0} + + + {1} + +'''.format(response, xml.sax.saxutils.escape(statusmsg)) + else: + if not 200 <= errorcode < 300: + response = response + '\n' + statusmsg + '\n' + + return flask.make_response(response, errorcode) + +def handler(catchall_path=None, fmt=None, target=None): + ''' + Main endpoint handler; generic for every endpoint, including catchall. + Handles the catchall, anything with <.fmt>, anything with embedded + . Partial match or ?help cause the HTML-table + "show_human_help" output. + ''' + + ep = catchall_path or flask.request.endpoint + ep = ep.replace('.', '') + + if ep[0] != '/': + ep = '/' + ep + + # demand that endpoint begin with app.ceph_baseurl + if not ep.startswith(app.ceph_baseurl): + return make_response(fmt, '', 'Page not found', 404) + + rel_ep = ep[len(app.ceph_baseurl)+1:] + + # Extensions override Accept: headers override defaults + if not fmt: + if 'application/json' in flask.request.accept_mimetypes.values(): + fmt = 'json' + elif 'application/xml' in flask.request.accept_mimetypes.values(): + fmt = 'xml' + + prefix = '' + pgid = None + cmdtarget = 'mon', '' + + if target: + # got tell/; validate osdid or pgid + name = CephOsdName() + pgidobj = CephPgid() + try: + name.valid(target) + except ArgumentError: + # try pgid + try: + pgidobj.valid(target) + except ArgumentError: + return flask.make_response("invalid osdid or pgid", 400) + else: + # it's a pgid + pgid = pgidobj.val + cmdtarget = 'pg', pgid + else: + # it's an osd + cmdtarget = name.nametype, name.nameid + + # prefix does not include tell// + prefix = ' '.join(rel_ep.split('/')[2:]).strip() + else: + # non-target command: prefix is entire path + prefix = ' '.join(rel_ep.split('/')).strip() + + # show "match as much as you gave me" help for unknown endpoints + if not ep in app.ceph_urls: + helptext = show_human_help(prefix) + if helptext: + resp = flask.make_response(helptext, 400) + resp.headers['Content-Type'] = 'text/html' + return resp + else: + return make_response(fmt, '', 'Invalid endpoint ' + ep, 400) + + found = None + exc = '' + for urldict in app.ceph_urls[ep]: + if flask.request.method not in urldict['methods']: + continue + paramsig = urldict['paramsig'] + + # allow '?help' for any specifically-known endpoint + if 'help' in flask.request.args: + response = flask.make_response('{0}: {1}'.\ + format(prefix + concise_sig(paramsig), urldict['help'])) + response.headers['Content-Type'] = 'text/plain' + return response + + # if there are parameters for this endpoint, process them + if paramsig: + args = {} + for k, l in flask.request.args.iterlists(): + if len(l) == 1: + args[k] = l[0] + else: + args[k] = l + + # is this a valid set of params? + try: + argdict = validate(args, paramsig) + found = urldict + break + except Exception as e: + exc += str(e) + continue + else: + if flask.request.args: + continue + found = urldict + argdict = {} + break + + if not found: + return make_response(fmt, '', exc + '\n', 400) + + argdict['format'] = fmt or 'plain' + argdict['module'] = found['module'] + argdict['perm'] = found['perm'] + if pgid: + argdict['pgid'] = pgid + + if not cmdtarget: + cmdtarget = ('mon', '') + + app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) + ret, outbuf, outs = json_command(app.ceph_cluster, prefix=prefix, + target=cmdtarget, + inbuf=flask.request.data, argdict=argdict) + if ret: + return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400) + + response = make_response(fmt, outbuf, outs or 'OK', 200) + if fmt: + contenttype = 'application/' + fmt.replace('-pretty','') + else: + contenttype = 'text/plain' + response.headers['Content-Type'] = contenttype + return response + +# +# Main entry point from wrapper/WSGI server: call with cmdline args, +# get back the WSGI app entry point +# +def generate_app(conf, cluster, clientname, clientid, args): + addr, port = api_setup(app, conf, cluster, clientname, clientid, args) + app.ceph_addr = addr + app.ceph_port = port + return app diff --git a/src/pybind/ceph-rest/ceph_rest/argparse.py b/src/pybind/ceph-rest/ceph_rest/argparse.py new file mode 100644 index 00000000000..1f6e90b6c1d --- /dev/null +++ b/src/pybind/ceph-rest/ceph_rest/argparse.py @@ -0,0 +1,1110 @@ +""" +Types and routines used by the ceph CLI as well as the RESTful +interface. These have to do with querying the daemons for +command-description information, validating user command input against +those descriptions, and submitting the command to the appropriate +daemon. + +Copyright (C) 2013 Inktank Storage, Inc. + +LGPL2. See file COPYING. +""" +import copy +import json +import os +import pprint +import re +import socket +import stat +import sys +import types +import uuid + +class ArgumentError(Exception): + """ + Something wrong with arguments + """ + pass + +class ArgumentNumber(ArgumentError): + """ + Wrong number of a repeated argument + """ + pass + +class ArgumentFormat(ArgumentError): + """ + Argument value has wrong format + """ + pass + +class ArgumentValid(ArgumentError): + """ + Argument value is otherwise invalid (doesn't match choices, for instance) + """ + pass + +class ArgumentTooFew(ArgumentError): + """ + Fewer arguments than descriptors in signature; may mean to continue + the search, so gets a special exception type + """ + +class ArgumentPrefix(ArgumentError): + """ + Special for mismatched prefix; less severe, don't report by default + """ + pass + +class JsonFormat(Exception): + """ + some syntactic or semantic issue with the JSON + """ + pass + +class CephArgtype(object): + """ + Base class for all Ceph argument types + + Instantiating an object sets any validation parameters + (allowable strings, numeric ranges, etc.). The 'valid' + method validates a string against that initialized instance, + throwing ArgumentError if there's a problem. + """ + def __init__(self, **kwargs): + """ + set any per-instance validation parameters here + from kwargs (fixed string sets, integer ranges, etc) + """ + pass + + def valid(self, s, partial=False): + """ + Run validation against given string s (generally one word); + partial means to accept partial string matches (begins-with). + If cool, set self.val to the value that should be returned + (a copy of the input string, or a numeric or boolean interpretation + thereof, for example) + if not, throw ArgumentError(msg-as-to-why) + """ + self.val = s + + def __repr__(self): + """ + return string representation of description of type. Note, + this is not a representation of the actual value. Subclasses + probably also override __str__() to give a more user-friendly + 'name/type' description for use in command format help messages. + """ + a = '' + if hasattr(self, 'typeargs'): + a = self.typeargs + return '{0}(\'{1}\')'.format(self.__class__.__name__, a) + + def __str__(self): + """ + where __repr__ (ideally) returns a string that could be used to + reproduce the object, __str__ returns one you'd like to see in + print messages. Use __str__ to format the argtype descriptor + as it would be useful in a command usage message. + """ + return '<{0}>'.format(self.__class__.__name__) + +class CephInt(CephArgtype): + """ + range-limited integers, [+|-][0-9]+ or 0x[0-9a-f]+ + range: list of 1 or 2 ints, [min] or [min,max] + """ + def __init__(self, range=''): + if range == '': + self.range = list() + else: + self.range = list(range.split('|')) + self.range = map(long, self.range) + + def valid(self, s, partial=False): + try: + val = long(s) + except ValueError: + raise ArgumentValid("{0} doesn't represent an int".format(s)) + if len(self.range) == 2: + if val < self.range[0] or val > self.range[1]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + elif len(self.range) == 1: + if val < self.range[0]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + self.val = val + + def __str__(self): + r = '' + if len(self.range) == 1: + r = '[{0}-]'.format(self.range[0]) + if len(self.range) == 2: + r = '[{0}-{1}]'.format(self.range[0], self.range[1]) + + return ''.format(r) + + +class CephFloat(CephArgtype): + """ + range-limited float type + range: list of 1 or 2 floats, [min] or [min, max] + """ + def __init__(self, range=''): + if range == '': + self.range = list() + else: + self.range = list(range.split('|')) + self.range = map(float, self.range) + + def valid(self, s, partial=False): + try: + val = float(s) + except ValueError: + raise ArgumentValid("{0} doesn't represent a float".format(s)) + if len(self.range) == 2: + if val < self.range[0] or val > self.range[1]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + elif len(self.range) == 1: + if val < self.range[0]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + self.val = val + + def __str__(self): + r = '' + if len(self.range) == 1: + r = '[{0}-]'.format(self.range[0]) + if len(self.range) == 2: + r = '[{0}-{1}]'.format(self.range[0], self.range[1]) + return ''.format(r) + +class CephString(CephArgtype): + """ + String; pretty generic. goodchars is a RE char class of valid chars + """ + def __init__(self, goodchars=''): + from string import printable + try: + re.compile(goodchars) + except: + raise ValueError('CephString(): "{0}" is not a valid RE'.\ + format(goodchars)) + self.goodchars = goodchars + self.goodset = frozenset( + [c for c in printable if re.match(goodchars, c)] + ) + + def valid(self, s, partial=False): + sset = set(s) + if self.goodset and not sset <= self.goodset: + raise ArgumentFormat("invalid chars {0} in {1}".\ + format(''.join(sset - self.goodset), s)) + self.val = s + + def __str__(self): + b = '' + if self.goodchars: + b += '(goodchars {0})'.format(self.goodchars) + return ''.format(b) + +class CephSocketpath(CephArgtype): + """ + Admin socket path; check that it's readable and S_ISSOCK + """ + def valid(self, s, partial=False): + mode = os.stat(s).st_mode + if not stat.S_ISSOCK(mode): + raise ArgumentValid('socket path {0} is not a socket'.format(s)) + self.val = s + + def __str__(self): + return '' + +class CephIPAddr(CephArgtype): + """ + IP address (v4 or v6) with optional port + """ + def valid(self, s, partial=False): + # parse off port, use socket to validate addr + type = 6 + if s.startswith('['): + type = 6 + elif s.find('.') != -1: + type = 4 + if type == 4: + port = s.find(':') + if (port != -1): + a = s[:port] + p = s[port+1:] + if int(p) > 65535: + raise ArgumentValid('{0}: invalid IPv4 port'.format(p)) + else: + a = s + p = None + try: + socket.inet_pton(socket.AF_INET, a) + except: + raise ArgumentValid('{0}: invalid IPv4 address'.format(a)) + else: + # v6 + if s.startswith('['): + end = s.find(']') + if end == -1: + raise ArgumentFormat('{0} missing terminating ]'.format(s)) + if s[end+1] == ':': + try: + p = int(s[end+2]) + except: + raise ArgumentValid('{0}: bad port number'.format(s)) + a = s[1:end] + else: + a = s + p = None + try: + socket.inet_pton(socket.AF_INET6, a) + except: + raise ArgumentValid('{0} not valid IPv6 address'.format(s)) + if p is not None and long(p) > 65535: + raise ArgumentValid("{0} not a valid port number".format(p)) + self.val = s + self.addr = a + self.port = p + + def __str__(self): + return '' + +class CephEntityAddr(CephIPAddr): + """ + EntityAddress, that is, IP address[/nonce] + """ + def valid(self, s, partial=False): + nonce = None + if '/' in s: + ip, nonce = s.split('/') + else: + ip = s + super(self.__class__, self).valid(ip) + if nonce: + nonce_long = None + try: + nonce_long = long(nonce) + except ValueError: + pass + if nonce_long is None or nonce_long < 0: + raise ArgumentValid( + '{0}: invalid entity, nonce {1} not integer > 0'.\ + format(s, nonce) + ) + self.val = s + + def __str__(self): + return '' + +class CephPoolname(CephArgtype): + """ + Pool name; very little utility + """ + def __str__(self): + return '' + +class CephObjectname(CephArgtype): + """ + Object name. Maybe should be combined with Pool name as they're always + present in pairs, and then could be checked for presence + """ + def __str__(self): + return '' + +class CephPgid(CephArgtype): + """ + pgid, in form N.xxx (N = pool number, xxx = hex pgnum) + """ + def valid(self, s, partial=False): + if s.find('.') == -1: + raise ArgumentFormat('pgid has no .') + poolid, pgnum = s.split('.') + if poolid < 0: + raise ArgumentFormat('pool {0} < 0'.format(poolid)) + try: + pgnum = int(pgnum, 16) + except: + raise ArgumentFormat('pgnum {0} not hex integer'.format(pgnum)) + self.val = s + + def __str__(self): + return '' + +class CephName(CephArgtype): + """ + Name (type.id) where: + type is osd|mon|client|mds + id is a base10 int, if type == osd, or a string otherwise + + Also accept '*' + """ + def __init__(self): + self.nametype = None + self.nameid = None + + def valid(self, s, partial=False): + if s == '*': + self.val = s + return + if s.find('.') == -1: + raise ArgumentFormat('CephName: no . in {0}'.format(s)) + else: + t, i = s.split('.') + if not t in ('osd', 'mon', 'client', 'mds'): + raise ArgumentValid('unknown type ' + t) + if t == 'osd': + if i != '*': + try: + i = int(i) + except: + raise ArgumentFormat('osd id ' + i + ' not integer') + self.nametype = t + self.val = s + self.nameid = i + + def __str__(self): + return '' + +class CephOsdName(CephArgtype): + """ + Like CephName, but specific to osds: allow alone + + osd., or , or *, where id is a base10 int + """ + def __init__(self): + self.nametype = None + self.nameid = None + + def valid(self, s, partial=False): + if s == '*': + self.val = s + return + if s.find('.') != -1: + t, i = s.split('.') + if t != 'osd': + raise ArgumentValid('unknown type ' + t) + else: + t = 'osd' + i = s + try: + i = int(i) + except: + raise ArgumentFormat('osd id ' + i + ' not integer') + self.nametype = t + self.nameid = i + self.val = i + + def __str__(self): + return '' + +class CephChoices(CephArgtype): + """ + Set of string literals; init with valid choices + """ + def __init__(self, strings='', **kwargs): + self.strings = strings.split('|') + + def valid(self, s, partial=False): + if not partial: + if not s in self.strings: + # show as __str__ does: {s1|s2..} + raise ArgumentValid("{0} not in {1}".format(s, self)) + self.val = s + return + + # partial + for t in self.strings: + if t.startswith(s): + self.val = s + return + raise ArgumentValid("{0} not in {1}". format(s, self)) + + def __str__(self): + if len(self.strings) == 1: + return '{0}'.format(self.strings[0]) + else: + return '{0}'.format('|'.join(self.strings)) + +class CephFilepath(CephArgtype): + """ + Openable file + """ + def valid(self, s, partial=False): + try: + f = open(s, 'a+') + except Exception as e: + raise ArgumentValid('can\'t open {0}: {1}'.format(s, e)) + f.close() + self.val = s + + def __str__(self): + return '' + +class CephFragment(CephArgtype): + """ + 'Fragment' ??? XXX + """ + def valid(self, s, partial=False): + if s.find('/') == -1: + raise ArgumentFormat('{0}: no /'.format(s)) + val, bits = s.split('/') + # XXX is this right? + if not val.startswith('0x'): + raise ArgumentFormat("{0} not a hex integer".format(val)) + try: + long(val) + except: + raise ArgumentFormat('can\'t convert {0} to integer'.format(val)) + try: + long(bits) + except: + raise ArgumentFormat('can\'t convert {0} to integer'.format(bits)) + self.val = s + + def __str__(self): + return "" + + +class CephUUID(CephArgtype): + """ + CephUUID: pretty self-explanatory + """ + def valid(self, s, partial=False): + try: + uuid.UUID(s) + except Exception as e: + raise ArgumentFormat('invalid UUID {0}: {1}'.format(s, e)) + self.val = s + + def __str__(self): + return '' + + +class CephPrefix(CephArgtype): + """ + CephPrefix: magic type for "all the first n fixed strings" + """ + def __init__(self, prefix=''): + self.prefix = prefix + + def valid(self, s, partial=False): + if partial: + if self.prefix.startswith(s): + self.val = s + return + else: + if (s == self.prefix): + self.val = s + return + + raise ArgumentPrefix("no match for {0}".format(s)) + + def __str__(self): + return self.prefix + + +class argdesc(object): + """ + argdesc(typename, name='name', n=numallowed|N, + req=False, helptext=helptext, **kwargs (type-specific)) + + validation rules: + typename: type(**kwargs) will be constructed + later, type.valid(w) will be called with a word in that position + + name is used for parse errors and for constructing JSON output + n is a numeric literal or 'n|N', meaning "at least one, but maybe more" + req=False means the argument need not be present in the list + helptext is the associated help for the command + anything else are arguments to pass to the type constructor. + + self.instance is an instance of type t constructed with typeargs. + + valid() will later be called with input to validate against it, + and will store the validated value in self.instance.val for extraction. + """ + def __init__(self, t, name=None, n=1, req=True, **kwargs): + if isinstance(t, types.StringTypes): + self.t = CephPrefix + self.typeargs = {'prefix':t} + self.req = True + else: + self.t = t + self.typeargs = kwargs + self.req = bool(req == True or req == 'True') + + self.name = name + self.N = (n in ['n', 'N']) + if self.N: + self.n = 1 + else: + self.n = int(n) + self.instance = self.t(**self.typeargs) + + def __repr__(self): + r = 'argdesc(' + str(self.t) + ', ' + internals = ['N', 'typeargs', 'instance', 't'] + for (k, v) in self.__dict__.iteritems(): + if k.startswith('__') or k in internals: + pass + else: + # undo modification from __init__ + if k == 'n' and self.N: + v = 'N' + r += '{0}={1}, '.format(k, v) + for (k, v) in self.typeargs.iteritems(): + r += '{0}={1}, '.format(k, v) + return r[:-2] + ')' + + def __str__(self): + if ((self.t == CephChoices and len(self.instance.strings) == 1) + or (self.t == CephPrefix)): + s = str(self.instance) + else: + s = '{0}({1})'.format(self.name, str(self.instance)) + if self.N: + s += ' [' + str(self.instance) + '...]' + if not self.req: + s = '{' + s + '}' + return s + + def helpstr(self): + """ + like str(), but omit parameter names (except for CephString, + which really needs them) + """ + if self.t == CephString: + chunk = '<{0}>'.format(self.name) + else: + chunk = str(self.instance) + s = chunk + if self.N: + s += ' [' + chunk + '...]' + if not self.req: + s = '{' + s + '}' + return s + +def concise_sig(sig): + """ + Return string representation of sig useful for syntax reference in help + """ + return ' '.join([d.helpstr() for d in sig]) + +def descsort(sh1, sh2): + """ + sort descriptors by prefixes, defined as the concatenation of all simple + strings in the descriptor; this works out to just the leading strings. + """ + return cmp(concise_sig(sh1['sig']), concise_sig(sh2['sig'])) + +def parse_funcsig(sig): + """ + parse a single descriptor (array of strings or dicts) into a + dict of function descriptor/validators (objects of CephXXX type) + """ + newsig = [] + argnum = 0 + for desc in sig: + argnum += 1 + if isinstance(desc, types.StringTypes): + t = CephPrefix + desc = {'type':t, 'name':'prefix', 'prefix':desc} + else: + # not a simple string, must be dict + if not 'type' in desc: + s = 'JSON descriptor {0} has no type'.format(sig) + raise JsonFormat(s) + # look up type string in our globals() dict; if it's an + # object of type types.TypeType, it must be a + # locally-defined class. otherwise, we haven't a clue. + if desc['type'] in globals(): + t = globals()[desc['type']] + if type(t) != types.TypeType: + s = 'unknown type {0}'.format(desc['type']) + raise JsonFormat(s) + else: + s = 'unknown type {0}'.format(desc['type']) + raise JsonFormat(s) + + kwargs = dict() + for key, val in desc.items(): + if key not in ['type', 'name', 'n', 'req']: + kwargs[key] = val + newsig.append(argdesc(t, + name=desc.get('name', None), + n=desc.get('n', 1), + req=desc.get('req', True), + **kwargs)) + return newsig + + +def parse_json_funcsigs(s, consumer): + """ + A function signature is mostly an array of argdesc; it's represented + in JSON as + { + "cmd001": {"sig":[ "type": type, "name": name, "n": num, "req":true|false ], "help":helptext, "module":modulename, "perm":perms, "avail":availability} + . + . + . + ] + + A set of sigs is in an dict mapped by a unique number: + { + "cmd1": { + "sig": ["type.. ], "help":helptext... + } + "cmd2"{ + "sig": [.. ], "help":helptext... + } + } + + Parse the string s and return a dict of dicts, keyed by opcode; + each dict contains 'sig' with the array of descriptors, and 'help' + with the helptext, 'module' with the module name, 'perm' with a + string representing required permissions in that module to execute + this command (and also whether it is a read or write command from + the cluster state perspective), and 'avail' as a hint for + whether the command should be advertised by CLI, REST, or both. + If avail does not contain 'consumer', don't include the command + in the returned dict. + """ + try: + overall = json.loads(s) + except Exception as e: + print >> sys.stderr, "Couldn't parse JSON {0}: {1}".format(s, e) + raise e + sigdict = {} + for cmdtag, cmd in overall.iteritems(): + if not 'sig' in cmd: + s = "JSON descriptor {0} has no 'sig'".format(cmdtag) + raise JsonFormat(s) + # check 'avail' and possibly ignore this command + if 'avail' in cmd: + if not consumer in cmd['avail']: + continue + # rewrite the 'sig' item with the argdesc-ized version, and... + cmd['sig'] = parse_funcsig(cmd['sig']) + # just take everything else as given + sigdict[cmdtag] = cmd + return sigdict + +def validate_one(word, desc, partial=False): + """ + validate_one(word, desc, partial=False) + + validate word against the constructed instance of the type + in desc. May raise exception. If it returns false (and doesn't + raise an exception), desc.instance.val will + contain the validated value (in the appropriate type). + """ + desc.instance.valid(word, partial) + desc.numseen += 1 + if desc.N: + desc.n = desc.numseen + 1 + +def matchnum(args, signature, partial=False): + """ + matchnum(s, signature, partial=False) + + Returns number of arguments matched in s against signature. + Can be used to determine most-likely command for full or partial + matches (partial applies to string matches). + """ + words = args[:] + mysig = copy.deepcopy(signature) + matchcnt = 0 + for desc in mysig: + setattr(desc, 'numseen', 0) + while desc.numseen < desc.n: + # if there are no more arguments, return + if not words: + return matchcnt + word = words.pop(0) + + try: + validate_one(word, desc, partial) + valid = True + except ArgumentError: + # matchnum doesn't care about type of error + valid = False + + if not valid: + if not desc.req: + # this wasn't required, so word may match the next desc + words.insert(0, word) + break + else: + # it was required, and didn't match, return + return matchcnt + if desc.req: + matchcnt += 1 + return matchcnt + +def get_next_arg(desc, args): + ''' + Get either the value matching key 'desc.name' or the next arg in + the non-dict list. Return None if args are exhausted. Used in + validate() below. + ''' + arg = None + if isinstance(args, dict): + arg = args.pop(desc.name, None) + # allow 'param=param' to be expressed as 'param' + if arg == '': + arg = desc.name + # Hack, or clever? If value is a list, keep the first element, + # push rest back onto myargs for later processing. + # Could process list directly, but nesting here is already bad + if arg and isinstance(arg, list): + args[desc.name] = arg[1:] + arg = arg[0] + elif args: + arg = args.pop(0) + if arg and isinstance(arg, list): + args = arg[1:] + args + arg = arg[0] + return arg + +def store_arg(desc, d): + ''' + Store argument described by, and held in, thanks to valid(), + desc into the dictionary d, keyed by desc.name. Three cases: + + 1) desc.N is set: value in d is a list + 2) prefix: multiple args are joined with ' ' into one d{} item + 3) single prefix or other arg: store as simple value + + Used in validate() below. + ''' + if desc.N: + # value should be a list + if desc.name in d: + d[desc.name] += [desc.instance.val] + else: + d[desc.name] = [desc.instance.val] + elif (desc.t == CephPrefix) and (desc.name in d): + # prefixes' values should be a space-joined concatenation + d[desc.name] += ' ' + desc.instance.val + else: + # if first CephPrefix or any other type, just set it + d[desc.name] = desc.instance.val + +def validate(args, signature, partial=False): + """ + validate(args, signature, partial=False) + + args is a list of either words or k,v pairs representing a possible + command input following format of signature. Runs a validation; no + exception means it's OK. Return a dict containing all arguments keyed + by their descriptor name, with duplicate args per name accumulated + into a list (or space-separated value for CephPrefix). + + Mismatches of prefix are non-fatal, as this probably just means the + search hasn't hit the correct command. Mismatches of non-prefix + arguments are treated as fatal, and an exception raised. + + This matching is modified if partial is set: allow partial matching + (with partial dict returned); in this case, there are no exceptions + raised. + """ + + myargs = copy.deepcopy(args) + mysig = copy.deepcopy(signature) + reqsiglen = len([desc for desc in mysig if desc.req]) + matchcnt = 0 + d = dict() + for desc in mysig: + setattr(desc, 'numseen', 0) + while desc.numseen < desc.n: + myarg = get_next_arg(desc, myargs) + + # no arg, but not required? Continue consuming mysig + # in case there are later required args + if not myarg and not desc.req: + break + + # out of arguments for a required param? + # Either return (if partial validation) or raise + if not myarg and desc.req: + if desc.N and desc.numseen < 1: + # wanted N, didn't even get 1 + if partial: + return d + raise ArgumentNumber( + 'saw {0} of {1}, expected at least 1'.\ + format(desc.numseen, desc) + ) + elif not desc.N and desc.numseen < desc.n: + # wanted n, got too few + if partial: + return d + # special-case the "0 expected 1" case + if desc.numseen == 0 and desc.n == 1: + raise ArgumentNumber( + 'missing required parameter {0}'.format(desc) + ) + raise ArgumentNumber( + 'saw {0} of {1}, expected {2}'.\ + format(desc.numseen, desc, desc.n) + ) + break + + # Have an arg; validate it + try: + validate_one(myarg, desc) + valid = True + except ArgumentError as e: + valid = False + if not valid: + # argument mismatch + if not desc.req: + # if not required, just push back; it might match + # the next arg + print >> sys.stderr, myarg, 'not valid: ', str(e) + myargs.insert(0, myarg) + break + else: + # hm, it was required, so time to return/raise + if partial: + return d + raise e + + # Whew, valid arg acquired. Store in dict + matchcnt += 1 + store_arg(desc, d) + + # Done with entire list of argdescs + if matchcnt < reqsiglen: + raise ArgumentTooFew("not enough arguments given") + + if myargs and not partial: + raise ArgumentError("unused arguments: " + str(myargs)) + + # Finally, success + return d + +def cmdsiglen(sig): + sigdict = sig.values() + assert len(sigdict) == 1 + return len(sig.values()[0]['sig']) + +def validate_command(sigdict, args, verbose=False): + """ + turn args into a valid dictionary ready to be sent off as JSON, + validated against sigdict. + """ + found = [] + valid_dict = {} + if args: + # look for best match, accumulate possibles in bestcmds + # (so we can maybe give a more-useful error message) + best_match_cnt = 0 + bestcmds = [] + for cmdtag, cmd in sigdict.iteritems(): + sig = cmd['sig'] + matched = matchnum(args, sig, partial=True) + if (matched > best_match_cnt): + if verbose: + print >> sys.stderr, \ + "better match: {0} > {1}: {2}:{3} ".format(matched, + best_match_cnt, cmdtag, concise_sig(sig)) + best_match_cnt = matched + bestcmds = [{cmdtag:cmd}] + elif matched == best_match_cnt: + if verbose: + print >> sys.stderr, \ + "equal match: {0} > {1}: {2}:{3} ".format(matched, + best_match_cnt, cmdtag, concise_sig(sig)) + bestcmds.append({cmdtag:cmd}) + + # Sort bestcmds by number of args so we can try shortest first + # (relies on a cmdsig being key,val where val is a list of len 1) + bestcmds_sorted = sorted(bestcmds, + cmp=lambda x,y:cmp(cmdsiglen(x), cmdsiglen(y))) + + if verbose: + print >> sys.stderr, "bestcmds_sorted: " + pprint.PrettyPrinter(stream=sys.stderr).pprint(bestcmds_sorted) + + # for everything in bestcmds, look for a true match + for cmdsig in bestcmds_sorted: + for cmd in cmdsig.itervalues(): + sig = cmd['sig'] + try: + valid_dict = validate(args, sig) + found = cmd + break + except ArgumentPrefix: + # ignore prefix mismatches; we just haven't found + # the right command yet + pass + except ArgumentTooFew: + # It looked like this matched the beginning, but it + # didn't have enough args supplied. If we're out of + # cmdsigs we'll fall out unfound; if we're not, maybe + # the next one matches completely. Whine, but pass. + if verbose: + print >> sys.stderr, 'Not enough args supplied for ', \ + concise_sig(sig) + except ArgumentError as e: + # Solid mismatch on an arg (type, range, etc.) + # Stop now, because we have the right command but + # some other input is invalid + print >> sys.stderr, "Invalid command: ", str(e) + print >> sys.stderr, concise_sig(sig), ': ', cmd['help'] + return {} + if found: + break + + if not found: + print >> sys.stderr, 'no valid command found; 10 closest matches:' + for cmdsig in bestcmds[:10]: + for (cmdtag, cmd) in cmdsig.iteritems(): + print >> sys.stderr, concise_sig(cmd['sig']) + return None + + return valid_dict + +def find_cmd_target(childargs): + """ + Using a minimal validation, figure out whether the command + should be sent to a monitor or an osd. We do this before even + asking for the 'real' set of command signatures, so we can ask the + right daemon. + Returns ('osd', osdid), ('pg', pgid), or ('mon', '') + """ + sig = parse_funcsig(['tell', {'name':'target', 'type':'CephName'}]) + try: + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: + if len(valid_dict) == 2: + # revalidate to isolate type and id + name = CephName() + # if this fails, something is horribly wrong, as it just + # validated successfully above + name.valid(valid_dict['target']) + return name.nametype, name.nameid + + sig = parse_funcsig(['tell', {'name':'pgid', 'type':'CephPgid'}]) + try: + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: + if len(valid_dict) == 2: + # pg doesn't need revalidation; the string is fine + return 'pg', valid_dict['pgid'] + + sig = parse_funcsig(['pg', {'name':'pgid', 'type':'CephPgid'}]) + try: + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: + if len(valid_dict) == 2: + return 'pg', valid_dict['pgid'] + + return 'mon', '' + +def send_command(cluster, target=('mon', ''), cmd=None, inbuf='', timeout=0, + verbose=False): + """ + Send a command to a daemon using librados's + mon_command, osd_command, or pg_command. Any bulk input data + comes in inbuf. + + Returns (ret, outbuf, outs); ret is the return code, outbuf is + the outbl "bulk useful output" buffer, and outs is any status + or error message (intended for stderr). + + If target is osd.N, send command to that osd (except for pgid cmds) + """ + cmd = cmd or [] + try: + if target[0] == 'osd': + osdid = target[1] + + if verbose: + print >> sys.stderr, 'submit {0} to osd.{1}'.\ + format(cmd, osdid) + ret, outbuf, outs = \ + cluster.osd_command(osdid, cmd, inbuf, timeout) + + elif target[0] == 'pg': + pgid = target[1] + # pgid will already be in the command for the pg + # form, but for tell , we need to put it in + if cmd: + cmddict = json.loads(cmd[0]) + cmddict['pgid'] = pgid + else: + cmddict = dict(pgid=pgid) + cmd = [json.dumps(cmddict)] + if verbose: + print >> sys.stderr, 'submit {0} for pgid {1}'.\ + format(cmd, pgid) + ret, outbuf, outs = \ + cluster.pg_command(pgid, cmd, inbuf, timeout) + + elif target[0] == 'mon': + if verbose: + print >> sys.stderr, '{0} to {1}'.\ + format(cmd, target[0]) + if target[1] == '': + ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout) + else: + ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout, target[1]) + + except Exception as e: + raise RuntimeError('"{0}": exception {1}'.format(cmd, e)) + + return ret, outbuf, outs + +def json_command(cluster, target=('mon', ''), prefix=None, argdict=None, + inbuf='', timeout=0, verbose=False): + """ + Format up a JSON command and send it with send_command() above. + Prefix may be supplied separately or in argdict. Any bulk input + data comes in inbuf. + + If target is osd.N, send command to that osd (except for pgid cmds) + """ + cmddict = {} + if prefix: + cmddict.update({'prefix':prefix}) + if argdict: + cmddict.update(argdict) + + # grab prefix for error messages + prefix = cmddict['prefix'] + + try: + if target[0] == 'osd': + osdtarg = CephName() + osdtarget = '{0}.{1}'.format(*target) + # prefer target from cmddict if present and valid + if 'target' in cmddict: + osdtarget = cmddict.pop('target') + try: + osdtarg.valid(osdtarget) + target = ('osd', osdtarg.nameid) + except: + # use the target we were originally given + pass + + ret, outbuf, outs = send_command(cluster, target, [json.dumps(cmddict)], + inbuf, timeout, verbose) + + except Exception as e: + raise RuntimeError('"{0}": exception {1}'.format(prefix, e)) + + return ret, outbuf, outs + + diff --git a/src/pybind/ceph-rest/setup.py b/src/pybind/ceph-rest/setup.py new file mode 100644 index 00000000000..8d0e6776ea6 --- /dev/null +++ b/src/pybind/ceph-rest/setup.py @@ -0,0 +1,33 @@ +from setuptools import setup, find_packages +import os + + +def long_description(): + readme = os.path.join(os.path.dirname(__file__), 'README.rst') + return open(readme).read() + + +setup( + name = 'ceph-rest', + description = 'Rest API for ceph', + packages=find_packages(), + author = 'Inktank', + author_email = 'ceph-devel@vger.kernel.org', + version = '0.67', + license = "LGPL2", + zip_safe = False, + keywords = "ceph, rest, bindings, api, cli", + install_requires = ['flask==0.10', 'rados'], + long_description = long_description(), + classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)', + 'Topic :: Software Development :: Libraries', + 'Topic :: Utilities', + 'Topic :: System :: Filesystems', + 'Operating System :: POSIX', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], +) diff --git a/src/pybind/ceph-rest/tox.ini b/src/pybind/ceph-rest/tox.ini new file mode 100644 index 00000000000..9c63bb9d884 --- /dev/null +++ b/src/pybind/ceph-rest/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py26, py27 + +[testenv] +deps= +commands= -- cgit v1.2.1 From a941ef7dfe6ab5e8907ff2894738a109f376f728 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 16 Aug 2013 16:14:57 -0400 Subject: ceph-rest-api is now able to import ceph_rest Signed-off-by: Alfredo Deza --- src/ceph-rest-api | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ceph-rest-api b/src/ceph-rest-api index 772b3d20fcd..98e7ce0dbcf 100755 --- a/src/ceph-rest-api +++ b/src/ceph-rest-api @@ -44,14 +44,14 @@ parsed_args, rest = parse_args() # import now that env vars are available to imported module try: - import ceph_rest_api + import ceph_rest except EnvironmentError as e: - print >> sys.stderr, "Error importing ceph_rest_api: ", str(e) + print >> sys.stderr, "Error importing ceph_rest: ", str(e) sys.exit(1) # let other exceptions generate traceback -app = ceph_rest_api.generate_app( +app = ceph_rest.generate_app( parsed_args.conf, parsed_args.cluster, parsed_args.name, -- cgit v1.2.1 From d1ca251c7fb19d50d736904ed7e81d4ea7a990c6 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 16 Aug 2013 16:24:30 -0400 Subject: created the ceph-argparse package Signed-off-by: Alfredo Deza --- src/pybind/ceph-argparse/MANIFEST.in | 2 + src/pybind/ceph-argparse/README.rst | 0 src/pybind/ceph-argparse/ceph_argparse.py | 1090 +++++++++++++++++++++++++++++ src/pybind/ceph-argparse/setup.py | 32 + src/pybind/ceph-argparse/tox.ini | 6 + 5 files changed, 1130 insertions(+) create mode 100644 src/pybind/ceph-argparse/MANIFEST.in create mode 100644 src/pybind/ceph-argparse/README.rst create mode 100644 src/pybind/ceph-argparse/ceph_argparse.py create mode 100644 src/pybind/ceph-argparse/setup.py create mode 100644 src/pybind/ceph-argparse/tox.ini diff --git a/src/pybind/ceph-argparse/MANIFEST.in b/src/pybind/ceph-argparse/MANIFEST.in new file mode 100644 index 00000000000..3c01b12a5a5 --- /dev/null +++ b/src/pybind/ceph-argparse/MANIFEST.in @@ -0,0 +1,2 @@ +include setup.py +include README.rst diff --git a/src/pybind/ceph-argparse/README.rst b/src/pybind/ceph-argparse/README.rst new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/ceph-argparse/ceph_argparse.py b/src/pybind/ceph-argparse/ceph_argparse.py new file mode 100644 index 00000000000..427a4621216 --- /dev/null +++ b/src/pybind/ceph-argparse/ceph_argparse.py @@ -0,0 +1,1090 @@ +""" +Types and routines used by the ceph CLI as well as the RESTful +interface. These have to do with querying the daemons for +command-description information, validating user command input against +those descriptions, and submitting the command to the appropriate +daemon. + +Copyright (C) 2013 Inktank Storage, Inc. + +LGPL2. See file COPYING. +""" +import copy +import json +import os +import pprint +import re +import socket +import stat +import sys +import types +import uuid + +class ArgumentError(Exception): + """ + Something wrong with arguments + """ + pass + +class ArgumentNumber(ArgumentError): + """ + Wrong number of a repeated argument + """ + pass + +class ArgumentFormat(ArgumentError): + """ + Argument value has wrong format + """ + pass + +class ArgumentValid(ArgumentError): + """ + Argument value is otherwise invalid (doesn't match choices, for instance) + """ + pass + +class ArgumentTooFew(ArgumentError): + """ + Fewer arguments than descriptors in signature; may mean to continue + the search, so gets a special exception type + """ + +class ArgumentPrefix(ArgumentError): + """ + Special for mismatched prefix; less severe, don't report by default + """ + pass + +class JsonFormat(Exception): + """ + some syntactic or semantic issue with the JSON + """ + pass + +class CephArgtype(object): + """ + Base class for all Ceph argument types + + Instantiating an object sets any validation parameters + (allowable strings, numeric ranges, etc.). The 'valid' + method validates a string against that initialized instance, + throwing ArgumentError if there's a problem. + """ + def __init__(self, **kwargs): + """ + set any per-instance validation parameters here + from kwargs (fixed string sets, integer ranges, etc) + """ + pass + + def valid(self, s, partial=False): + """ + Run validation against given string s (generally one word); + partial means to accept partial string matches (begins-with). + If cool, set self.val to the value that should be returned + (a copy of the input string, or a numeric or boolean interpretation + thereof, for example) + if not, throw ArgumentError(msg-as-to-why) + """ + self.val = s + + def __repr__(self): + """ + return string representation of description of type. Note, + this is not a representation of the actual value. Subclasses + probably also override __str__() to give a more user-friendly + 'name/type' description for use in command format help messages. + """ + a = '' + if hasattr(self, 'typeargs'): + a = self.typeargs + return '{0}(\'{1}\')'.format(self.__class__.__name__, a) + + def __str__(self): + """ + where __repr__ (ideally) returns a string that could be used to + reproduce the object, __str__ returns one you'd like to see in + print messages. Use __str__ to format the argtype descriptor + as it would be useful in a command usage message. + """ + return '<{0}>'.format(self.__class__.__name__) + +class CephInt(CephArgtype): + """ + range-limited integers, [+|-][0-9]+ or 0x[0-9a-f]+ + range: list of 1 or 2 ints, [min] or [min,max] + """ + def __init__(self, range=''): + if range == '': + self.range = list() + else: + self.range = list(range.split('|')) + self.range = map(long, self.range) + + def valid(self, s, partial=False): + try: + val = long(s) + except ValueError: + raise ArgumentValid("{0} doesn't represent an int".format(s)) + if len(self.range) == 2: + if val < self.range[0] or val > self.range[1]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + elif len(self.range) == 1: + if val < self.range[0]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + self.val = val + + def __str__(self): + r = '' + if len(self.range) == 1: + r = '[{0}-]'.format(self.range[0]) + if len(self.range) == 2: + r = '[{0}-{1}]'.format(self.range[0], self.range[1]) + + return ''.format(r) + + +class CephFloat(CephArgtype): + """ + range-limited float type + range: list of 1 or 2 floats, [min] or [min, max] + """ + def __init__(self, range=''): + if range == '': + self.range = list() + else: + self.range = list(range.split('|')) + self.range = map(float, self.range) + + def valid(self, s, partial=False): + try: + val = float(s) + except ValueError: + raise ArgumentValid("{0} doesn't represent a float".format(s)) + if len(self.range) == 2: + if val < self.range[0] or val > self.range[1]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + elif len(self.range) == 1: + if val < self.range[0]: + raise ArgumentValid("{0} not in range {1}".format(val, self.range)) + self.val = val + + def __str__(self): + r = '' + if len(self.range) == 1: + r = '[{0}-]'.format(self.range[0]) + if len(self.range) == 2: + r = '[{0}-{1}]'.format(self.range[0], self.range[1]) + return ''.format(r) + +class CephString(CephArgtype): + """ + String; pretty generic. goodchars is a RE char class of valid chars + """ + def __init__(self, goodchars=''): + from string import printable + try: + re.compile(goodchars) + except: + raise ValueError('CephString(): "{0}" is not a valid RE'.\ + format(goodchars)) + self.goodchars = goodchars + self.goodset = frozenset( + [c for c in printable if re.match(goodchars, c)] + ) + + def valid(self, s, partial=False): + sset = set(s) + if self.goodset and not sset <= self.goodset: + raise ArgumentFormat("invalid chars {0} in {1}".\ + format(''.join(sset - self.goodset), s)) + self.val = s + + def __str__(self): + b = '' + if self.goodchars: + b += '(goodchars {0})'.format(self.goodchars) + return ''.format(b) + +class CephSocketpath(CephArgtype): + """ + Admin socket path; check that it's readable and S_ISSOCK + """ + def valid(self, s, partial=False): + mode = os.stat(s).st_mode + if not stat.S_ISSOCK(mode): + raise ArgumentValid('socket path {0} is not a socket'.format(s)) + self.val = s + + def __str__(self): + return '' + +class CephIPAddr(CephArgtype): + """ + IP address (v4 or v6) with optional port + """ + def valid(self, s, partial=False): + # parse off port, use socket to validate addr + type = 6 + if s.startswith('['): + type = 6 + elif s.find('.') != -1: + type = 4 + if type == 4: + port = s.find(':') + if (port != -1): + a = s[:port] + p = s[port+1:] + if int(p) > 65535: + raise ArgumentValid('{0}: invalid IPv4 port'.format(p)) + else: + a = s + p = None + try: + socket.inet_pton(socket.AF_INET, a) + except: + raise ArgumentValid('{0}: invalid IPv4 address'.format(a)) + else: + # v6 + if s.startswith('['): + end = s.find(']') + if end == -1: + raise ArgumentFormat('{0} missing terminating ]'.format(s)) + if s[end+1] == ':': + try: + p = int(s[end+2]) + except: + raise ArgumentValid('{0}: bad port number'.format(s)) + a = s[1:end] + else: + a = s + p = None + try: + socket.inet_pton(socket.AF_INET6, a) + except: + raise ArgumentValid('{0} not valid IPv6 address'.format(s)) + if p is not None and long(p) > 65535: + raise ArgumentValid("{0} not a valid port number".format(p)) + self.val = s + self.addr = a + self.port = p + + def __str__(self): + return '' + +class CephEntityAddr(CephIPAddr): + """ + EntityAddress, that is, IP address/nonce + """ + def valid(self, s, partial=False): + ip, nonce = s.split('/') + super(self.__class__, self).valid(ip) + self.nonce = nonce + self.val = s + + def __str__(self): + return '' + +class CephPoolname(CephArgtype): + """ + Pool name; very little utility + """ + def __str__(self): + return '' + +class CephObjectname(CephArgtype): + """ + Object name. Maybe should be combined with Pool name as they're always + present in pairs, and then could be checked for presence + """ + def __str__(self): + return '' + +class CephPgid(CephArgtype): + """ + pgid, in form N.xxx (N = pool number, xxx = hex pgnum) + """ + def valid(self, s, partial=False): + if s.find('.') == -1: + raise ArgumentFormat('pgid has no .') + poolid, pgnum = s.split('.') + if poolid < 0: + raise ArgumentFormat('pool {0} < 0'.format(poolid)) + try: + pgnum = int(pgnum, 16) + except: + raise ArgumentFormat('pgnum {0} not hex integer'.format(pgnum)) + self.val = s + + def __str__(self): + return '' + +class CephName(CephArgtype): + """ + Name (type.id) where: + type is osd|mon|client|mds + id is a base10 int, if type == osd, or a string otherwise + + Also accept '*' + """ + def __init__(self): + self.nametype = None + self.nameid = None + + def valid(self, s, partial=False): + if s == '*': + self.val = s + return + if s.find('.') == -1: + raise ArgumentFormat('CephName: no . in {0}'.format(s)) + else: + t, i = s.split('.') + if not t in ('osd', 'mon', 'client', 'mds'): + raise ArgumentValid('unknown type ' + t) + if t == 'osd': + if i != '*': + try: + i = int(i) + except: + raise ArgumentFormat('osd id ' + i + ' not integer') + self.nametype = t + self.val = s + self.nameid = i + + def __str__(self): + return '' + +class CephOsdName(CephArgtype): + """ + Like CephName, but specific to osds: allow alone + + osd., or , or *, where id is a base10 int + """ + def __init__(self): + self.nametype = None + self.nameid = None + + def valid(self, s, partial=False): + if s == '*': + self.val = s + return + if s.find('.') != -1: + t, i = s.split('.') + if t != 'osd': + raise ArgumentValid('unknown type ' + t) + else: + t = 'osd' + i = s + try: + i = int(i) + except: + raise ArgumentFormat('osd id ' + i + ' not integer') + self.nametype = t + self.nameid = i + self.val = i + + def __str__(self): + return '' + +class CephChoices(CephArgtype): + """ + Set of string literals; init with valid choices + """ + def __init__(self, strings='', **kwargs): + self.strings = strings.split('|') + + def valid(self, s, partial=False): + if not partial: + if not s in self.strings: + # show as __str__ does: {s1|s2..} + raise ArgumentValid("{0} not in {1}".format(s, self)) + self.val = s + return + + # partial + for t in self.strings: + if t.startswith(s): + self.val = s + return + raise ArgumentValid("{0} not in {1}". format(s, self)) + + def __str__(self): + if len(self.strings) == 1: + return '{0}'.format(self.strings[0]) + else: + return '{0}'.format('|'.join(self.strings)) + +class CephFilepath(CephArgtype): + """ + Openable file + """ + def valid(self, s, partial=False): + try: + f = open(s, 'a+') + except Exception as e: + raise ArgumentValid('can\'t open {0}: {1}'.format(s, e)) + f.close() + self.val = s + + def __str__(self): + return '' + +class CephFragment(CephArgtype): + """ + 'Fragment' ??? XXX + """ + def valid(self, s, partial=False): + if s.find('/') == -1: + raise ArgumentFormat('{0}: no /'.format(s)) + val, bits = s.split('/') + # XXX is this right? + if not val.startswith('0x'): + raise ArgumentFormat("{0} not a hex integer".format(val)) + try: + long(val) + except: + raise ArgumentFormat('can\'t convert {0} to integer'.format(val)) + try: + long(bits) + except: + raise ArgumentFormat('can\'t convert {0} to integer'.format(bits)) + self.val = s + + def __str__(self): + return "" + + +class CephUUID(CephArgtype): + """ + CephUUID: pretty self-explanatory + """ + def valid(self, s, partial=False): + try: + uuid.UUID(s) + except Exception as e: + raise ArgumentFormat('invalid UUID {0}: {1}'.format(s, e)) + self.val = s + + def __str__(self): + return '' + + +class CephPrefix(CephArgtype): + """ + CephPrefix: magic type for "all the first n fixed strings" + """ + def __init__(self, prefix=''): + self.prefix = prefix + + def valid(self, s, partial=False): + if partial: + if self.prefix.startswith(s): + self.val = s + return + else: + if (s == self.prefix): + self.val = s + return + + raise ArgumentPrefix("no match for {0}".format(s)) + + def __str__(self): + return self.prefix + + +class argdesc(object): + """ + argdesc(typename, name='name', n=numallowed|N, + req=False, helptext=helptext, **kwargs (type-specific)) + + validation rules: + typename: type(**kwargs) will be constructed + later, type.valid(w) will be called with a word in that position + + name is used for parse errors and for constructing JSON output + n is a numeric literal or 'n|N', meaning "at least one, but maybe more" + req=False means the argument need not be present in the list + helptext is the associated help for the command + anything else are arguments to pass to the type constructor. + + self.instance is an instance of type t constructed with typeargs. + + valid() will later be called with input to validate against it, + and will store the validated value in self.instance.val for extraction. + """ + def __init__(self, t, name=None, n=1, req=True, **kwargs): + if isinstance(t, types.StringTypes): + self.t = CephPrefix + self.typeargs = {'prefix':t} + self.req = True + else: + self.t = t + self.typeargs = kwargs + self.req = bool(req == True or req == 'True') + + self.name = name + self.N = (n in ['n', 'N']) + if self.N: + self.n = 1 + else: + self.n = int(n) + self.instance = self.t(**self.typeargs) + + def __repr__(self): + r = 'argdesc(' + str(self.t) + ', ' + internals = ['N', 'typeargs', 'instance', 't'] + for (k, v) in self.__dict__.iteritems(): + if k.startswith('__') or k in internals: + pass + else: + # undo modification from __init__ + if k == 'n' and self.N: + v = 'N' + r += '{0}={1}, '.format(k, v) + for (k, v) in self.typeargs.iteritems(): + r += '{0}={1}, '.format(k, v) + return r[:-2] + ')' + + def __str__(self): + if ((self.t == CephChoices and len(self.instance.strings) == 1) + or (self.t == CephPrefix)): + s = str(self.instance) + else: + s = '{0}({1})'.format(self.name, str(self.instance)) + if self.N: + s += ' [' + str(self.instance) + '...]' + if not self.req: + s = '{' + s + '}' + return s + + def helpstr(self): + """ + like str(), but omit parameter names (except for CephString, + which really needs them) + """ + if self.t == CephString: + chunk = '<{0}>'.format(self.name) + else: + chunk = str(self.instance) + s = chunk + if self.N: + s += ' [' + chunk + '...]' + if not self.req: + s = '{' + s + '}' + return s + +def concise_sig(sig): + """ + Return string representation of sig useful for syntax reference in help + """ + return ' '.join([d.helpstr() for d in sig]) + +def descsort(sh1, sh2): + """ + sort descriptors by prefixes, defined as the concatenation of all simple + strings in the descriptor; this works out to just the leading strings. + """ + return cmp(concise_sig(sh1['sig']), concise_sig(sh2['sig'])) + +def parse_funcsig(sig): + """ + parse a single descriptor (array of strings or dicts) into a + dict of function descriptor/validators (objects of CephXXX type) + """ + newsig = [] + argnum = 0 + for desc in sig: + argnum += 1 + if isinstance(desc, types.StringTypes): + t = CephPrefix + desc = {'type':t, 'name':'prefix', 'prefix':desc} + else: + # not a simple string, must be dict + if not 'type' in desc: + s = 'JSON descriptor {0} has no type'.format(sig) + raise JsonFormat(s) + # look up type string in our globals() dict; if it's an + # object of type types.TypeType, it must be a + # locally-defined class. otherwise, we haven't a clue. + if desc['type'] in globals(): + t = globals()[desc['type']] + if type(t) != types.TypeType: + s = 'unknown type {0}'.format(desc['type']) + raise JsonFormat(s) + else: + s = 'unknown type {0}'.format(desc['type']) + raise JsonFormat(s) + + kwargs = dict() + for key, val in desc.items(): + if key not in ['type', 'name', 'n', 'req']: + kwargs[key] = val + newsig.append(argdesc(t, + name=desc.get('name', None), + n=desc.get('n', 1), + req=desc.get('req', True), + **kwargs)) + return newsig + + +def parse_json_funcsigs(s, consumer): + """ + A function signature is mostly an array of argdesc; it's represented + in JSON as + { + "cmd001": {"sig":[ "type": type, "name": name, "n": num, "req":true|false ], "help":helptext, "module":modulename, "perm":perms, "avail":availability} + . + . + . + ] + + A set of sigs is in an dict mapped by a unique number: + { + "cmd1": { + "sig": ["type.. ], "help":helptext... + } + "cmd2"{ + "sig": [.. ], "help":helptext... + } + } + + Parse the string s and return a dict of dicts, keyed by opcode; + each dict contains 'sig' with the array of descriptors, and 'help' + with the helptext, 'module' with the module name, 'perm' with a + string representing required permissions in that module to execute + this command (and also whether it is a read or write command from + the cluster state perspective), and 'avail' as a hint for + whether the command should be advertised by CLI, REST, or both. + If avail does not contain 'consumer', don't include the command + in the returned dict. + """ + try: + overall = json.loads(s) + except Exception as e: + print >> sys.stderr, "Couldn't parse JSON {0}: {1}".format(s, e) + raise e + sigdict = {} + for cmdtag, cmd in overall.iteritems(): + if not 'sig' in cmd: + s = "JSON descriptor {0} has no 'sig'".format(cmdtag) + raise JsonFormat(s) + # check 'avail' and possibly ignore this command + if 'avail' in cmd: + if not consumer in cmd['avail']: + continue + # rewrite the 'sig' item with the argdesc-ized version, and... + cmd['sig'] = parse_funcsig(cmd['sig']) + # just take everything else as given + sigdict[cmdtag] = cmd + return sigdict + +def validate_one(word, desc, partial=False): + """ + validate_one(word, desc, partial=False) + + validate word against the constructed instance of the type + in desc. May raise exception. If it returns false (and doesn't + raise an exception), desc.instance.val will + contain the validated value (in the appropriate type). + """ + desc.instance.valid(word, partial) + desc.numseen += 1 + if desc.N: + desc.n = desc.numseen + 1 + +def matchnum(args, signature, partial=False): + """ + matchnum(s, signature, partial=False) + + Returns number of arguments matched in s against signature. + Can be used to determine most-likely command for full or partial + matches (partial applies to string matches). + """ + words = args[:] + mysig = copy.deepcopy(signature) + matchcnt = 0 + for desc in mysig: + setattr(desc, 'numseen', 0) + while desc.numseen < desc.n: + # if there are no more arguments, return + if not words: + return matchcnt + word = words.pop(0) + + try: + validate_one(word, desc, partial) + valid = True + except ArgumentError: + # matchnum doesn't care about type of error + valid = False + + if not valid: + if not desc.req: + # this wasn't required, so word may match the next desc + words.insert(0, word) + break + else: + # it was required, and didn't match, return + return matchcnt + if desc.req: + matchcnt += 1 + return matchcnt + +def get_next_arg(desc, args): + ''' + Get either the value matching key 'desc.name' or the next arg in + the non-dict list. Return None if args are exhausted. Used in + validate() below. + ''' + arg = None + if isinstance(args, dict): + arg = args.pop(desc.name, None) + # allow 'param=param' to be expressed as 'param' + if arg == '': + arg = desc.name + # Hack, or clever? If value is a list, keep the first element, + # push rest back onto myargs for later processing. + # Could process list directly, but nesting here is already bad + if arg and isinstance(arg, list): + args[desc.name] = arg[1:] + arg = arg[0] + elif args: + arg = args.pop(0) + if arg and isinstance(arg, list): + args = arg[1:] + args + arg = arg[0] + return arg + +def store_arg(desc, d): + ''' + Store argument described by, and held in, thanks to valid(), + desc into the dictionary d, keyed by desc.name. Three cases: + + 1) desc.N is set: value in d is a list + 2) prefix: multiple args are joined with ' ' into one d{} item + 3) single prefix or other arg: store as simple value + + Used in validate() below. + ''' + if desc.N: + # value should be a list + if desc.name in d: + d[desc.name] += [desc.instance.val] + else: + d[desc.name] = [desc.instance.val] + elif (desc.t == CephPrefix) and (desc.name in d): + # prefixes' values should be a space-joined concatenation + d[desc.name] += ' ' + desc.instance.val + else: + # if first CephPrefix or any other type, just set it + d[desc.name] = desc.instance.val + +def validate(args, signature, partial=False): + """ + validate(args, signature, partial=False) + + args is a list of either words or k,v pairs representing a possible + command input following format of signature. Runs a validation; no + exception means it's OK. Return a dict containing all arguments keyed + by their descriptor name, with duplicate args per name accumulated + into a list (or space-separated value for CephPrefix). + + Mismatches of prefix are non-fatal, as this probably just means the + search hasn't hit the correct command. Mismatches of non-prefix + arguments are treated as fatal, and an exception raised. + + This matching is modified if partial is set: allow partial matching + (with partial dict returned); in this case, there are no exceptions + raised. + """ + + myargs = copy.deepcopy(args) + mysig = copy.deepcopy(signature) + reqsiglen = len([desc for desc in mysig if desc.req]) + matchcnt = 0 + d = dict() + for desc in mysig: + setattr(desc, 'numseen', 0) + while desc.numseen < desc.n: + myarg = get_next_arg(desc, myargs) + + # no arg, but not required? Continue consuming mysig + # in case there are later required args + if not myarg and not desc.req: + break + + # out of arguments for a required param? + # Either return (if partial validation) or raise + if not myarg and desc.req: + if desc.N and desc.numseen < 1: + # wanted N, didn't even get 1 + if partial: + return d + raise ArgumentNumber( + 'saw {0} of {1}, expected at least 1'.\ + format(desc.numseen, desc) + ) + elif not desc.N and desc.numseen < desc.n: + # wanted n, got too few + if partial: + return d + raise ArgumentNumber( + 'saw {0} of {1}, expected {2}'.\ + format(desc.numseen, desc, desc.n) + ) + break + + # Have an arg; validate it + try: + validate_one(myarg, desc) + valid = True + except ArgumentError as e: + valid = False + if not valid: + # argument mismatch + if not desc.req: + # if not required, just push back; it might match + # the next arg + print >> sys.stderr, myarg, 'not valid: ', str(e) + myargs.insert(0, myarg) + break + else: + # hm, it was required, so time to return/raise + if partial: + return d + raise e + + # Whew, valid arg acquired. Store in dict + matchcnt += 1 + store_arg(desc, d) + + # Done with entire list of argdescs + if matchcnt < reqsiglen: + raise ArgumentTooFew("not enough arguments given") + + if myargs and not partial: + raise ArgumentError("unused arguments: " + str(myargs)) + + # Finally, success + return d + +def cmdsiglen(sig): + sigdict = sig.values() + assert len(sigdict) == 1 + return len(sig.values()[0]['sig']) + +def validate_command(sigdict, args, verbose=False): + """ + turn args into a valid dictionary ready to be sent off as JSON, + validated against sigdict. + """ + found = [] + valid_dict = {} + if args: + # look for best match, accumulate possibles in bestcmds + # (so we can maybe give a more-useful error message) + best_match_cnt = 0 + bestcmds = [] + for cmdtag, cmd in sigdict.iteritems(): + sig = cmd['sig'] + matched = matchnum(args, sig, partial=True) + if (matched > best_match_cnt): + if verbose: + print >> sys.stderr, \ + "better match: {0} > {1}: {2}:{3} ".format(matched, + best_match_cnt, cmdtag, concise_sig(sig)) + best_match_cnt = matched + bestcmds = [{cmdtag:cmd}] + elif matched == best_match_cnt: + if verbose: + print >> sys.stderr, \ + "equal match: {0} > {1}: {2}:{3} ".format(matched, + best_match_cnt, cmdtag, concise_sig(sig)) + bestcmds.append({cmdtag:cmd}) + + # Sort bestcmds by number of args so we can try shortest first + # (relies on a cmdsig being key,val where val is a list of len 1) + bestcmds_sorted = sorted(bestcmds, + cmp=lambda x,y:cmp(cmdsiglen(x), cmdsiglen(y))) + + if verbose: + print >> sys.stderr, "bestcmds_sorted: " + pprint.PrettyPrinter(stream=sys.stderr).pprint(bestcmds_sorted) + + # for everything in bestcmds, look for a true match + for cmdsig in bestcmds_sorted: + for cmd in cmdsig.itervalues(): + sig = cmd['sig'] + try: + valid_dict = validate(args, sig) + found = cmd + break + except ArgumentPrefix: + # ignore prefix mismatches; we just haven't found + # the right command yet + pass + except ArgumentTooFew: + # It looked like this matched the beginning, but it + # didn't have enough args supplied. If we're out of + # cmdsigs we'll fall out unfound; if we're not, maybe + # the next one matches completely. Whine, but pass. + if verbose: + print >> sys.stderr, 'Not enough args supplied for ', \ + concise_sig(sig) + except ArgumentError as e: + # Solid mismatch on an arg (type, range, etc.) + # Stop now, because we have the right command but + # some other input is invalid + print >> sys.stderr, "Invalid command: ", str(e) + return {} + if found: + break + + if not found: + print >> sys.stderr, 'no valid command found; 10 closest matches:' + for cmdsig in bestcmds[:10]: + for (cmdtag, cmd) in cmdsig.iteritems(): + print >> sys.stderr, concise_sig(cmd['sig']) + return None + + return valid_dict + +def find_cmd_target(childargs): + """ + Using a minimal validation, figure out whether the command + should be sent to a monitor or an osd. We do this before even + asking for the 'real' set of command signatures, so we can ask the + right daemon. + Returns ('osd', osdid), ('pg', pgid), or ('mon', '') + """ + sig = parse_funcsig(['tell', {'name':'target', 'type':'CephName'}]) + try: + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: + if len(valid_dict) == 2: + # revalidate to isolate type and id + name = CephName() + # if this fails, something is horribly wrong, as it just + # validated successfully above + name.valid(valid_dict['target']) + return name.nametype, name.nameid + + sig = parse_funcsig(['tell', {'name':'pgid', 'type':'CephPgid'}]) + try: + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: + if len(valid_dict) == 2: + # pg doesn't need revalidation; the string is fine + return 'pg', valid_dict['pgid'] + + sig = parse_funcsig(['pg', {'name':'pgid', 'type':'CephPgid'}]) + try: + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: + if len(valid_dict) == 2: + return 'pg', valid_dict['pgid'] + + return 'mon', '' + +def send_command(cluster, target=('mon', ''), cmd=None, inbuf='', timeout=0, + verbose=False): + """ + Send a command to a daemon using librados's + mon_command, osd_command, or pg_command. Any bulk input data + comes in inbuf. + + Returns (ret, outbuf, outs); ret is the return code, outbuf is + the outbl "bulk useful output" buffer, and outs is any status + or error message (intended for stderr). + + If target is osd.N, send command to that osd (except for pgid cmds) + """ + cmd = cmd or [] + try: + if target[0] == 'osd': + osdid = target[1] + + if verbose: + print >> sys.stderr, 'submit {0} to osd.{1}'.\ + format(cmd, osdid) + ret, outbuf, outs = \ + cluster.osd_command(osdid, cmd, inbuf, timeout) + + elif target[0] == 'pg': + pgid = target[1] + # pgid will already be in the command for the pg + # form, but for tell , we need to put it in + if cmd: + cmddict = json.loads(cmd[0]) + cmddict['pgid'] = pgid + else: + cmddict = dict(pgid=pgid) + cmd = [json.dumps(cmddict)] + if verbose: + print >> sys.stderr, 'submit {0} for pgid {1}'.\ + format(cmd, pgid) + ret, outbuf, outs = \ + cluster.pg_command(pgid, cmd, inbuf, timeout) + + elif target[0] == 'mon': + if verbose: + print >> sys.stderr, '{0} to {1}'.\ + format(cmd, target[0]) + if target[1] == '': + ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout) + else: + ret, outbuf, outs = cluster.mon_command(cmd, inbuf, timeout, target[1]) + + except Exception as e: + raise RuntimeError('"{0}": exception {1}'.format(cmd, e)) + + return ret, outbuf, outs + +def json_command(cluster, target=('mon', ''), prefix=None, argdict=None, + inbuf='', timeout=0, verbose=False): + """ + Format up a JSON command and send it with send_command() above. + Prefix may be supplied separately or in argdict. Any bulk input + data comes in inbuf. + + If target is osd.N, send command to that osd (except for pgid cmds) + """ + cmddict = {} + if prefix: + cmddict.update({'prefix':prefix}) + if argdict: + cmddict.update(argdict) + + # grab prefix for error messages + prefix = cmddict['prefix'] + + try: + if target[0] == 'osd': + osdtarg = CephName() + osdtarget = '{0}.{1}'.format(*target) + # prefer target from cmddict if present and valid + if 'target' in cmddict: + osdtarget = cmddict.pop('target') + try: + osdtarg.valid(osdtarget) + target = ('osd', osdtarg.nameid) + except: + # use the target we were originally given + pass + + ret, outbuf, outs = send_command(cluster, target, [json.dumps(cmddict)], + inbuf, timeout, verbose) + + except Exception as e: + raise RuntimeError('"{0}": exception {1}'.format(prefix, e)) + + return ret, outbuf, outs + + diff --git a/src/pybind/ceph-argparse/setup.py b/src/pybind/ceph-argparse/setup.py new file mode 100644 index 00000000000..0d2ac34c1fd --- /dev/null +++ b/src/pybind/ceph-argparse/setup.py @@ -0,0 +1,32 @@ +from setuptools import setup, find_packages +import os + + +def long_description(): + readme = os.path.join(os.path.dirname(__file__), 'README.rst') + return open(readme).read() + + +setup( + name = 'ceph-argparse', + description = 'argparse library for ceph CLIs', + packages=find_packages(), + author = 'Inktank', + author_email = 'ceph-devel@vger.kernel.org', + version = '0.67', + license = "LGPL2", + zip_safe = False, + keywords = "ceph, rest, bindings, api, cli", + long_description = long_description(), + classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)', + 'Topic :: Software Development :: Libraries', + 'Topic :: Utilities', + 'Topic :: System :: Filesystems', + 'Operating System :: POSIX', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + ], +) diff --git a/src/pybind/ceph-argparse/tox.ini b/src/pybind/ceph-argparse/tox.ini new file mode 100644 index 00000000000..9c63bb9d884 --- /dev/null +++ b/src/pybind/ceph-argparse/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py26, py27 + +[testenv] +deps= +commands= -- cgit v1.2.1 From 6156ca2a6d560ce9836afa4531a5e7fc135a485f Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 16 Aug 2013 16:25:17 -0400 Subject: update the requirements for ceph-rest Signed-off-by: Alfredo Deza --- src/pybind/ceph-rest/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/ceph-rest/setup.py b/src/pybind/ceph-rest/setup.py index 8d0e6776ea6..7411aac78a9 100644 --- a/src/pybind/ceph-rest/setup.py +++ b/src/pybind/ceph-rest/setup.py @@ -17,7 +17,7 @@ setup( license = "LGPL2", zip_safe = False, keywords = "ceph, rest, bindings, api, cli", - install_requires = ['flask==0.10', 'rados'], + install_requires = ['flask==0.10', 'rados', 'ceph-argparse'], long_description = long_description(), classifiers = [ 'Development Status :: 5 - Production/Stable', -- cgit v1.2.1 From b08efa8a2db655051ca8e48423837c075722d671 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 24 Sep 2013 13:51:07 -0400 Subject: remove argparse from ceph-rest, fix imports Signed-off-by: Alfredo Deza --- src/pybind/ceph-rest/ceph_rest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/ceph-rest/ceph_rest/__init__.py b/src/pybind/ceph-rest/ceph_rest/__init__.py index a1a138d6d95..f4eacecb5fc 100755 --- a/src/pybind/ceph-rest/ceph_rest/__init__.py +++ b/src/pybind/ceph-rest/ceph_rest/__init__.py @@ -11,7 +11,7 @@ import xml.etree.ElementTree import xml.sax.saxutils import flask -from ceph_rest.argparse import \ +from ceph_argparse import \ ArgumentError, CephPgid, CephOsdName, CephChoices, CephPrefix, \ concise_sig, descsort, parse_funcsig, parse_json_funcsigs, \ validate, json_command -- cgit v1.2.1 From e5cb774331baa2b5b3007eb0b65a4f5ce5b61fc9 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Thu, 22 Aug 2013 16:35:26 -0400 Subject: create a bootstrap file for pybind Signed-off-by: Alfredo Deza --- src/pybind/bootstrap | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 src/pybind/bootstrap diff --git a/src/pybind/bootstrap b/src/pybind/bootstrap new file mode 100755 index 00000000000..d0dac2f9f71 --- /dev/null +++ b/src/pybind/bootstrap @@ -0,0 +1,60 @@ +#!/bin/sh +set -e + +if command -v lsb_release >/dev/null 2>&1; then + case "$(lsb_release --id --short)" in + Ubuntu|Debian) + for package in python-virtualenv; do + if [ "$(dpkg --status -- $package 2>/dev/null|sed -n 's/^Status: //p')" != "install ok installed" ]; then + # add a space after old values + missing="${missing:+$missing }$package" + fi + done + if [ -n "$missing" ]; then + echo "$0: missing required packages, please install them:" 1>&2 + echo " sudo apt-get install $missing" + exit 1 + fi + ;; + esac + + case "$(lsb_release --id --short | awk '{print $1}')" in + openSUSE|SUSE) + for package in python-virtualenv; do + if [ "$(rpm -qa $package 2>/dev/null)" == "" ]; then + missing="${missing:+$missing }$package" + fi + done + if [ -n "$missing" ]; then + echo "$0: missing required packages, please install them:" 1>&2 + echo " sudo zypper install $missing" + exit 1 + fi + ;; + esac + +else + if [ -f /etc/redhat-release ]; then + case "$(cat /etc/redhat-release | awk '{print $1}')" in + CentOS) + for package in python-virtualenv; do + if [ "$(rpm -qa $package 2>/dev/null)" == "" ]; then + missing="${missing:+$missing }$package" + fi + done + if [ -n "$missing" ]; then + echo "$0: missing required packages, please install them:" 1>&2 + echo " sudo yum install $missing" + exit 1 + fi + ;; + esac + fi +fi + +test -d virtualenv || virtualenv virtualenv +cd ceph-argparse && ./virtualenv/bin/python setup.py develop && cd - +cd ceph-rest && ./virtualenv/bin/python setup.py develop && cd - +cd cephfs && ./virtualenv/bin/python setup.py develop && cd - +cd rados && ./virtualenv/bin/python setup.py develop && cd - +cd rbd && ./virtualenv/bin/python setup.py develop && cd - -- cgit v1.2.1 From 48d99d1901e3ec50eda833baf818cff56754fd65 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Thu, 22 Aug 2013 16:40:38 -0400 Subject: fix the order of the calls to setup.py develop so dependencies are met Signed-off-by: Alfredo Deza --- src/pybind/bootstrap | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pybind/bootstrap b/src/pybind/bootstrap index d0dac2f9f71..1ea35fe2438 100755 --- a/src/pybind/bootstrap +++ b/src/pybind/bootstrap @@ -53,8 +53,8 @@ else fi test -d virtualenv || virtualenv virtualenv -cd ceph-argparse && ./virtualenv/bin/python setup.py develop && cd - -cd ceph-rest && ./virtualenv/bin/python setup.py develop && cd - -cd cephfs && ./virtualenv/bin/python setup.py develop && cd - -cd rados && ./virtualenv/bin/python setup.py develop && cd - -cd rbd && ./virtualenv/bin/python setup.py develop && cd - +cd cephfs && ../virtualenv/bin/python setup.py develop && cd - +cd rados && ../virtualenv/bin/python setup.py develop && cd - +cd rbd && ../virtualenv/bin/python setup.py develop && cd - +cd ceph-argparse && ../virtualenv/bin/python setup.py develop && cd - +cd ceph-rest && ../virtualenv/bin/python setup.py develop && cd - -- cgit v1.2.1 From 2d87ea227392485f90e83f26aa8d2536d7f0b035 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 24 Sep 2013 14:00:14 -0400 Subject: add the targets to the makefile.am Signed-off-by: Alfredo Deza --- src/Makefile.am | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Makefile.am b/src/Makefile.am index 280b268479e..232a62b9dba 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -75,7 +75,7 @@ ceph_syn_SOURCES += client/SyntheticClient.cc # uses g_conf.. needs cleanup ceph_syn_LDADD = $(LIBCLIENT) $(CEPH_GLOBAL) bin_PROGRAMS += ceph-syn -rbd_SOURCES = rbd.cc +rbd_SOURCES = rbd.cc rbd_LDADD = $(LIBRBD) $(LIBRADOS) $(CEPH_GLOBAL) if LINUX bin_PROGRAMS += rbd @@ -395,3 +395,11 @@ project.tgz: clean coverity-submit: scp project.tgz ceph.com:/home/ceph_site/ceph.com/coverity/`git describe`.tgz curl --data "project=ceph&password=`cat ~/coverity.build.pass.txt`&email=sage@newdream.net&url=http://ceph.com/coverity/`git describe`.tgz" http://scan5.coverity.com/cgi-bin/submit_build.py + +# Install Python bindings into a virtualenv +install-pybind: + # bootstrap requires us to be in the same directory + @cd pybind && ./bootstrap + +uninstall-pybind: + rm -rf pybind/virtualenv -- cgit v1.2.1 From 66e78bdb5b70645cae8cadef68954818e42eb31d Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 27 Aug 2013 15:35:27 -0400 Subject: remove invalid comment in target Signed-off-by: Alfredo Deza --- src/Makefile.am | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Makefile.am b/src/Makefile.am index 232a62b9dba..31a2c5b4e10 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -398,7 +398,6 @@ coverity-submit: # Install Python bindings into a virtualenv install-pybind: - # bootstrap requires us to be in the same directory @cd pybind && ./bootstrap uninstall-pybind: -- cgit v1.2.1 From 987c0dc9fe0fd6ce3753567b6e877dc894b99b5a Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 27 Aug 2013 11:42:58 -0700 Subject: ceph.in, ceph-rest-api: adapt "run from src/" hack to virtualenv 1) change python to the virtualenv version, which means 2) we can avoid setting sys.path directly Also: 3) remove the "DEVELOPER MODE" message Signed-off-by: Dan Mick --- src/ceph-rest-api | 10 +++------- src/ceph.in | 16 ++++++---------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/ceph-rest-api b/src/ceph-rest-api index 98e7ce0dbcf..42015006f6b 100755 --- a/src/ceph-rest-api +++ b/src/ceph-rest-api @@ -10,7 +10,7 @@ import sys MYPATH = os.path.abspath(__file__) MYDIR = os.path.dirname(MYPATH) -DEVMODEMSG = '*** DEVELOPER MODE: setting PYTHONPATH and LD_LIBRARY_PATH' +VIRTPYTHON = './pybind/virtualenv/bin/python' if MYDIR.endswith('src') and \ os.path.exists(os.path.join(MYDIR, '.libs')) and \ @@ -19,14 +19,10 @@ if MYDIR.endswith('src') and \ if 'LD_LIBRARY_PATH' in os.environ: if MYLIBPATH not in os.environ['LD_LIBRARY_PATH']: os.environ['LD_LIBRARY_PATH'] += ':' + MYLIBPATH - print >> sys.stderr, DEVMODEMSG - os.execvp('python', ['python'] + sys.argv) + os.execvp(VIRTPYTHON, [VIRTPYTHON] + sys.argv) else: os.environ['LD_LIBRARY_PATH'] = MYLIBPATH - print >> sys.stderr, DEVMODEMSG - os.execvp('python', ['python'] + sys.argv) - sys.path.insert(0, os.path.join(MYDIR, 'pybind')) - + os.execvp(VIRTPYTHON, [VIRTPYTHON] + sys.argv) def parse_args(): parser = argparse.ArgumentParser(description="Ceph REST API webapp") diff --git a/src/ceph.in b/src/ceph.in index 075ec80c20b..c4179fef2a0 100755 --- a/src/ceph.in +++ b/src/ceph.in @@ -26,7 +26,7 @@ import sys MYPATH = os.path.abspath(__file__) MYDIR = os.path.dirname(MYPATH) -DEVMODEMSG = '*** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***' +VIRTPYTHON = './pybind/virtualenv/bin/python' if MYDIR.endswith('src') and \ os.path.exists(os.path.join(MYDIR, '.libs')) and \ @@ -35,15 +35,11 @@ if MYDIR.endswith('src') and \ if 'LD_LIBRARY_PATH' in os.environ: if MYLIBPATH not in os.environ['LD_LIBRARY_PATH']: os.environ['LD_LIBRARY_PATH'] += ':' + MYLIBPATH - print >> sys.stderr, DEVMODEMSG - os.execvp('python', ['python'] + sys.argv) + os.environ['PATH'] += ':' + MYDIR + os.execvp(VIRTPYTHON, [VIRTPYTHON] + sys.argv) else: os.environ['LD_LIBRARY_PATH'] = MYLIBPATH - print >> sys.stderr, DEVMODEMSG - os.execvp('python', ['python'] + sys.argv) - sys.path.insert(0, os.path.join(MYDIR, 'pybind')) - if MYDIR not in os.environ['PATH']: - os.environ['PATH'] += ':' + MYDIR + os.execvp(VIRTPYTHON, [VIRTPYTHON] + sys.argv) import argparse import errno @@ -176,7 +172,7 @@ def do_help(parser, args): def help_for_target(target, partial=None): ret, outbuf, outs = json_command(cluster_handle, target=target, - prefix='get_command_descriptions', + prefix='get_command_descriptions', timeout=10) if ret: print >> sys.stderr, \ @@ -415,7 +411,7 @@ def new_style_command(parsed_args, cmdargs, target, sigdict, inbuf, verbose): def complete(sigdict, args, target): """ Command completion. Match as much of [args] as possible, - and print every possible match separated by newlines. + and print every possible match separated by newlines. Return exitcode. """ # XXX this looks a lot like the front of validate_command(). Refactor? -- cgit v1.2.1 From 8d034b8a7abe5c81f659f88c06590ec4c820873a Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 28 Aug 2013 09:59:50 -0400 Subject: add the spec and debian files for argparse packaging Signed-off-by: Alfredo Deza --- src/pybind/ceph-argparse/ceph-argparse.spec | 70 ++++++++++++++++++++++ .../ceph-argparse/debian/ceph-argparse.install | 1 + src/pybind/ceph-argparse/debian/changelog | 0 src/pybind/ceph-argparse/debian/compat | 1 + src/pybind/ceph-argparse/debian/control | 19 ++++++ src/pybind/ceph-argparse/debian/copyright | 3 + src/pybind/ceph-argparse/debian/rules | 8 +++ src/pybind/ceph-argparse/debian/source/format | 1 + 8 files changed, 103 insertions(+) create mode 100644 src/pybind/ceph-argparse/ceph-argparse.spec create mode 100644 src/pybind/ceph-argparse/debian/ceph-argparse.install create mode 100644 src/pybind/ceph-argparse/debian/changelog create mode 100644 src/pybind/ceph-argparse/debian/compat create mode 100644 src/pybind/ceph-argparse/debian/control create mode 100644 src/pybind/ceph-argparse/debian/copyright create mode 100755 src/pybind/ceph-argparse/debian/rules create mode 100644 src/pybind/ceph-argparse/debian/source/format diff --git a/src/pybind/ceph-argparse/ceph-argparse.spec b/src/pybind/ceph-argparse/ceph-argparse.spec new file mode 100644 index 00000000000..a475fcdee84 --- /dev/null +++ b/src/pybind/ceph-argparse/ceph-argparse.spec @@ -0,0 +1,70 @@ +# +# spec file for package ceph-argparse +# + +%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} +%endif + +################################################################################# +# common +################################################################################# +Name: ceph-argparse +Version: 0.67 +Release: 0 +Summary: Argument parser for the Ceph CLI +License: MIT +Group: System/Filesystems +URL: http://ceph.com/ +Source0: %{name}-%{version}.tar.bz2 +BuildRoot: %{_tmppath}/%{name}-%{version}-build +BuildRequires: python-devel +BuildRequires: python-setuptools +%if 0%{?suse_version} && 0%{?suse_version} <= 1110 +%{!?python_sitelib: %global python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%else +BuildArch: noarch +%endif + +################################################################################# +# specific +################################################################################# +%if 0%{defined suse_version} +%py_requires +%if 0%{?suse_version} > 1210 +Requires: gptfdisk +%else +Requires: scsirastools +%endif +%else +Requires: gdisk +%endif + +%if 0%{?rhel} +BuildRequires: python >= %{pyver} +Requires: python >= %{pyver} +%endif + +%description +An argument parser utility for the Ceph CLI. + +%prep +#%setup -q -n %{name} +%setup -q + +%build +#python setup.py build + +%install +python setup.py install --prefix=%{_prefix} --root=%{buildroot} + +%clean +[ "$RPM_BUILD_ROOT" != "/" ] && rm -rf "$RPM_BUILD_ROOT" + +%files +%defattr(-,root,root) +%doc LICENSE README.rst +%{python_sitelib}/* + +%changelog diff --git a/src/pybind/ceph-argparse/debian/ceph-argparse.install b/src/pybind/ceph-argparse/debian/ceph-argparse.install new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/src/pybind/ceph-argparse/debian/ceph-argparse.install @@ -0,0 +1 @@ + diff --git a/src/pybind/ceph-argparse/debian/changelog b/src/pybind/ceph-argparse/debian/changelog new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/ceph-argparse/debian/compat b/src/pybind/ceph-argparse/debian/compat new file mode 100644 index 00000000000..7f8f011eb73 --- /dev/null +++ b/src/pybind/ceph-argparse/debian/compat @@ -0,0 +1 @@ +7 diff --git a/src/pybind/ceph-argparse/debian/control b/src/pybind/ceph-argparse/debian/control new file mode 100644 index 00000000000..817104745c7 --- /dev/null +++ b/src/pybind/ceph-argparse/debian/control @@ -0,0 +1,19 @@ +Source: ceph-argparse +Maintainer: Sage Weil +Uploaders: Sage Weil +Section: admin +Priority: optional +Build-Depends: debhelper (>= 7), python-setuptools +X-Python-Version: >= 2.4 +Standards-Version: 3.9.2 +Homepage: http://ceph.com/ +Vcs-Git: git://github.com/ceph/ceph-deploy.git +Vcs-Browser: https://github.com/ceph/ceph-deploy + +Package: ceph-argparse +Architecture: all +Depends: python, + python-setuptools, + ${misc:Depends}, + ${python:Depends} +Description: Argument parser utility for the Ceph CLI diff --git a/src/pybind/ceph-argparse/debian/copyright b/src/pybind/ceph-argparse/debian/copyright new file mode 100644 index 00000000000..93bc5303367 --- /dev/null +++ b/src/pybind/ceph-argparse/debian/copyright @@ -0,0 +1,3 @@ +Files: * +Copyright: (c) 2004-2012 by Sage Weil +License: LGPL2.1 (see /usr/share/common-licenses/LGPL-2.1) diff --git a/src/pybind/ceph-argparse/debian/rules b/src/pybind/ceph-argparse/debian/rules new file mode 100755 index 00000000000..45200da0e78 --- /dev/null +++ b/src/pybind/ceph-argparse/debian/rules @@ -0,0 +1,8 @@ +#!/usr/bin/make -f + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +%: + dh $@ --buildsystem python_distutils --with python2 + diff --git a/src/pybind/ceph-argparse/debian/source/format b/src/pybind/ceph-argparse/debian/source/format new file mode 100644 index 00000000000..d3827e75a5c --- /dev/null +++ b/src/pybind/ceph-argparse/debian/source/format @@ -0,0 +1 @@ +1.0 -- cgit v1.2.1 From 8ab5f9bb9b25a274177d37d6d9da2840322352bb Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 28 Aug 2013 10:38:44 -0400 Subject: add the spec and debian files for cephfs packaging Signed-off-by: Alfredo Deza --- src/pybind/cephfs/cephfs.spec | 70 +++++++++++++++++++++++++++++++++ src/pybind/cephfs/debian/cephfs.install | 1 + src/pybind/cephfs/debian/changelog | 0 src/pybind/cephfs/debian/compat | 1 + src/pybind/cephfs/debian/control | 19 +++++++++ src/pybind/cephfs/debian/copyright | 3 ++ src/pybind/cephfs/debian/rules | 8 ++++ src/pybind/cephfs/debian/source/format | 1 + 8 files changed, 103 insertions(+) create mode 100644 src/pybind/cephfs/cephfs.spec create mode 100644 src/pybind/cephfs/debian/cephfs.install create mode 100644 src/pybind/cephfs/debian/changelog create mode 100644 src/pybind/cephfs/debian/compat create mode 100644 src/pybind/cephfs/debian/control create mode 100644 src/pybind/cephfs/debian/copyright create mode 100755 src/pybind/cephfs/debian/rules create mode 100644 src/pybind/cephfs/debian/source/format diff --git a/src/pybind/cephfs/cephfs.spec b/src/pybind/cephfs/cephfs.spec new file mode 100644 index 00000000000..92f57cb05bc --- /dev/null +++ b/src/pybind/cephfs/cephfs.spec @@ -0,0 +1,70 @@ +# +# spec file for package cephfs +# + +%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} +%endif + +################################################################################# +# common +################################################################################# +Name: cephfs +Version: 0.67 +Release: 0 +Summary: Python bindings for the Ceph filesystem +License: MIT +Group: System/Filesystems +URL: http://ceph.com/ +Source0: %{name}-%{version}.tar.bz2 +BuildRoot: %{_tmppath}/%{name}-%{version}-build +BuildRequires: python-devel +BuildRequires: python-setuptools +%if 0%{?suse_version} && 0%{?suse_version} <= 1110 +%{!?python_sitelib: %global python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%else +BuildArch: noarch +%endif + +################################################################################# +# specific +################################################################################# +%if 0%{defined suse_version} +%py_requires +%if 0%{?suse_version} > 1210 +Requires: gptfdisk +%else +Requires: scsirastools +%endif +%else +Requires: gdisk +%endif + +%if 0%{?rhel} +BuildRequires: python >= %{pyver} +Requires: python >= %{pyver} +%endif + +%description +Python bindings for the Ceph filesystem + +%prep +#%setup -q -n %{name} +%setup -q + +%build +#python setup.py build + +%install +python setup.py install --prefix=%{_prefix} --root=%{buildroot} + +%clean +[ "$RPM_BUILD_ROOT" != "/" ] && rm -rf "$RPM_BUILD_ROOT" + +%files +%defattr(-,root,root) +%doc LICENSE README.rst +%{python_sitelib}/* + +%changelog diff --git a/src/pybind/cephfs/debian/cephfs.install b/src/pybind/cephfs/debian/cephfs.install new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/src/pybind/cephfs/debian/cephfs.install @@ -0,0 +1 @@ + diff --git a/src/pybind/cephfs/debian/changelog b/src/pybind/cephfs/debian/changelog new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/cephfs/debian/compat b/src/pybind/cephfs/debian/compat new file mode 100644 index 00000000000..7f8f011eb73 --- /dev/null +++ b/src/pybind/cephfs/debian/compat @@ -0,0 +1 @@ +7 diff --git a/src/pybind/cephfs/debian/control b/src/pybind/cephfs/debian/control new file mode 100644 index 00000000000..658908d9fe7 --- /dev/null +++ b/src/pybind/cephfs/debian/control @@ -0,0 +1,19 @@ +Source: cephfs +Maintainer: Sage Weil +Uploaders: Sage Weil +Section: admin +Priority: optional +Build-Depends: debhelper (>= 7), python-setuptools +X-Python-Version: >= 2.4 +Standards-Version: 3.9.2 +Homepage: http://ceph.com/ +Vcs-Git: git://github.com/ceph/ceph-deploy.git +Vcs-Browser: https://github.com/ceph/ceph-deploy + +Package: cephfs +Architecture: all +Depends: python, + python-setuptools, + ${misc:Depends}, + ${python:Depends} +Description: Argument parser utility for the Ceph CLI diff --git a/src/pybind/cephfs/debian/copyright b/src/pybind/cephfs/debian/copyright new file mode 100644 index 00000000000..93bc5303367 --- /dev/null +++ b/src/pybind/cephfs/debian/copyright @@ -0,0 +1,3 @@ +Files: * +Copyright: (c) 2004-2012 by Sage Weil +License: LGPL2.1 (see /usr/share/common-licenses/LGPL-2.1) diff --git a/src/pybind/cephfs/debian/rules b/src/pybind/cephfs/debian/rules new file mode 100755 index 00000000000..45200da0e78 --- /dev/null +++ b/src/pybind/cephfs/debian/rules @@ -0,0 +1,8 @@ +#!/usr/bin/make -f + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +%: + dh $@ --buildsystem python_distutils --with python2 + diff --git a/src/pybind/cephfs/debian/source/format b/src/pybind/cephfs/debian/source/format new file mode 100644 index 00000000000..d3827e75a5c --- /dev/null +++ b/src/pybind/cephfs/debian/source/format @@ -0,0 +1 @@ +1.0 -- cgit v1.2.1 From 300081a1b94e7c43194fcfff4b4dc2f241f59a46 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 28 Aug 2013 10:41:34 -0400 Subject: add the spec and debian files for ceph-rest packaging Signed-off-by: Alfredo Deza --- src/pybind/ceph-rest/ceph-rest.spec | 70 +++++++++++++++++++++++++++ src/pybind/ceph-rest/debian/ceph-rest.install | 1 + src/pybind/ceph-rest/debian/changelog | 0 src/pybind/ceph-rest/debian/compat | 1 + src/pybind/ceph-rest/debian/control | 19 ++++++++ src/pybind/ceph-rest/debian/copyright | 3 ++ src/pybind/ceph-rest/debian/rules | 8 +++ src/pybind/ceph-rest/debian/source/format | 1 + 8 files changed, 103 insertions(+) create mode 100644 src/pybind/ceph-rest/ceph-rest.spec create mode 100644 src/pybind/ceph-rest/debian/ceph-rest.install create mode 100644 src/pybind/ceph-rest/debian/changelog create mode 100644 src/pybind/ceph-rest/debian/compat create mode 100644 src/pybind/ceph-rest/debian/control create mode 100644 src/pybind/ceph-rest/debian/copyright create mode 100755 src/pybind/ceph-rest/debian/rules create mode 100644 src/pybind/ceph-rest/debian/source/format diff --git a/src/pybind/ceph-rest/ceph-rest.spec b/src/pybind/ceph-rest/ceph-rest.spec new file mode 100644 index 00000000000..ad8d29965c6 --- /dev/null +++ b/src/pybind/ceph-rest/ceph-rest.spec @@ -0,0 +1,70 @@ +# +# spec file for package ceph-rest +# + +%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} +%endif + +################################################################################# +# common +################################################################################# +Name: ceph-rest +Version: 0.67 +Release: 0 +Summary: Restful HTTP API for Ceph +License: MIT +Group: System/Filesystems +URL: http://ceph.com/ +Source0: %{name}-%{version}.tar.bz2 +BuildRoot: %{_tmppath}/%{name}-%{version}-build +BuildRequires: python-devel +BuildRequires: python-setuptools +%if 0%{?suse_version} && 0%{?suse_version} <= 1110 +%{!?python_sitelib: %global python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%else +BuildArch: noarch +%endif + +################################################################################# +# specific +################################################################################# +%if 0%{defined suse_version} +%py_requires +%if 0%{?suse_version} > 1210 +Requires: gptfdisk +%else +Requires: scsirastools +%endif +%else +Requires: gdisk +%endif + +%if 0%{?rhel} +BuildRequires: python >= %{pyver} +Requires: python >= %{pyver} +%endif + +%description +Restful HTTP API for Ceph + +%prep +#%setup -q -n %{name} +%setup -q + +%build +#python setup.py build + +%install +python setup.py install --prefix=%{_prefix} --root=%{buildroot} + +%clean +[ "$RPM_BUILD_ROOT" != "/" ] && rm -rf "$RPM_BUILD_ROOT" + +%files +%defattr(-,root,root) +%doc LICENSE README.rst +%{python_sitelib}/* + +%changelog diff --git a/src/pybind/ceph-rest/debian/ceph-rest.install b/src/pybind/ceph-rest/debian/ceph-rest.install new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/src/pybind/ceph-rest/debian/ceph-rest.install @@ -0,0 +1 @@ + diff --git a/src/pybind/ceph-rest/debian/changelog b/src/pybind/ceph-rest/debian/changelog new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/ceph-rest/debian/compat b/src/pybind/ceph-rest/debian/compat new file mode 100644 index 00000000000..7f8f011eb73 --- /dev/null +++ b/src/pybind/ceph-rest/debian/compat @@ -0,0 +1 @@ +7 diff --git a/src/pybind/ceph-rest/debian/control b/src/pybind/ceph-rest/debian/control new file mode 100644 index 00000000000..3a53105928a --- /dev/null +++ b/src/pybind/ceph-rest/debian/control @@ -0,0 +1,19 @@ +Source: ceph-rest +Maintainer: Sage Weil +Uploaders: Sage Weil +Section: admin +Priority: optional +Build-Depends: debhelper (>= 7), python-setuptools +X-Python-Version: >= 2.4 +Standards-Version: 3.9.2 +Homepage: http://ceph.com/ +Vcs-Git: git://github.com/ceph/ceph-deploy.git +Vcs-Browser: https://github.com/ceph/ceph-deploy + +Package: ceph-rest +Architecture: all +Depends: python, + python-setuptools, + ${misc:Depends}, + ${python:Depends} +Description: Argument parser utility for the Ceph CLI diff --git a/src/pybind/ceph-rest/debian/copyright b/src/pybind/ceph-rest/debian/copyright new file mode 100644 index 00000000000..93bc5303367 --- /dev/null +++ b/src/pybind/ceph-rest/debian/copyright @@ -0,0 +1,3 @@ +Files: * +Copyright: (c) 2004-2012 by Sage Weil +License: LGPL2.1 (see /usr/share/common-licenses/LGPL-2.1) diff --git a/src/pybind/ceph-rest/debian/rules b/src/pybind/ceph-rest/debian/rules new file mode 100755 index 00000000000..45200da0e78 --- /dev/null +++ b/src/pybind/ceph-rest/debian/rules @@ -0,0 +1,8 @@ +#!/usr/bin/make -f + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +%: + dh $@ --buildsystem python_distutils --with python2 + diff --git a/src/pybind/ceph-rest/debian/source/format b/src/pybind/ceph-rest/debian/source/format new file mode 100644 index 00000000000..d3827e75a5c --- /dev/null +++ b/src/pybind/ceph-rest/debian/source/format @@ -0,0 +1 @@ +1.0 -- cgit v1.2.1 From a75a578e4174c6a0a93010093416d74dea0d32f2 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 28 Aug 2013 10:59:44 -0400 Subject: add the spec and debian files for rados packaging Signed-off-by: Alfredo Deza --- src/pybind/rados/debian/changelog | 0 src/pybind/rados/debian/compat | 1 + src/pybind/rados/debian/control | 19 ++++++++++ src/pybind/rados/debian/copyright | 3 ++ src/pybind/rados/debian/rados.install | 1 + src/pybind/rados/debian/rules | 8 ++++ src/pybind/rados/debian/source/format | 1 + src/pybind/rados/rados.spec | 70 +++++++++++++++++++++++++++++++++++ 8 files changed, 103 insertions(+) create mode 100644 src/pybind/rados/debian/changelog create mode 100644 src/pybind/rados/debian/compat create mode 100644 src/pybind/rados/debian/control create mode 100644 src/pybind/rados/debian/copyright create mode 100644 src/pybind/rados/debian/rados.install create mode 100755 src/pybind/rados/debian/rules create mode 100644 src/pybind/rados/debian/source/format create mode 100644 src/pybind/rados/rados.spec diff --git a/src/pybind/rados/debian/changelog b/src/pybind/rados/debian/changelog new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/rados/debian/compat b/src/pybind/rados/debian/compat new file mode 100644 index 00000000000..7f8f011eb73 --- /dev/null +++ b/src/pybind/rados/debian/compat @@ -0,0 +1 @@ +7 diff --git a/src/pybind/rados/debian/control b/src/pybind/rados/debian/control new file mode 100644 index 00000000000..872cf88cd8c --- /dev/null +++ b/src/pybind/rados/debian/control @@ -0,0 +1,19 @@ +Source: rados +Maintainer: Sage Weil +Uploaders: Sage Weil +Section: admin +Priority: optional +Build-Depends: debhelper (>= 7), python-setuptools +X-Python-Version: >= 2.4 +Standards-Version: 3.9.2 +Homepage: http://ceph.com/ +Vcs-Git: git://github.com/ceph/ceph-deploy.git +Vcs-Browser: https://github.com/ceph/ceph-deploy + +Package: rados +Architecture: all +Depends: python, + python-setuptools, + ${misc:Depends}, + ${python:Depends} +Description: Argument parser utility for the Ceph CLI diff --git a/src/pybind/rados/debian/copyright b/src/pybind/rados/debian/copyright new file mode 100644 index 00000000000..93bc5303367 --- /dev/null +++ b/src/pybind/rados/debian/copyright @@ -0,0 +1,3 @@ +Files: * +Copyright: (c) 2004-2012 by Sage Weil +License: LGPL2.1 (see /usr/share/common-licenses/LGPL-2.1) diff --git a/src/pybind/rados/debian/rados.install b/src/pybind/rados/debian/rados.install new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/src/pybind/rados/debian/rados.install @@ -0,0 +1 @@ + diff --git a/src/pybind/rados/debian/rules b/src/pybind/rados/debian/rules new file mode 100755 index 00000000000..45200da0e78 --- /dev/null +++ b/src/pybind/rados/debian/rules @@ -0,0 +1,8 @@ +#!/usr/bin/make -f + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +%: + dh $@ --buildsystem python_distutils --with python2 + diff --git a/src/pybind/rados/debian/source/format b/src/pybind/rados/debian/source/format new file mode 100644 index 00000000000..d3827e75a5c --- /dev/null +++ b/src/pybind/rados/debian/source/format @@ -0,0 +1 @@ +1.0 diff --git a/src/pybind/rados/rados.spec b/src/pybind/rados/rados.spec new file mode 100644 index 00000000000..111fffb6ea7 --- /dev/null +++ b/src/pybind/rados/rados.spec @@ -0,0 +1,70 @@ +# +# spec file for package rados +# + +%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} +%endif + +################################################################################# +# common +################################################################################# +Name: rados +Version: 0.67 +Release: 0 +Summary: Python bindings for Ceph rados +License: MIT +Group: System/Filesystems +URL: http://ceph.com/ +Source0: %{name}-%{version}.tar.bz2 +BuildRoot: %{_tmppath}/%{name}-%{version}-build +BuildRequires: python-devel +BuildRequires: python-setuptools +%if 0%{?suse_version} && 0%{?suse_version} <= 1110 +%{!?python_sitelib: %global python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%else +BuildArch: noarch +%endif + +################################################################################# +# specific +################################################################################# +%if 0%{defined suse_version} +%py_requires +%if 0%{?suse_version} > 1210 +Requires: gptfdisk +%else +Requires: scsirastools +%endif +%else +Requires: gdisk +%endif + +%if 0%{?rhel} +BuildRequires: python >= %{pyver} +Requires: python >= %{pyver} +%endif + +%description +Python bindings for Ceph rados + +%prep +#%setup -q -n %{name} +%setup -q + +%build +#python setup.py build + +%install +python setup.py install --prefix=%{_prefix} --root=%{buildroot} + +%clean +[ "$RPM_BUILD_ROOT" != "/" ] && rm -rf "$RPM_BUILD_ROOT" + +%files +%defattr(-,root,root) +%doc LICENSE README.rst +%{python_sitelib}/* + +%changelog -- cgit v1.2.1 From ebe48981264cb079cd0ad56a76d8831d63cf8ff0 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Wed, 28 Aug 2013 11:06:24 -0400 Subject: add the spec and debian files for rbd packaging Signed-off-by: Alfredo Deza --- src/pybind/rbd/debian/changelog | 0 src/pybind/rbd/debian/compat | 1 + src/pybind/rbd/debian/control | 19 ++++++++++ src/pybind/rbd/debian/copyright | 3 ++ src/pybind/rbd/debian/rbd.install | 1 + src/pybind/rbd/debian/rules | 8 +++++ src/pybind/rbd/debian/source/format | 1 + src/pybind/rbd/rbd.spec | 70 +++++++++++++++++++++++++++++++++++++ 8 files changed, 103 insertions(+) create mode 100644 src/pybind/rbd/debian/changelog create mode 100644 src/pybind/rbd/debian/compat create mode 100644 src/pybind/rbd/debian/control create mode 100644 src/pybind/rbd/debian/copyright create mode 100644 src/pybind/rbd/debian/rbd.install create mode 100755 src/pybind/rbd/debian/rules create mode 100644 src/pybind/rbd/debian/source/format create mode 100644 src/pybind/rbd/rbd.spec diff --git a/src/pybind/rbd/debian/changelog b/src/pybind/rbd/debian/changelog new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/rbd/debian/compat b/src/pybind/rbd/debian/compat new file mode 100644 index 00000000000..7f8f011eb73 --- /dev/null +++ b/src/pybind/rbd/debian/compat @@ -0,0 +1 @@ +7 diff --git a/src/pybind/rbd/debian/control b/src/pybind/rbd/debian/control new file mode 100644 index 00000000000..80182b236f0 --- /dev/null +++ b/src/pybind/rbd/debian/control @@ -0,0 +1,19 @@ +Source: rbd +Maintainer: Sage Weil +Uploaders: Sage Weil +Section: admin +Priority: optional +Build-Depends: debhelper (>= 7), python-setuptools +X-Python-Version: >= 2.4 +Standards-Version: 3.9.2 +Homepage: http://ceph.com/ +Vcs-Git: git://github.com/ceph/ceph-deploy.git +Vcs-Browser: https://github.com/ceph/ceph-deploy + +Package: rbd +Architecture: all +Depends: python, + python-setuptools, + ${misc:Depends}, + ${python:Depends} +Description: Argument parser utility for the Ceph CLI diff --git a/src/pybind/rbd/debian/copyright b/src/pybind/rbd/debian/copyright new file mode 100644 index 00000000000..93bc5303367 --- /dev/null +++ b/src/pybind/rbd/debian/copyright @@ -0,0 +1,3 @@ +Files: * +Copyright: (c) 2004-2012 by Sage Weil +License: LGPL2.1 (see /usr/share/common-licenses/LGPL-2.1) diff --git a/src/pybind/rbd/debian/rbd.install b/src/pybind/rbd/debian/rbd.install new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/src/pybind/rbd/debian/rbd.install @@ -0,0 +1 @@ + diff --git a/src/pybind/rbd/debian/rules b/src/pybind/rbd/debian/rules new file mode 100755 index 00000000000..45200da0e78 --- /dev/null +++ b/src/pybind/rbd/debian/rules @@ -0,0 +1,8 @@ +#!/usr/bin/make -f + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +%: + dh $@ --buildsystem python_distutils --with python2 + diff --git a/src/pybind/rbd/debian/source/format b/src/pybind/rbd/debian/source/format new file mode 100644 index 00000000000..d3827e75a5c --- /dev/null +++ b/src/pybind/rbd/debian/source/format @@ -0,0 +1 @@ +1.0 diff --git a/src/pybind/rbd/rbd.spec b/src/pybind/rbd/rbd.spec new file mode 100644 index 00000000000..64ca73e2427 --- /dev/null +++ b/src/pybind/rbd/rbd.spec @@ -0,0 +1,70 @@ +# +# spec file for package rbd +# + +%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} +%endif + +################################################################################# +# common +################################################################################# +Name: rbd +Version: 0.67 +Release: 0 +Summary: Python bindings for Ceph RBD +License: MIT +Group: System/Filesystems +URL: http://ceph.com/ +Source0: %{name}-%{version}.tar.bz2 +BuildRoot: %{_tmppath}/%{name}-%{version}-build +BuildRequires: python-devel +BuildRequires: python-setuptools +%if 0%{?suse_version} && 0%{?suse_version} <= 1110 +%{!?python_sitelib: %global python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +%else +BuildArch: noarch +%endif + +################################################################################# +# specific +################################################################################# +%if 0%{defined suse_version} +%py_requires +%if 0%{?suse_version} > 1210 +Requires: gptfdisk +%else +Requires: scsirastools +%endif +%else +Requires: gdisk +%endif + +%if 0%{?rhel} +BuildRequires: python >= %{pyver} +Requires: python >= %{pyver} +%endif + +%description +Python bindings for Ceph RBD + +%prep +#%setup -q -n %{name} +%setup -q + +%build +#python setup.py build + +%install +python setup.py install --prefix=%{_prefix} --root=%{buildroot} + +%clean +[ "$RPM_BUILD_ROOT" != "/" ] && rm -rf "$RPM_BUILD_ROOT" + +%files +%defattr(-,root,root) +%doc LICENSE README.rst +%{python_sitelib}/* + +%changelog -- cgit v1.2.1 From a821b9ca435970127963b3be08ec864096dd54bc Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Thu, 29 Aug 2013 11:54:54 -0400 Subject: remove single file python targets from makefile Signed-off-by: Alfredo Deza --- src/Makefile.am | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 31a2c5b4e10..ffa2038b68a 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -311,15 +311,6 @@ clean-local: -rm *.so *.gcno *.gcda -# pybind - -python_PYTHON = pybind/rados.py \ - pybind/rbd.py \ - pybind/cephfs.py \ - pybind/ceph_argparse.py \ - pybind/ceph_rest_api.py - - # everything else we want to include in a 'make dist' noinst_HEADERS += \ -- cgit v1.2.1 From 2b6ebb8efd1a088fb4c3888e3fccd179209a99dd Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 30 Aug 2013 12:56:13 -0400 Subject: remove python-ceph from ceph.spec.in Signed-off-by: Alfredo Deza --- ceph.spec.in | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/ceph.spec.in b/ceph.spec.in index a60d87ad814..c02b0ece782 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -21,7 +21,6 @@ Requires: librados2 = %{version}-%{release} Requires: libcephfs1 = %{version}-%{release} Requires: python Requires: python-argparse -Requires: python-ceph Requires: xfsprogs Requires: cryptsetup Requires: parted @@ -185,22 +184,6 @@ performance, reliability, and scalability. This is a shared library allowing applications to access a Ceph distributed file system via a POSIX-like interface. -%package -n python-ceph -Summary: Python libraries for the Ceph distributed filesystem -Group: System Environment/Libraries -License: LGPL-2.0 -Requires: librados2 = %{version}-%{release} -Requires: librbd1 = %{version}-%{release} -Requires: libcephfs1 = %{version}-%{release} -Requires: python-flask -Requires: python-requests -%if 0%{defined suse_version} -%py_requires -%endif -%description -n python-ceph -This package contains Python libraries for interacting with Cephs RADOS -object storage. - %package -n rest-bench Summary: RESTful benchmark Group: System Environment/Libraries @@ -341,7 +324,7 @@ mkdir -p $RPM_BUILD_ROOT%{_localstatedir}/lib/ceph/bootstrap-mds # Fedora seems to have some problems with this macro, use it only on SUSE %fdupes -s $RPM_BUILD_ROOT/%{python_sitelib} %fdupes %buildroot -%endif +%endif %clean rm -rf $RPM_BUILD_ROOT @@ -371,7 +354,7 @@ fi %endif # Package removal cleanup if [ "$1" -eq "0" ] ; then - rm -rf /var/log/ceph + rm -rf /var/log/ceph rm -rf /etc/ceph fi @@ -584,15 +567,6 @@ fi %postun -n libcephfs1 /sbin/ldconfig -################################################################################# -%files -n python-ceph -%defattr(-,root,root,-) -%{python_sitelib}/rados.py* -%{python_sitelib}/rbd.py* -%{python_sitelib}/cephfs.py* -%{python_sitelib}/ceph_argparse.py* -%{python_sitelib}/ceph_rest_api.py* - ################################################################################# %files -n rest-bench %defattr(-,root,root,-) -- cgit v1.2.1 From 8a072d670339ead21ab81274a700833d67a715ea Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Fri, 30 Aug 2013 14:27:18 -0400 Subject: remove python-ceph from debian scripts Signed-off-by: Alfredo Deza --- debian/control | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/debian/control b/debian/control index 1aec592c9f8..8fe233c48cb 100644 --- a/debian/control +++ b/debian/control @@ -153,8 +153,7 @@ Description: debugging symbols for rbd-fuse Package: ceph-common Architecture: linux-any -Depends: librbd1 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends}, - python-ceph (= ${binary:Version}) +Depends: librbd1 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends}) Conflicts: ceph-client-tools Replaces: ceph-client-tools Suggests: ceph, ceph-mds @@ -388,18 +387,6 @@ Priority: extra Depends: ceph-common, curl, xml2, ${misc:Depends}, ${shlibs:Depends} Description: Ceph test and benchmarking tools. -Package: python-ceph -Architecture: linux-any -Section: python -Depends: librados2, librbd1, python-flask, ${misc:Depends}, ${python:Depends}, python-requests -X-Python-Version: >= 2.6 -Description: Python libraries for the Ceph distributed filesystem - Ceph is a distributed storage and network file system designed to provide - excellent performance, reliability, and scalability. - . - This package contains Python libraries for interacting with Ceph's - RADOS object storage, and RBD (RADOS block device). - Package: libcephfs-java Section: java Architecture: all -- cgit v1.2.1 From 20312b785a12c20fb90b9acf0bf9531b3c5120fb Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Tue, 3 Sep 2013 12:53:14 -0400 Subject: remove extra parens Signed-off-by: Alfredo Deza --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 8fe233c48cb..379919c3ace 100644 --- a/debian/control +++ b/debian/control @@ -153,7 +153,7 @@ Description: debugging symbols for rbd-fuse Package: ceph-common Architecture: linux-any -Depends: librbd1 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends}) +Depends: librbd1 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends} Conflicts: ceph-client-tools Replaces: ceph-client-tools Suggests: ceph, ceph-mds -- cgit v1.2.1 From 49d9e1522eb06261f799ec484eefa6f05f4b8337 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sun, 15 Sep 2013 16:40:25 +0200 Subject: pybind: catch EntityAddress missing / If the / is missing in an EntityAddress, an ArgumentValid exception must be raised so that it can be caught in the same way other argument validation exceptions are. http://tracker.ceph.com/issues/6274 refs #6274 Reviewed-by: Joao Eduardo Luis Signed-off-by: Loic Dachary --- src/pybind/ceph-argparse/ceph_argparse.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pybind/ceph-argparse/ceph_argparse.py b/src/pybind/ceph-argparse/ceph_argparse.py index 427a4621216..f115d3791af 100644 --- a/src/pybind/ceph-argparse/ceph_argparse.py +++ b/src/pybind/ceph-argparse/ceph_argparse.py @@ -278,7 +278,10 @@ class CephEntityAddr(CephIPAddr): EntityAddress, that is, IP address/nonce """ def valid(self, s, partial=False): - ip, nonce = s.split('/') + try: + ip, nonce = s.split('/') + except: + raise ArgumentValid('{0} must contain a /'.format(s)) super(self.__class__, self).valid(ip) self.nonce = nonce self.val = s -- cgit v1.2.1 From 68d59156894a12ce77aa90b6cd287469cd14533e Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 26 Sep 2013 18:00:31 -0700 Subject: ceph_argparse.py, cephtool/test.sh: fix blacklist with no nonce It's legal to give a CephEntityAddr to osd blacklist with no nonce, so allow it in the valid() method; also add validation of any nonce given that it's a long >= 0. Also fix comment on CephEntityAddr type description in MonCommands.h, and add tests for invalid nonces (while fixing the existing tests to remove the () around expect_false args). Fixes: #6425 Signed-off-by: Dan Mick --- src/pybind/ceph-argparse/ceph_argparse.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/pybind/ceph-argparse/ceph_argparse.py b/src/pybind/ceph-argparse/ceph_argparse.py index f115d3791af..7ec7b8b2f0c 100644 --- a/src/pybind/ceph-argparse/ceph_argparse.py +++ b/src/pybind/ceph-argparse/ceph_argparse.py @@ -275,15 +275,26 @@ class CephIPAddr(CephArgtype): class CephEntityAddr(CephIPAddr): """ - EntityAddress, that is, IP address/nonce + EntityAddress, that is, IP address[/nonce] """ def valid(self, s, partial=False): - try: + nonce = None + if '/' in s: ip, nonce = s.split('/') - except: - raise ArgumentValid('{0} must contain a /'.format(s)) + else: + ip = s super(self.__class__, self).valid(ip) - self.nonce = nonce + if nonce: + nonce_long = None + try: + nonce_long = long(nonce) + except ValueError: + pass + if nonce_long is None or nonce_long < 0: + raise ArgumentValid( + '{0}: invalid entity, nonce {1} not integer > 0'.\ + format(s, nonce) + ) self.val = s def __str__(self): -- cgit v1.2.1