diff options
Diffstat (limited to 'src/pybind/ceph_rest_api.py')
| -rwxr-xr-x | src/pybind/ceph_rest_api.py | 366 |
1 files changed, 253 insertions, 113 deletions
diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py index 28a0419c33c..830fb2249ea 100755 --- a/src/pybind/ceph_rest_api.py +++ b/src/pybind/ceph_rest_api.py @@ -1,12 +1,14 @@ #!/usr/bin/python # vim: ts=4 sw=4 smarttab expandtab -import os import collections import ConfigParser +import contextlib +import errno import json import logging import logging.handlers +import os import rados import textwrap import xml.etree.ElementTree @@ -19,13 +21,13 @@ from ceph_argparse import * # Globals # -APPNAME = '__main__' DEFAULT_BASEURL = '/api/v0.1' DEFAULT_ADDR = '0.0.0.0:5000' DEFAULT_LOG_LEVEL = 'warning' DEFAULT_CLIENTNAME = 'client.restapi' DEFAULT_LOG_FILE = '/var/log/ceph/' + DEFAULT_CLIENTNAME + '.log' +APPNAME = '__main__' app = flask.Flask(APPNAME) LOGLEVELS = { @@ -36,7 +38,9 @@ LOGLEVELS = { 'debug':logging.DEBUG, } -# my globals, in a named tuple for usage clarity +# my globals, in a named tuple for usage clarity. I promise +# these are never written once initialized, and are global +# to every thread. glob = collections.namedtuple('gvars', 'cluster urls sigdict baseurl') glob.cluster = None @@ -45,8 +49,15 @@ glob.sigdict = {} glob.baseurl = '' def load_conf(clustername='ceph', conffile=None): - import contextlib + ''' + Load the ceph conf file using ConfigParser. Use the standard + fallback order: + 1) the passed in arg (from CEPH_CONF) + 2) /etc/ceph/{cluster}.conf + 3) ~/.ceph/{cluster}.conf + 4) {cluster}.conf + ''' class _TrimIndentFile(object): def __init__(self, fp): @@ -91,6 +102,10 @@ def load_conf(clustername='ceph', conffile=None): raise EnvironmentError('No conf file found for "{0}"'.format(clustername)) def get_conf(cfg, clientname, key): + ''' + Get config entry from conf file, first in [clientname], then [client], + then [global]. + ''' fullkey = 'restapi_' + key for sectionname in clientname, 'client', 'global': try: @@ -99,16 +114,53 @@ def get_conf(cfg, clientname, key): pass return None -# XXX this is done globally, and cluster connection kept open; there -# are facilities to pass around global info to requests and to -# tear down connections between requests if it becomes important +def find_up_osd(): + ''' + Find an up OSD. Return the last one that's up. + Returns id as an int. + ''' + ret, outbuf, outs = json_command(glob.cluster, prefix="osd dump", + argdict=dict(format='json')) + if ret: + raise EnvironmentError(ret, 'Can\'t get osd dump output') + try: + osddump = json.loads(outbuf) + except: + raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump') + osds = [osd['osd'] for osd in osddump['osds'] if osd['up']] + if not osds: + raise EnvironmentError(errno.ENOENT, 'No up OSDs found') + return int(osds[-1]) + + +METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} def api_setup(): - """ + ''' + This is done globally, and cluster connection kept open for + the lifetime of the daemon. librados should assure that even + if the cluster goes away and comes back, our connection remains. + Initialize the running instance. Open the cluster, get the command signatures, module, perms, and help; stuff them away in the glob.urls - dict. - """ + dict. Also save glob.sigdict for help() handling. + ''' + def get_command_descriptions(target=('mon','')): + ret, outbuf, outs = json_command(glob.cluster, target, + prefix='get_command_descriptions', + timeout=30) + if ret: + err = "Can't get command descriptions: {0}".format(outs) + app.logger.error(err) + raise EnvironmentError(ret, err) + + try: + sigdict = parse_json_funcsigs(outbuf, 'rest') + except Exception as e: + err = "Can't parse command descriptions: {}".format(e) + app.logger.error(err) + raise EnvironmentError(err) + return sigdict conffile = os.environ.get('CEPH_CONF', '') clustername = os.environ.get('CEPH_CLUSTER_NAME', 'ceph') @@ -148,19 +200,22 @@ def api_setup(): h.setFormatter(logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s')) - ret, outbuf, outs = json_command(glob.cluster, - prefix='get_command_descriptions') - if ret: - err = "Can't contact cluster for command descriptions: {0}".format(outs) - app.logger.error(err) - raise EnvironmentError(ret, err) + glob.sigdict = get_command_descriptions() - try: - glob.sigdict = parse_json_funcsigs(outbuf, 'rest') - except Exception as e: - err = "Can't parse command descriptions: {}".format(e) - app.logger.error(err) - raise EnvironmentError(err) + osdid = find_up_osd() + if osdid: + osd_sigdict = get_command_descriptions(target=('osd', int(osdid))) + + # shift osd_sigdict keys up to fit at the end of the mon's glob.sigdict + maxkey = sorted(glob.sigdict.keys())[-1] + maxkey = int(maxkey.replace('cmd', '')) + osdkey = maxkey + 1 + for k, v in osd_sigdict.iteritems(): + newv = v + newv['flavor'] = 'tell' + globk = 'cmd' + str(osdkey) + glob.sigdict[globk] = newv + osdkey += 1 # glob.sigdict maps "cmdNNN" to a dict containing: # 'sig', an array of argdescs @@ -173,27 +228,37 @@ def api_setup(): glob.urls = {} for cmdnum, cmddict in glob.sigdict.iteritems(): cmdsig = cmddict['sig'] - url, params = generate_url_and_params(cmdsig) - if url in glob.urls: - continue + flavor = cmddict.get('flavor', 'mon') + url, params = generate_url_and_params(cmdsig, flavor) + perm = cmddict['perm'] + for k in METHOD_DICT.iterkeys(): + if k in perm: + methods = METHOD_DICT[k] + urldict = {'paramsig':params, + 'help':cmddict['help'], + 'module':cmddict['module'], + 'perm':perm, + 'flavor':flavor, + 'methods':methods, + } + + # glob.urls contains a list of urldicts (usually only one long) + if url not in glob.urls: + glob.urls[url] = [urldict] else: - perm = cmddict['perm'] - urldict = {'paramsig':params, - 'help':cmddict['help'], - 'module':cmddict['module'], - 'perm':perm, - } - method_dict = {'r':['GET'], - 'w':['PUT', 'DELETE']} - for k in method_dict.iterkeys(): - if k in perm: - methods = method_dict[k] - app.add_url_rule(url, url, handler, methods=methods) - glob.urls[url] = urldict - - url += '.<fmt>' - app.add_url_rule(url, url, handler, methods=methods) - glob.urls[url] = urldict + # If more than one, need to make union of methods of all. + # Method must be checked in handler + methodset = set(methods) + for old_urldict in glob.urls[url]: + methodset |= set(old_urldict['methods']) + methods = list(methodset) + glob.urls[url].append(urldict) + + # add, or re-add, rule with all methods and urldicts + app.add_url_rule(url, url, handler, methods=methods) + url += '.<fmt>' + app.add_url_rule(url, url, handler, methods=methods) + app.logger.debug("urls added: %d", len(glob.urls)) app.add_url_rule('/<path:catchall_path>', '/<path:catchall_path>', @@ -201,63 +266,89 @@ def api_setup(): return addr, port -def generate_url_and_params(sig): - """ +def generate_url_and_params(sig, flavor): + ''' Digest command signature from cluster; generate an absolute (including glob.baseurl) endpoint from all the prefix words, - and a dictionary of non-prefix parameters - """ + and a list of non-prefix param descs + ''' url = '' params = [] + # the OSD command descriptors don't include the 'tell <target>', so + # tack it onto the front of sig + if flavor == 'tell': + tellsig = parse_funcsig(['tell', + {'name':'target', 'type':'CephOsdName'}]) + sig = tellsig + sig + for desc in sig: + # prefixes go in the URL path if desc.t == CephPrefix: url += '/' + desc.instance.prefix + # CephChoices with 1 required string (not --) do too, unless + # we've already started collecting params, in which case they + # too are params elif desc.t == CephChoices and \ len(desc.instance.strings) == 1 and \ desc.req and \ - not str(desc.instance).startswith('--'): + not str(desc.instance).startswith('--') and \ + not params: url += '/' + str(desc.instance) else: - params.append(desc) + # tell/<target> is a weird case; the URL includes what + # would everywhere else be a parameter + if flavor == 'tell' and \ + (desc.t, desc.name) == (CephOsdName, 'target'): + url += '/<target>' + else: + params.append(desc) + return glob.baseurl + url, params -def concise_sig_for_uri(sig): - """ +# +# end setup (import-time) functions, begin request-time functions +# + +def concise_sig_for_uri(sig, flavor): + ''' Return a generic description of how one would send a REST request for sig - """ + ''' prefix = [] args = [] + ret = '' + if flavor == 'tell': + ret = 'tell/<osdid-or-pgid>/' for d in sig: if d.t == CephPrefix: prefix.append(d.instance.prefix) else: args.append(d.name + '=' + str(d)) - sig = '/'.join(prefix) + ret += '/'.join(prefix) if args: - sig += '?' + '&'.join(args) - return sig + ret += '?' + '&'.join(args) + return ret def show_human_help(prefix): - """ + ''' Dump table showing commands matching prefix - """ - # XXX this really needs to be a template - #s = '<html><body><style>.colhalf { width: 50%;} body{word-wrap:break-word;}</style>' - #s += '<table border=1><col class=colhalf /><col class=colhalf />' - #s += '<th>Possible commands:</th>' - # XXX the above mucking with css doesn't cause sensible columns. + ''' + # XXX There ought to be a better discovery mechanism than an HTML table s = '<html><body><table border=1><th>Possible commands:</th><th>Method</th><th>Description</th>' - possible = [] permmap = {'r':'GET', 'rw':'PUT'} line = '' for cmdsig in sorted(glob.sigdict.itervalues(), cmp=descsort): concise = concise_sig(cmdsig['sig']) + flavor = cmdsig.get('flavor', 'mon') + if flavor == 'tell': + concise = 'tell/<target>/' + concise if concise.startswith(prefix): line = ['<tr><td>'] - wrapped_sig = textwrap.wrap(concise_sig_for_uri(cmdsig['sig']), 40) + wrapped_sig = textwrap.wrap( + concise_sig_for_uri(cmdsig['sig'], flavor), 40 + ) for sigline in wrapped_sig: line.append(flask.escape(sigline) + '\n') line.append('</td><td>') @@ -275,23 +366,22 @@ def show_human_help(prefix): @app.before_request def log_request(): - """ + ''' For every request, log it. XXX Probably overkill for production - """ + ''' app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string) app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values()) - @app.route('/') def root_redir(): return flask.redirect(glob.baseurl) def make_response(fmt, output, statusmsg, errorcode): - """ + ''' If formatted output, cobble up a response object that contains the output and status wrapped in enclosing objects; if nonformatted, just - use output. Return HTTP status errorcode in any event. - """ + use output+status. Return HTTP status errorcode in any event. + ''' response = output if fmt: if 'json' in fmt: @@ -303,6 +393,7 @@ def make_response(fmt, output, statusmsg, errorcode): return flask.make_response("Error decoding JSON from " + output, 500) elif 'xml' in fmt: + # XXX # one is tempted to do this with xml.etree, but figuring out how # to 'un-XML' the XML-dumped output so it can be reassembled into # a piece of the tree here is beyond me right now. @@ -328,19 +419,26 @@ def make_response(fmt, output, statusmsg, errorcode): return flask.make_response(response, errorcode) -def handler(catchall_path=None, fmt=None): - """ - Main endpoint handler; generic for every endpoint - """ +def handler(catchall_path=None, fmt=None, target=None): + ''' + Main endpoint handler; generic for every endpoint, including catchall. + Handles the catchall, anything with <.fmt>, anything with embedded + <target>. Partial match or ?help cause the HTML-table + "show_human_help" output. + ''' - if (catchall_path): - ep = catchall_path.replace('.<fmt>', '') - else: - ep = flask.request.endpoint.replace('.<fmt>', '') + ep = catchall_path or flask.request.endpoint + ep = ep.replace('.<fmt>', '') if ep[0] != '/': ep = '/' + ep + # demand that endpoint begin with glob.baseurl + if not ep.startswith(glob.baseurl): + return make_response(fmt, '', 'Page not found', 404) + + rel_ep = ep[len(glob.baseurl)+1:] + # Extensions override Accept: headers override defaults if not fmt: if 'application/json' in flask.request.accept_mimetypes.values(): @@ -348,12 +446,35 @@ def handler(catchall_path=None, fmt=None): elif 'application/xml' in flask.request.accept_mimetypes.values(): fmt = 'xml' - # demand that endpoint begin with glob.baseurl - if not ep.startswith(glob.baseurl): - return make_response(fmt, '', 'Page not found', 404) + prefix = '' + pgid = None + cmdtarget = 'mon', '' - relative_endpoint = ep[len(glob.baseurl)+1:] - prefix = ' '.join(relative_endpoint.split('/')).strip() + if target: + # got tell/<target>; validate osdid or pgid + name = CephOsdName() + pgidobj = CephPgid() + try: + name.valid(target) + except ArgumentError: + # try pgid + try: + pgidobj.valid(target) + except ArgumentError: + return flask.make_response("invalid osdid or pgid", 400) + else: + # it's a pgid + pgid = pgidobj.val + cmdtarget = 'pg', pgid + else: + # it's an osd + cmdtarget = name.nametype, name.nameid + + # prefix does not include tell/<target>/ + prefix = ' '.join(rel_ep.split('/')[2:]).strip() + else: + # non-target command: prefix is entire path + prefix = ' '.join(rel_ep.split('/')).strip() # show "match as much as you gave me" help for unknown endpoints if not ep in glob.urls: @@ -365,43 +486,59 @@ def handler(catchall_path=None, fmt=None): else: return make_response(fmt, '', 'Invalid endpoint ' + ep, 400) - urldict = glob.urls[ep] - paramsig = urldict['paramsig'] - - # allow '?help' for any specifically-known endpoint - if 'help' in flask.request.args: - response = flask.make_response('{0}: {1}'.\ - format(prefix + concise_sig(paramsig), urldict['help'])) - response.headers['Content-Type'] = 'text/plain' - return response - - # if there are parameters for this endpoint, process them - if paramsig: - args = {} - for k, l in flask.request.args.iterlists(): - if len(l) == 1: - args[k] = l[0] - else: - args[k] = l - - # is this a valid set of params? - try: - argdict = validate(args, paramsig) - except Exception as e: - return make_response(fmt, '', str(e) + '\n', 400) - else: - # no parameters for this endpoint; complain if args are supplied - if flask.request.args: - return make_response(fmt, '', ep + 'takes no params', 400) - argdict = {} + found = None + exc = '' + for urldict in glob.urls[ep]: + if flask.request.method not in urldict['methods']: + continue + paramsig = urldict['paramsig'] + + # allow '?help' for any specifically-known endpoint + if 'help' in flask.request.args: + response = flask.make_response('{0}: {1}'.\ + format(prefix + concise_sig(paramsig), urldict['help'])) + response.headers['Content-Type'] = 'text/plain' + return response + + # if there are parameters for this endpoint, process them + if paramsig: + args = {} + for k, l in flask.request.args.iterlists(): + if len(l) == 1: + args[k] = l[0] + else: + args[k] = l + + # is this a valid set of params? + try: + argdict = validate(args, paramsig) + found = urldict + break + except Exception as e: + exc += str(e) + continue + else: + if flask.request.args: + continue + found = urldict + argdict = {} + break + if not found: + return make_response(fmt, '', exc + '\n', 400) argdict['format'] = fmt or 'plain' - argdict['module'] = urldict['module'] - argdict['perm'] = urldict['perm'] + argdict['module'] = found['module'] + argdict['perm'] = found['perm'] + if pgid: + argdict['pgid'] = pgid + + if not cmdtarget: + cmdtarget = ('mon', '') app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) ret, outbuf, outs = json_command(glob.cluster, prefix=prefix, + target=cmdtarget, inbuf=flask.request.data, argdict=argdict) if ret: return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400) @@ -414,4 +551,7 @@ def handler(catchall_path=None, fmt=None): response.headers['Content-Type'] = contenttype return response +# +# Last module-level (import-time) ask: set up the cluster connection +# addr, port = api_setup() |