diff options
| author | AmalaBasha <amala.alungal@RACKSPACE.COM> | 2014-07-01 14:45:12 +0530 |
|---|---|---|
| committer | AmalaBasha <amala.alungal@RACKSPACE.COM> | 2014-07-10 13:22:05 +0530 |
| commit | dbb242b776908ca50ed8557ebfe7cfcd879366c8 (patch) | |
| tree | 597e583588dfb4313cf9d10bca34cc956c716e9f /glanceclient | |
| parent | 1db17aaad9a04cc98fc848b23a76db52a5f62965 (diff) | |
| download | python-glanceclient-dbb242b776908ca50ed8557ebfe7cfcd879366c8.tar.gz | |
Replace old httpclient with requests
This review implements blueprint python-request and replaces the old
http client implementation in favor of a new one based on
python-requests.
Major changes:
* raw_request and json_request removed since everything is now being
handled by the same method "_request"
* New methods that match HTTP's methods were added:
- get
- put
- post
- head
- patch
- delete
* Content-Type is now being "inferred" based on the data being sent:
- if it is file-like object it chunks the request
- if it is a python type not instance of basestring then it'll try
to serialize it to json
- Every other case will keep the incoming content-type and will send
the data as is.
* Glanceclient's HTTPSConnection implementation will be used if
no-compression flag is set to True.
Co-Author: Flavio Percoco<flaper87@gmail.com>
Change-Id: I09f70eee3e2777f52ce040296015d41649c2586a
Diffstat (limited to 'glanceclient')
| -rw-r--r-- | glanceclient/common/http.py | 576 | ||||
| -rw-r--r-- | glanceclient/common/https.py | 274 | ||||
| -rw-r--r-- | glanceclient/common/utils.py | 20 | ||||
| -rw-r--r-- | glanceclient/exc.py | 2 | ||||
| -rw-r--r-- | glanceclient/v1/client.py | 14 | ||||
| -rw-r--r-- | glanceclient/v1/image_members.py | 10 | ||||
| -rw-r--r-- | glanceclient/v1/images.py | 49 | ||||
| -rw-r--r-- | glanceclient/v2/image_members.py | 13 | ||||
| -rw-r--r-- | glanceclient/v2/image_tags.py | 4 | ||||
| -rw-r--r-- | glanceclient/v2/images.py | 37 | ||||
| -rw-r--r-- | glanceclient/v2/schemas.py | 2 |
11 files changed, 457 insertions, 544 deletions
diff --git a/glanceclient/common/http.py b/glanceclient/common/http.py index 84714df..a990be5 100644 --- a/glanceclient/common/http.py +++ b/glanceclient/common/http.py @@ -14,16 +14,11 @@ # under the License. import copy -import errno -import hashlib import logging -import posixpath import socket -import ssl -import struct +import requests import six -from six.moves import http_client from six.moves.urllib import parse try: @@ -36,9 +31,7 @@ if not hasattr(parse, 'parse_qsl'): import cgi parse.parse_qsl = cgi.parse_qsl -import OpenSSL - -from glanceclient.common import utils +from glanceclient.common import https from glanceclient import exc from glanceclient.openstack.common import importutils from glanceclient.openstack.common import network_utils @@ -46,48 +39,15 @@ from glanceclient.openstack.common import strutils osprofiler_web = importutils.try_import("osprofiler.web") -try: - from eventlet import patcher - # Handle case where we are running in a monkey patched environment - if patcher.is_monkey_patched('socket'): - from eventlet.green.httplib import HTTPSConnection - from eventlet.green.OpenSSL.SSL import GreenConnection as Connection - from eventlet.greenio import GreenSocket - # TODO(mclaren): A getsockopt workaround: see 'getsockopt' doc string - GreenSocket.getsockopt = utils.getsockopt - else: - raise ImportError -except ImportError: - HTTPSConnection = http_client.HTTPSConnection - from OpenSSL.SSL import Connection as Connection - - LOG = logging.getLogger(__name__) USER_AGENT = 'python-glanceclient' CHUNKSIZE = 1024 * 64 # 64kB -def to_bytes(s): - if isinstance(s, six.string_types): - return six.b(s) - else: - return s - - class HTTPClient(object): def __init__(self, endpoint, **kwargs): self.endpoint = endpoint - endpoint_parts = self.parse_endpoint(self.endpoint) - self.endpoint_scheme = endpoint_parts.scheme - self.endpoint_hostname = endpoint_parts.hostname - self.endpoint_port = endpoint_parts.port - self.endpoint_path = endpoint_parts.path - - self.connection_class = self.get_connection_class(self.endpoint_scheme) - self.connection_kwargs = self.get_connection_kwargs( - self.endpoint_scheme, **kwargs) - self.identity_headers = kwargs.get('identity_headers') self.auth_token = kwargs.get('token') if self.identity_headers: @@ -95,71 +55,58 @@ class HTTPClient(object): self.auth_token = self.identity_headers.get('X-Auth-Token') del self.identity_headers['X-Auth-Token'] - @staticmethod - def parse_endpoint(endpoint): - return network_utils.urlsplit(endpoint) + self.session = requests.Session() + self.session.headers["User-Agent"] = USER_AGENT + self.session.headers["X-Auth-Token"] = self.auth_token - @staticmethod - def get_connection_class(scheme): - if scheme == 'https': - return VerifiedHTTPSConnection - else: - return http_client.HTTPConnection + self.timeout = float(kwargs.get('timeout', 600)) - @staticmethod - def get_connection_kwargs(scheme, **kwargs): - _kwargs = {'timeout': float(kwargs.get('timeout', 600))} + if self.endpoint.startswith("https"): + compression = kwargs.get('ssl_compression', True) - if scheme == 'https': - _kwargs['cacert'] = kwargs.get('cacert', None) - _kwargs['cert_file'] = kwargs.get('cert_file', None) - _kwargs['key_file'] = kwargs.get('key_file', None) - _kwargs['insecure'] = kwargs.get('insecure', False) - _kwargs['ssl_compression'] = kwargs.get('ssl_compression', True) + if not compression: + self.session.mount("https://", https.HTTPSAdapter()) - return _kwargs + self.session.verify = kwargs.get('cacert', + not kwargs.get('insecure', True)) + self.session.cert = (kwargs.get('cert_file'), + kwargs.get('key_file')) - def get_connection(self): - _class = self.connection_class - try: - return _class(self.endpoint_hostname, self.endpoint_port, - **self.connection_kwargs) - except http_client.InvalidURL: - raise exc.InvalidEndpoint() + @staticmethod + def parse_endpoint(endpoint): + return network_utils.urlsplit(endpoint) - def log_curl_request(self, method, url, kwargs): + def log_curl_request(self, method, url, headers, data, kwargs): curl = ['curl -i -X %s' % method] - for (key, value) in kwargs['headers'].items(): + for (key, value) in self.session.headers.items(): if key.lower() == 'x-auth-token': value = '*' * 3 header = '-H \'%s: %s\'' % (key, value) - curl.append(header) - - conn_params_fmt = [ - ('key_file', '--key %s'), - ('cert_file', '--cert %s'), - ('cacert', '--cacert %s'), - ] - for (key, fmt) in conn_params_fmt: - value = self.connection_kwargs.get(key) - if value: - curl.append(fmt % value) - - if self.connection_kwargs.get('insecure'): + curl.append(strutils.safe_encode(header)) + + if not self.session.verify: curl.append('-k') + else: + if isinstance(self.session.verify, six.string_types): + curl.append(' --cacert %s' % self.session.verify) + + if self.session.cert: + curl.append(' --cert %s --key %s' % self.session.cert) - if kwargs.get('body') is not None: - curl.append('-d \'%s\'' % kwargs['body']) + if data and isinstance(data, six.string_types): + curl.append('-d \'%s\'' % data) - curl.append('%s%s' % (self.endpoint, url)) + if "//:" not in url: + url = '%s%s' % (self.endpoint, url) + curl.append(url) LOG.debug(strutils.safe_encode(' '.join(curl), errors='ignore')) @staticmethod def log_http_response(resp, body=None): - status = (resp.version / 10.0, resp.status, resp.reason) + status = (resp.raw.version / 10.0, resp.status_code, resp.reason) dump = ['\nHTTP/%.1f %s %s' % status] - headers = resp.getheaders() + headers = resp.headers.items() if 'X-Auth-Token' in headers: headers['X-Auth-Token'] = '*' * 3 dump.extend(['%s: %s' % (k, v) for k, v in headers]) @@ -183,69 +130,59 @@ class HTTPClient(object): return dict((strutils.safe_encode(h), strutils.safe_encode(v)) for h, v in six.iteritems(headers)) - def _http_request(self, url, method, **kwargs): + def _request(self, method, url, **kwargs): """Send an http request with the specified characteristics. - Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects - kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {})) - kwargs['headers'].setdefault('User-Agent', USER_AGENT) + headers = kwargs.pop("headers", {}) + headers = headers and copy.deepcopy(headers) or {} - if osprofiler_web: - kwargs['headers'].update(osprofiler_web.get_trace_id_headers()) + # Default Content-Type is octet-stream + content_type = headers.get('Content-Type', 'application/octet-stream') - if self.auth_token: - kwargs['headers'].setdefault('X-Auth-Token', self.auth_token) + def chunk_body(body): + chunk = body + while chunk: + chunk = body.read(CHUNKSIZE) + yield chunk - if self.identity_headers: - for k, v in six.iteritems(self.identity_headers): - kwargs['headers'].setdefault(k, v) + data = kwargs.pop("data", None) + if data is not None and not isinstance(data, six.string_types): + try: + data = json.dumps(data) + content_type = 'application/json' + except TypeError: + # Here we assume it's + # a file-like object + # and we'll chunk it + data = chunk_body(data) - self.log_curl_request(method, url, kwargs) - conn = self.get_connection() + headers['Content-Type'] = content_type # Note(flaper87): Before letting headers / url fly, # they should be encoded otherwise httplib will - # complain. If we decide to rely on python-request - # this wont be necessary anymore. - kwargs['headers'] = self.encode_headers(kwargs['headers']) + # complain. + headers = self.encode_headers(headers) try: - if self.endpoint_path: - # NOTE(yuyangbj): this method _http_request could either be - # called by API layer, or be called recursively with - # redirection. For example, url would be '/v1/images/detail' - # from API layer, but url would be 'https://example.com:92/ - # v1/images/detail' from recursion. - # See bug #1230032 and bug #1208618. - if url is not None: - all_parts = parse.urlparse(url) - if not (all_parts.scheme and all_parts.netloc): - norm_parse = posixpath.normpath - url = norm_parse('/'.join([self.endpoint_path, url])) - else: - url = self.endpoint_path - - conn_url = parse.urlsplit(url).geturl() - # Note(flaper87): Ditto, headers / url - # encoding to make httplib happy. - conn_url = strutils.safe_encode(conn_url) - if kwargs['headers'].get('Transfer-Encoding') == 'chunked': - conn.putrequest(method, conn_url) - for header, value in kwargs['headers'].items(): - conn.putheader(header, value) - conn.endheaders() - chunk = kwargs['body'].read(CHUNKSIZE) - # Chunk it, baby... - while chunk: - conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) - chunk = kwargs['body'].read(CHUNKSIZE) - conn.send('0\r\n\r\n') - else: - conn.request(method, conn_url, **kwargs) - resp = conn.getresponse() + conn_url = "%s/%s" % (self.endpoint, url) + self.log_curl_request(method, conn_url, headers, data, kwargs) + resp = self.session.request(method, + conn_url, + data=data, + stream=True, + headers=headers, + **kwargs) + except requests.exceptions.Timeout as e: + message = ("Error communicating with %(endpoint)s %(e)s" % + dict(url=conn_url, e=e)) + raise exc.InvalidEndpoint(message=message) + except requests.exceptions.ConnectionError as e: + message = ("Error finding address for %(url)s: %(e)s" % + dict(url=conn_url, e=e)) + raise exc.CommunicationError(message=message) except socket.gaierror as e: message = "Error finding address for %s: %s" % ( self.endpoint_hostname, e) @@ -256,357 +193,46 @@ class HTTPClient(object): {'endpoint': endpoint, 'e': e}) raise exc.CommunicationError(message=message) - body_iter = ResponseBodyIterator(resp) - - # Read body into string if it isn't obviously image data - if resp.getheader('content-type', None) != 'application/octet-stream': - body_str = b''.join([to_bytes(chunk) for chunk in body_iter]) - self.log_http_response(resp, body_str) - body_iter = six.BytesIO(body_str) - else: - self.log_http_response(resp) - - if 400 <= resp.status < 600: - LOG.debug("Request returned failure status: %d" % resp.status) - raise exc.from_response(resp, body_str) - elif resp.status in (301, 302, 305): - # Redirected. Reissue the request to the new location. - return self._http_request(resp.getheader('location', None), method, - **kwargs) - elif resp.status == 300: + if not resp.ok: + LOG.error("Request returned failure status %s." % resp.status_code) + raise exc.from_response(resp, resp.content) + elif resp.status_code == requests.codes.MULTIPLE_CHOICES: raise exc.from_response(resp) - return resp, body_iter - - def json_request(self, method, url, **kwargs): - kwargs.setdefault('headers', {}) - kwargs['headers'].setdefault('Content-Type', 'application/json') - - if 'body' in kwargs: - kwargs['body'] = json.dumps(kwargs['body']) - - resp, body_iter = self._http_request(url, method, **kwargs) - - if 'application/json' in resp.getheader('content-type', ''): - body = ''.join([chunk for chunk in body_iter]) - try: - body = json.loads(body) - except ValueError: - LOG.error('Could not decode response body as JSON') - else: - body = None - - return resp, body + content_type = resp.headers.get('Content-Type') - def raw_request(self, method, url, **kwargs): - kwargs.setdefault('headers', {}) - kwargs['headers'].setdefault('Content-Type', - 'application/octet-stream') - - if 'content_length' in kwargs: - content_length = kwargs.pop('content_length') + # Read body into string if it isn't obviously image data + if content_type == 'application/octet-stream': + # Do not read all response in memory when + # downloading an image. + body_iter = resp.iter_content(chunk_size=CHUNKSIZE) + self.log_http_response(resp) else: - content_length = None - - if (('body' in kwargs) and (hasattr(kwargs['body'], 'read') and - method.lower() in ('post', 'put'))): - - # NOTE(dosaboy): only use chunked transfer if not setting a - # content length since setting it will implicitly disable - # chunking. - - file_content_length = utils.get_file_size(kwargs['body']) - if content_length is None: - content_length = file_content_length - elif (file_content_length and - (content_length != file_content_length)): - errmsg = ("supplied content-length (%s) does not match " - "length of supplied data (%s)" % - (content_length, file_content_length)) - raise AttributeError(errmsg) - - if content_length is None: - # We use 'Transfer-Encoding: chunked' because - # body size may not always be known in advance. - kwargs['headers']['Transfer-Encoding'] = 'chunked' + content = resp.content + self.log_http_response(resp, content) + if content_type and content_type.startswith('application/json'): + # Let's use requests json method, + # it should take care of response + # encoding + body_iter = resp.json() else: - kwargs['headers']['Content-Length'] = str(content_length) - - return self._http_request(url, method, **kwargs) - - def client_request(self, method, url, **kwargs): - # NOTE(akurilin): this method provides compatibility with methods which - # expects requests.Response object(for example - methods of - # class Managers from common code). - if 'json' in kwargs and 'body' not in kwargs: - kwargs['body'] = kwargs.pop('json') - resp, body = self.json_request(method, url, **kwargs) - resp.json = lambda: body - resp.content = bool(body) - resp.status_code = resp.status - return resp + body_iter = six.StringIO(content) + return resp, body_iter def head(self, url, **kwargs): - return self.client_request("HEAD", url, **kwargs) + return self._request('HEAD', url, **kwargs) def get(self, url, **kwargs): - return self.client_request("GET", url, **kwargs) + return self._request('GET', url, **kwargs) def post(self, url, **kwargs): - return self.client_request("POST", url, **kwargs) + return self._request('POST', url, **kwargs) def put(self, url, **kwargs): - return self.client_request("PUT", url, **kwargs) - - def delete(self, url, **kwargs): - return self.raw_request("DELETE", url, **kwargs) + return self._request('PUT', url, **kwargs) def patch(self, url, **kwargs): - return self.client_request("PATCH", url, **kwargs) - - -class OpenSSLConnectionDelegator(object): - """ - An OpenSSL.SSL.Connection delegator. - - Supplies an additional 'makefile' method which httplib requires - and is not present in OpenSSL.SSL.Connection. - - Note: Since it is not possible to inherit from OpenSSL.SSL.Connection - a delegator must be used. - """ - def __init__(self, *args, **kwargs): - self.connection = Connection(*args, **kwargs) - - def __getattr__(self, name): - return getattr(self.connection, name) - - def makefile(self, *args, **kwargs): - # Making sure socket is closed when this file is closed - # since we now avoid closing socket on connection close - # see new close method under VerifiedHTTPSConnection - kwargs['close'] = True - - return socket._fileobject(self.connection, *args, **kwargs) - - -class VerifiedHTTPSConnection(HTTPSConnection): - """ - Extended HTTPSConnection which uses the OpenSSL library - for enhanced SSL support. - Note: Much of this functionality can eventually be replaced - with native Python 3.3 code. - """ - def __init__(self, host, port=None, key_file=None, cert_file=None, - cacert=None, timeout=None, insecure=False, - ssl_compression=True): - # List of exceptions reported by Python3 instead of - # SSLConfigurationError - if six.PY3: - excp_lst = (TypeError, FileNotFoundError, ssl.SSLError) - else: - excp_lst = () - try: - HTTPSConnection.__init__(self, host, port, - key_file=key_file, - cert_file=cert_file) - self.key_file = key_file - self.cert_file = cert_file - self.timeout = timeout - self.insecure = insecure - self.ssl_compression = ssl_compression - self.cacert = None if cacert is None else str(cacert) - self.setcontext() - # ssl exceptions are reported in various form in Python 3 - # so to be compatible, we report the same kind as under - # Python2 - except excp_lst as e: - raise exc.SSLConfigurationError(str(e)) - - @staticmethod - def host_matches_cert(host, x509): - """ - Verify that the x509 certificate we have received - from 'host' correctly identifies the server we are - connecting to, i.e. that the certificate's Common Name - or a Subject Alternative Name matches 'host'. - """ - def check_match(name): - # Directly match the name - if name == host: - return True - - # Support single wildcard matching - if name.startswith('*.') and host.find('.') > 0: - if name[2:] == host.split('.', 1)[1]: - return True - - common_name = x509.get_subject().commonName - - # First see if we can match the CN - if check_match(common_name): - return True - - # Also try Subject Alternative Names for a match - san_list = None - for i in range(x509.get_extension_count()): - ext = x509.get_extension(i) - if ext.get_short_name() == b'subjectAltName': - san_list = str(ext) - for san in ''.join(san_list.split()).split(','): - if san.startswith('DNS:'): - if check_match(san.split(':', 1)[1]): - return True - - # Server certificate does not match host - msg = ('Host "%s" does not match x509 certificate contents: ' - 'CommonName "%s"' % (host, common_name)) - if san_list is not None: - msg = msg + ', subjectAltName "%s"' % san_list - raise exc.SSLCertificateError(msg) - - def verify_callback(self, connection, x509, errnum, - depth, preverify_ok): - # NOTE(leaman): preverify_ok may be a non-boolean type - preverify_ok = bool(preverify_ok) - if x509.has_expired(): - msg = "SSL Certificate expired on '%s'" % x509.get_notAfter() - raise exc.SSLCertificateError(msg) - - if depth == 0 and preverify_ok: - # We verify that the host matches against the last - # certificate in the chain - return self.host_matches_cert(self.host, x509) - else: - # Pass through OpenSSL's default result - return preverify_ok - - def setcontext(self): - """ - Set up the OpenSSL context. - """ - self.context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) - - if self.ssl_compression is False: - self.context.set_options(0x20000) # SSL_OP_NO_COMPRESSION + return self._request('PATCH', url, **kwargs) - if self.insecure is not True: - self.context.set_verify(OpenSSL.SSL.VERIFY_PEER, - self.verify_callback) - else: - self.context.set_verify(OpenSSL.SSL.VERIFY_NONE, - lambda *args: True) - - if self.cert_file: - try: - self.context.use_certificate_file(self.cert_file) - except Exception as e: - msg = 'Unable to load cert from "%s" %s' % (self.cert_file, e) - raise exc.SSLConfigurationError(msg) - if self.key_file is None: - # We support having key and cert in same file - try: - self.context.use_privatekey_file(self.cert_file) - except Exception as e: - msg = ('No key file specified and unable to load key ' - 'from "%s" %s' % (self.cert_file, e)) - raise exc.SSLConfigurationError(msg) - - if self.key_file: - try: - self.context.use_privatekey_file(self.key_file) - except Exception as e: - msg = 'Unable to load key from "%s" %s' % (self.key_file, e) - raise exc.SSLConfigurationError(msg) - - if self.cacert: - try: - self.context.load_verify_locations(to_bytes(self.cacert)) - except Exception as e: - msg = ('Unable to load CA from "%(cacert)s" %(exc)s' % - dict(cacert=self.cacert, exc=e)) - raise exc.SSLConfigurationError(msg) - else: - self.context.set_default_verify_paths() - - def connect(self): - """ - Connect to an SSL port using the OpenSSL library and apply - per-connection parameters. - """ - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if self.timeout is not None: - # '0' microseconds - sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, - struct.pack('fL', self.timeout, 0)) - self.sock = OpenSSLConnectionDelegator(self.context, sock) - self.sock.connect((self.host, self.port)) - - def close(self): - if self.sock: - # Removing reference to socket but don't close it yet. - # Response close will close both socket and associated - # file. Closing socket too soon will cause response - # reads to fail with socket IO error 'Bad file descriptor'. - self.sock = None - - # Calling close on HTTPConnection to continue doing that cleanup. - HTTPSConnection.close(self) - - -class ResponseBodyIterator(object): - """ - A class that acts as an iterator over an HTTP response. - - This class will also check response body integrity when iterating over - the instance and if a checksum was supplied using `set_checksum` method, - else by default the class will not do any integrity check. - """ - - def __init__(self, resp): - self._resp = resp - self._checksum = None - self._size = int(resp.getheader('content-length', 0)) - self._end_reached = False - - def set_checksum(self, checksum): - """ - Set checksum to check against when iterating over this instance. - - :raise: AttributeError if iterator is already consumed. - """ - if self._end_reached: - raise AttributeError("Can't set checksum for an already consumed" - " iterator") - self._checksum = checksum - - def __len__(self): - return int(self._size) - - def __iter__(self): - md5sum = hashlib.md5() - while True: - try: - chunk = self.next() - except StopIteration: - self._end_reached = True - # NOTE(mouad): Check image integrity when the end of response - # body is reached. - md5sum = md5sum.hexdigest() - if self._checksum is not None and md5sum != self._checksum: - raise IOError(errno.EPIPE, - 'Corrupted image. Checksum was %s ' - 'expected %s' % (md5sum, self._checksum)) - raise - else: - yield chunk - if isinstance(chunk, six.string_types): - chunk = six.b(chunk) - md5sum.update(chunk) - - def next(self): - chunk = self._resp.read(CHUNKSIZE) - if chunk: - return chunk - else: - raise StopIteration() + def delete(self, url, **kwargs): + return self._request('DELETE', url, **kwargs) diff --git a/glanceclient/common/https.py b/glanceclient/common/https.py new file mode 100644 index 0000000..6416c19 --- /dev/null +++ b/glanceclient/common/https.py @@ -0,0 +1,274 @@ +# Copyright 2014 Red Hat, Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import socket +import struct + +import OpenSSL +from requests import adapters +try: + from requests.packages.urllib3 import connectionpool + from requests.packages.urllib3 import poolmanager +except ImportError: + from urllib3 import connectionpool + from urllib3 import poolmanager + +import six +import ssl + +from glanceclient.common import utils + +try: + from eventlet import patcher + # Handle case where we are running in a monkey patched environment + if patcher.is_monkey_patched('socket'): + from eventlet.green.httplib import HTTPSConnection + from eventlet.green.OpenSSL.SSL import GreenConnection as Connection + from eventlet.greenio import GreenSocket + # TODO(mclaren): A getsockopt workaround: see 'getsockopt' doc string + GreenSocket.getsockopt = utils.getsockopt + else: + raise ImportError +except ImportError: + try: + from httplib import HTTPSConnection + except ImportError: + from http.client import HTTPSConnection + from OpenSSL.SSL import Connection as Connection + + +from glanceclient import exc + + +def to_bytes(s): + if isinstance(s, six.string_types): + return six.b(s) + else: + return s + + +class HTTPSAdapter(adapters.HTTPAdapter): + """ + This adapter will be used just when + ssl compression should be disabled. + + The init method overwrites the default + https pool by setting glanceclient's + one. + """ + + def __init__(self, *args, **kwargs): + # NOTE(flaper87): This line forces poolmanager to use + # glanceclient HTTPSConnection + poolmanager.pool_classes_by_scheme["https"] = HTTPSConnectionPool + super(HTTPSAdapter, self).__init__(*args, **kwargs) + + def cert_verify(self, conn, url, verify, cert): + super(HTTPSAdapter, self).cert_verify(conn, url, verify, cert) + conn.insecure = not verify + + +class HTTPSConnectionPool(connectionpool.HTTPSConnectionPool): + """ + HTTPSConnectionPool will be instantiated when a new + connection is requested to the HTTPSAdapter.This + implementation overwrites the _new_conn method and + returns an instances of glanceclient's VerifiedHTTPSConnection + which handles no compression. + + ssl_compression is hard-coded to False because this will + be used just when the user sets --no-ssl-compression. + """ + + scheme = 'https' + + def _new_conn(self): + self.num_connections += 1 + return VerifiedHTTPSConnection(host=self.host, + port=self.port, + key_file=self.key_file, + cert_file=self.cert_file, + cacert=self.ca_certs, + insecure=self.insecure, + ssl_compression=False) + + +class OpenSSLConnectionDelegator(object): + """ + An OpenSSL.SSL.Connection delegator. + + Supplies an additional 'makefile' method which httplib requires + and is not present in OpenSSL.SSL.Connection. + + Note: Since it is not possible to inherit from OpenSSL.SSL.Connection + a delegator must be used. + """ + def __init__(self, *args, **kwargs): + self.connection = Connection(*args, **kwargs) + + def __getattr__(self, name): + return getattr(self.connection, name) + + def makefile(self, *args, **kwargs): + return socket._fileobject(self.connection, *args, **kwargs) + + +class VerifiedHTTPSConnection(HTTPSConnection): + """ + Extended HTTPSConnection which uses the OpenSSL library + for enhanced SSL support. + Note: Much of this functionality can eventually be replaced + with native Python 3.3 code. + """ + def __init__(self, host, port=None, key_file=None, cert_file=None, + cacert=None, timeout=None, insecure=False, + ssl_compression=True): + # List of exceptions reported by Python3 instead of + # SSLConfigurationError + if six.PY3: + excp_lst = (TypeError, FileNotFoundError, ssl.SSLError) + else: + excp_lst = () + try: + HTTPSConnection.__init__(self, host, port, + key_file=key_file, + cert_file=cert_file) + self.key_file = key_file + self.cert_file = cert_file + self.timeout = timeout + self.insecure = insecure + self.ssl_compression = ssl_compression + self.cacert = None if cacert is None else str(cacert) + self.set_context() + # ssl exceptions are reported in various form in Python 3 + # so to be compatible, we report the same kind as under + # Python2 + except excp_lst as e: + raise exc.SSLConfigurationError(str(e)) + + @staticmethod + def host_matches_cert(host, x509): + """ + Verify that the x509 certificate we have received + from 'host' correctly identifies the server we are + connecting to, ie that the certificate's Common Name + or a Subject Alternative Name matches 'host'. + """ + def check_match(name): + # Directly match the name + if name == host: + return True + + # Support single wildcard matching + if name.startswith('*.') and host.find('.') > 0: + if name[2:] == host.split('.', 1)[1]: + return True + + common_name = x509.get_subject().commonName + + # First see if we can match the CN + if check_match(common_name): + return True + # Also try Subject Alternative Names for a match + san_list = None + for i in range(x509.get_extension_count()): + ext = x509.get_extension(i) + if ext.get_short_name() == b'subjectAltName': + san_list = str(ext) + for san in ''.join(san_list.split()).split(','): + if san.startswith('DNS:'): + if check_match(san.split(':', 1)[1]): + return True + + # Server certificate does not match host + msg = ('Host "%s" does not match x509 certificate contents: ' + 'CommonName "%s"' % (host, common_name)) + if san_list is not None: + msg = msg + ', subjectAltName "%s"' % san_list + raise exc.SSLCertificateError(msg) + + def verify_callback(self, connection, x509, errnum, + depth, preverify_ok): + if x509.has_expired(): + msg = "SSL Certificate expired on '%s'" % x509.get_notAfter() + raise exc.SSLCertificateError(msg) + + if depth == 0 and preverify_ok: + # We verify that the host matches against the last + # certificate in the chain + return self.host_matches_cert(self.host, x509) + else: + # Pass through OpenSSL's default result + return preverify_ok + + def set_context(self): + """ + Set up the OpenSSL context. + """ + self.context = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) + + if self.ssl_compression is False: + self.context.set_options(0x20000) # SSL_OP_NO_COMPRESSION + + if self.insecure is not True: + self.context.set_verify(OpenSSL.SSL.VERIFY_PEER, + self.verify_callback) + else: + self.context.set_verify(OpenSSL.SSL.VERIFY_NONE, + lambda *args: True) + + if self.cert_file: + try: + self.context.use_certificate_file(self.cert_file) + except Exception as e: + msg = 'Unable to load cert from "%s" %s' % (self.cert_file, e) + raise exc.SSLConfigurationError(msg) + if self.key_file is None: + # We support having key and cert in same file + try: + self.context.use_privatekey_file(self.cert_file) + except Exception as e: + msg = ('No key file specified and unable to load key ' + 'from "%s" %s' % (self.cert_file, e)) + raise exc.SSLConfigurationError(msg) + + if self.key_file: + try: + self.context.use_privatekey_file(self.key_file) + except Exception as e: + msg = 'Unable to load key from "%s" %s' % (self.key_file, e) + raise exc.SSLConfigurationError(msg) + + if self.cacert: + try: + self.context.load_verify_locations(to_bytes(self.cacert)) + except Exception as e: + msg = 'Unable to load CA from "%s" %s' % (self.cacert, e) + raise exc.SSLConfigurationError(msg) + else: + self.context.set_default_verify_paths() + + def connect(self): + """ + Connect to an SSL port using the OpenSSL library and apply + per-connection parameters. + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if self.timeout is not None: + # '0' microseconds + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, + struct.pack('LL', self.timeout, 0)) + self.sock = OpenSSLConnectionDelegator(self.context, sock) + self.sock.connect((self.host, self.port)) diff --git a/glanceclient/common/utils.py b/glanceclient/common/utils.py index 6cdc364..04f5add 100644 --- a/glanceclient/common/utils.py +++ b/glanceclient/common/utils.py @@ -16,6 +16,7 @@ from __future__ import print_function import errno +import hashlib import os import re import sys @@ -335,3 +336,22 @@ def print_image(image_obj, max_col_width=None): print_dict(image, max_column_width=max_col_width) else: print_dict(image) + + +def integrity_iter(iter, checksum): + """ + Check image data integrity. + + :raises: IOError + """ + md5sum = hashlib.md5() + for chunk in iter: + yield chunk + if isinstance(chunk, six.string_types): + chunk = six.b(chunk) + md5sum.update(chunk) + md5sum = md5sum.hexdigest() + if md5sum != checksum: + raise IOError(errno.EPIPE, + 'Corrupt image download. Checksum was %s expected %s' % + (md5sum, checksum)) diff --git a/glanceclient/exc.py b/glanceclient/exc.py index 9caa24d..3eeaffa 100644 --- a/glanceclient/exc.py +++ b/glanceclient/exc.py @@ -152,7 +152,7 @@ for obj_name in dir(sys.modules[__name__]): def from_response(response, body=None): """Return an instance of an HTTPException based on httplib response.""" - cls = _code_map.get(response.status, HTTPException) + cls = _code_map.get(response.status_code, HTTPException) if body: details = body.replace('\n\n', '\n') return cls(details=details) diff --git a/glanceclient/v1/client.py b/glanceclient/v1/client.py index 23bb737..aeb94a2 100644 --- a/glanceclient/v1/client.py +++ b/glanceclient/v1/client.py @@ -13,10 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. -from glanceclient.common import http +from glanceclient.common.http import HTTPClient from glanceclient.common import utils -from glanceclient.v1 import image_members -from glanceclient.v1 import images +from glanceclient.v1.image_members import ImageMemberManager +from glanceclient.v1.images import ImageManager class Client(object): @@ -31,7 +31,7 @@ class Client(object): def __init__(self, endpoint, *args, **kwargs): """Initialize a new client for the Images v1 API.""" - self.http_client = http.HTTPClient(utils.strip_version(endpoint), - *args, **kwargs) - self.images = images.ImageManager(self.http_client) - self.image_members = image_members.ImageMemberManager(self.http_client) + self.http_client = HTTPClient(utils.strip_version(endpoint), + *args, **kwargs) + self.images = ImageManager(self.http_client) + self.image_members = ImageMemberManager(self.http_client) diff --git a/glanceclient/v1/image_members.py b/glanceclient/v1/image_members.py index f464fb8..d940a5f 100644 --- a/glanceclient/v1/image_members.py +++ b/glanceclient/v1/image_members.py @@ -34,7 +34,7 @@ class ImageMemberManager(base.ManagerWithFind): def get(self, image, member_id): image_id = base.getid(image) url = '/v1/images/%s/members/%s' % (image_id, member_id) - resp, body = self.client.json_request('GET', url) + resp, body = self.client.get(url) member = body['member'] member['image_id'] = image_id return ImageMember(self, member, loaded=True) @@ -60,7 +60,7 @@ class ImageMemberManager(base.ManagerWithFind): def _list_by_image(self, image): image_id = base.getid(image) url = '/v1/images/%s/members' % image_id - resp, body = self.client.json_request('GET', url) + resp, body = self.client.get(url) out = [] for member in body['members']: member['image_id'] = image_id @@ -70,7 +70,7 @@ class ImageMemberManager(base.ManagerWithFind): def _list_by_member(self, member): member_id = base.getid(member) url = '/v1/shared-images/%s' % member_id - resp, body = self.client.json_request('GET', url) + resp, body = self.client.get(url) out = [] for member in body['shared_images']: member['member_id'] = member_id @@ -84,7 +84,7 @@ class ImageMemberManager(base.ManagerWithFind): """Creates an image.""" url = '/v1/images/%s/members/%s' % (base.getid(image), member_id) body = {'member': {'can_share': can_share}} - self._put(url, json=body) + self.client.put(url, data=body) def replace(self, image, members): memberships = [] @@ -100,4 +100,4 @@ class ImageMemberManager(base.ManagerWithFind): obj['can_share'] = member['can_share'] memberships.append(obj) url = '/v1/images/%s/members' % base.getid(image) - self.client.json_request('PUT', url, {}, {'memberships': memberships}) + self.client.put(url, data={'memberships': memberships}) diff --git a/glanceclient/v1/images.py b/glanceclient/v1/images.py index d2af595..87060c2 100644 --- a/glanceclient/v1/images.py +++ b/glanceclient/v1/images.py @@ -14,10 +14,9 @@ # under the License. import copy -import json import six -from six.moves.urllib import parse +import six.moves.urllib.parse as urlparse from glanceclient.common import utils from glanceclient.openstack.common.apiclient import base @@ -60,12 +59,12 @@ class ImageManager(base.ManagerWithFind): resource_class = Image def _list(self, url, response_key, obj_class=None, body=None): - resp = self.client.get(url) + resp, body = self.client.get(url) if obj_class is None: obj_class = self.resource_class - data = resp.json()[response_key] + data = body[response_key] return ([obj_class(self, res, loaded=True) for res in data if res], resp) @@ -123,13 +122,12 @@ class ImageManager(base.ManagerWithFind): :rtype: :class:`Image` """ image_id = base.getid(image) - resp, body = self.client.raw_request( - 'HEAD', '/v1/images/%s' % parse.quote(str(image_id))) - meta = self._image_meta_from_headers(dict(resp.getheaders())) + resp, body = self.client.head('/v1/images/%s' + % urlparse.quote(str(image_id))) + meta = self._image_meta_from_headers(resp.headers) return_request_id = kwargs.get('return_req_id', None) if return_request_id is not None: - return_request_id.append(resp.getheader(OS_REQ_ID_HDR, None)) - + return_request_id.append(resp.headers.get(OS_REQ_ID_HDR, None)) return Image(self, meta) def data(self, image, do_checksum=True, **kwargs): @@ -140,14 +138,14 @@ class ImageManager(base.ManagerWithFind): :rtype: iterable containing image data """ image_id = base.getid(image) - resp, body = self.client.raw_request( - 'GET', '/v1/images/%s' % parse.quote(str(image_id))) - checksum = resp.getheader('x-image-meta-checksum', None) + resp, body = self.client.get('/v1/images/%s' + % urlparse.quote(str(image_id))) + checksum = resp.headers.get('x-image-meta-checksum', None) if do_checksum and checksum is not None: - body.set_checksum(checksum) + return utils.integrity_iter(body, checksum) return_request_id = kwargs.get('return_req_id', None) if return_request_id is not None: - return_request_id.append(resp.getheader(OS_REQ_ID_HDR, None)) + return_request_id.append(resp.headers.get(OS_REQ_ID_HDR, None)) return body @@ -194,11 +192,11 @@ class ImageManager(base.ManagerWithFind): # trying to encode them qp[param] = strutils.safe_encode(value) - url = '/v1/images/detail?%s' % parse.urlencode(qp) + url = '/v1/images/detail?%s' % urlparse.urlencode(qp) images, resp = self._list(url, "images") if return_request_id is not None: - return_request_id.append(resp.getheader(OS_REQ_ID_HDR, None)) + return_request_id.append(resp.headers.get(OS_REQ_ID_HDR, None)) for image in images: if filter_owner(owner, image): @@ -253,10 +251,11 @@ class ImageManager(base.ManagerWithFind): def delete(self, image, **kwargs): """Delete an image.""" - resp = self._delete("/v1/images/%s" % base.getid(image))[0] + url = "/v1/images/%s" % base.getid(image) + resp, body = self.client.delete(url) return_request_id = kwargs.get('return_req_id', None) if return_request_id is not None: - return_request_id.append(resp.getheader(OS_REQ_ID_HDR, None)) + return_request_id.append(resp.headers.get(OS_REQ_ID_HDR, None)) def create(self, **kwargs): """Create an image @@ -284,12 +283,12 @@ class ImageManager(base.ManagerWithFind): if copy_from is not None: hdrs['x-glance-api-copy-from'] = copy_from - resp, body_iter = self.client.raw_request( - 'POST', '/v1/images', headers=hdrs, body=image_data) - body = json.loads(''.join([c for c in body_iter])) + resp, body = self.client.post('/v1/images', + headers=hdrs, + data=image_data) return_request_id = kwargs.get('return_req_id', None) if return_request_id is not None: - return_request_id.append(resp.getheader(OS_REQ_ID_HDR, None)) + return_request_id.append(resp.headers.get(OS_REQ_ID_HDR, None)) return Image(self, self._format_image_meta_for_user(body['image'])) @@ -327,11 +326,9 @@ class ImageManager(base.ManagerWithFind): hdrs['x-glance-api-copy-from'] = copy_from url = '/v1/images/%s' % base.getid(image) - resp, body_iter = self.client.raw_request( - 'PUT', url, headers=hdrs, body=image_data) - body = json.loads(''.join([c for c in body_iter])) + resp, body = self.client.put(url, headers=hdrs, data=image_data) return_request_id = kwargs.get('return_req_id', None) if return_request_id is not None: - return_request_id.append(resp.getheader(OS_REQ_ID_HDR, None)) + return_request_id.append(resp.headers.get(OS_REQ_ID_HDR, None)) return Image(self, self._format_image_meta_for_user(body['image'])) diff --git a/glanceclient/v2/image_members.py b/glanceclient/v2/image_members.py index b2632a2..dcf4ac2 100644 --- a/glanceclient/v2/image_members.py +++ b/glanceclient/v2/image_members.py @@ -21,25 +21,22 @@ class Controller(object): def list(self, image_id): url = '/v2/images/%s/members' % image_id - resp, body = self.http_client.json_request('GET', url) + resp, body = self.http_client.get(url) for member in body['members']: yield self.model(member) def delete(self, image_id, member_id): - self.http_client.json_request('DELETE', - '/v2/images/%s/members/%s' % - (image_id, member_id)) + self.http_client.delete('/v2/images/%s/members/%s' % + (image_id, member_id)) def update(self, image_id, member_id, member_status): url = '/v2/images/%s/members/%s' % (image_id, member_id) body = {'status': member_status} - resp, updated_member = self.http_client.json_request('PUT', url, - body=body) + resp, updated_member = self.http_client.put(url, data=body) return self.model(updated_member) def create(self, image_id, member_id): url = '/v2/images/%s/members' % image_id body = {'member': member_id} - resp, created_member = self.http_client.json_request('POST', url, - body=body) + resp, created_member = self.http_client.post(url, data=body) return self.model(created_member) diff --git a/glanceclient/v2/image_tags.py b/glanceclient/v2/image_tags.py index a943d6a..5c03648 100644 --- a/glanceclient/v2/image_tags.py +++ b/glanceclient/v2/image_tags.py @@ -27,7 +27,7 @@ class Controller(object): :param tag_value: value of the tag. """ url = '/v2/images/%s/tags/%s' % (image_id, tag_value) - self.http_client.json_request('PUT', url) + self.http_client.put(url) def delete(self, image_id, tag_value): """ @@ -37,4 +37,4 @@ class Controller(object): :param tag_value: tag value to be deleted. """ url = '/v2/images/%s/tags/%s' % (image_id, tag_value) - self.http_client.json_request('DELETE', url) + self.http_client.delete(url) diff --git a/glanceclient/v2/images.py b/glanceclient/v2/images.py index 10b7864..1144e6f 100644 --- a/glanceclient/v2/images.py +++ b/glanceclient/v2/images.py @@ -16,7 +16,6 @@ import json import six from six.moves.urllib import parse - import warlock from glanceclient.common import utils @@ -42,7 +41,7 @@ class Controller(object): empty_fun = lambda *args, **kwargs: None def paginate(url): - resp, body = self.http_client.json_request('GET', url) + resp, body = self.http_client.get(url) for image in body['images']: # NOTE(bcwaldon): remove 'self' for now until we have # an elegant way to pass it into the model constructor @@ -94,7 +93,7 @@ class Controller(object): def get(self, image_id): url = '/v2/images/%s' % image_id - resp, body = self.http_client.json_request('GET', url) + resp, body = self.http_client.get(url) #NOTE(bcwaldon): remove 'self' for now until we have an elegant # way to pass it into the model constructor without conflict body.pop('self', None) @@ -108,11 +107,12 @@ class Controller(object): :param do_checksum: Enable/disable checksum validation. """ url = '/v2/images/%s/file' % image_id - resp, body = self.http_client.raw_request('GET', url) - checksum = resp.getheader('content-md5', None) + resp, body = self.http_client.get(url) + checksum = resp.headers.get('content-md5', None) if do_checksum and checksum is not None: - body.set_checksum(checksum) - return body + return utils.integrity_iter(body, checksum) + else: + return body def upload(self, image_id, image_data, image_size=None): """ @@ -124,14 +124,17 @@ class Controller(object): """ url = '/v2/images/%s/file' % image_id hdrs = {'Content-Type': 'application/octet-stream'} - self.http_client.raw_request('PUT', url, - headers=hdrs, - body=image_data, - content_length=image_size) + if image_size: + body = {'image_data': image_data, + 'image_size': image_size} + else: + body = image_data + self.http_client.put(url, headers=hdrs, data=body) def delete(self, image_id): """Delete an image.""" - self.http_client.json_request('DELETE', '/v2/images/%s' % image_id) + url = '/v2/images/%s' % image_id + self.http_client.delete(url) def create(self, **kwargs): """Create an image.""" @@ -144,7 +147,7 @@ class Controller(object): except warlock.InvalidOperation as e: raise TypeError(utils.exception_to_str(e)) - resp, body = self.http_client.json_request('POST', url, body=image) + resp, body = self.http_client.post(url, data=image) #NOTE(esheffield): remove 'self' for now until we have an elegant # way to pass it into the model constructor without conflict body.pop('self', None) @@ -178,9 +181,7 @@ class Controller(object): url = '/v2/images/%s' % image_id hdrs = {'Content-Type': 'application/openstack-images-v2.1-json-patch'} - self.http_client.raw_request('PATCH', url, - headers=hdrs, - body=image.patch) + self.http_client.patch(url, headers=hdrs, data=image.patch) #NOTE(bcwaldon): calling image.patch doesn't clear the changes, so # we need to fetch the image again to get a clean history. This is @@ -197,9 +198,7 @@ class Controller(object): def _send_image_update_request(self, image_id, patch_body): url = '/v2/images/%s' % image_id hdrs = {'Content-Type': 'application/openstack-images-v2.1-json-patch'} - self.http_client.raw_request('PATCH', url, - headers=hdrs, - body=json.dumps(patch_body)) + self.http_client.patch(url, headers=hdrs, data=json.dumps(patch_body)) def add_location(self, image_id, url, metadata): """Add a new location entry to an image's list of locations. diff --git a/glanceclient/v2/schemas.py b/glanceclient/v2/schemas.py index 57f6cc7..7cd169d 100644 --- a/glanceclient/v2/schemas.py +++ b/glanceclient/v2/schemas.py @@ -81,5 +81,5 @@ class Controller(object): def get(self, schema_name): uri = '/v2/schemas/%s' % schema_name - _, raw_schema = self.http_client.json_request('GET', uri) + _, raw_schema = self.http_client.get(uri) return Schema(raw_schema) |
