Copy s3_token middleware from keystone

The s3_token middleware was in keystone but it should be in
python-keystoneclient. As a first step to removing s3_token from
keystone, copy it to python-keystoneclient. Also copy the tests.

A couple of changes were required

- Changed oslo-incubator imports from keystone to keystoneclient.
- Changed logging from oslo-incubator to use Python logging and
  configured the same way as the auth_token middleware.

I checked and I didn't see anything to be added to requirements.txt
or test-requirements.txt.

bp s3-token-to-keystoneclient

Change-Id: I64fef4101180e5aa661442d538c3237bdad0c37c
Closes-Bug: #1178741

2 files changed, 496 insertions, 0 deletions

diff --git a/keystoneclient/middleware/s3_token.py b/keystoneclient/middleware/s3_token.py
new file mode 100644
index 0000000..f35b154
--- /dev/null
+++ b/keystoneclient/middleware/s3_token.py
@@ -0,0 +1,263 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 OpenStack Foundation
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011,2012 Akira YOSHIYAMA <akirayoshiyama@gmail.com>
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This source code is based ./auth_token.py and ./ec2_token.py.
+# See them for their copyright.
+
+"""
+S3 TOKEN MIDDLEWARE
+
+This WSGI component:
+
+* Get a request from the swift3 middleware with an S3 Authorization
+  access key.
+* Validate s3 token in Keystone.
+* Transform the account name to AUTH_%(tenant_name).
+
+"""
+
+import httplib
+import logging
+import urllib
+import webob
+
+from keystoneclient.openstack.common import jsonutils
+
+
+PROTOCOL_NAME = 'S3 Token Authentication'
+
+
+# TODO(kun): remove it after oslo merge this.
+def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False):
+    """Validate and split the given HTTP request path.
+
+    **Examples**::
+
+        ['a'] = split_path('/a')
+        ['a', None] = split_path('/a', 1, 2)
+        ['a', 'c'] = split_path('/a/c', 1, 2)
+        ['a', 'c', 'o/r'] = split_path('/a/c/o/r', 1, 3, True)
+
+    :param path: HTTP Request path to be split
+    :param minsegs: Minimum number of segments to be extracted
+    :param maxsegs: Maximum number of segments to be extracted
+    :param rest_with_last: If True, trailing data will be returned as part
+                           of last segment.  If False, and there is
+                           trailing data, raises ValueError.
+    :returns: list of segments with a length of maxsegs (non-existant
+              segments will return as None)
+    :raises: ValueError if given an invalid path
+    """
+    if not maxsegs:
+        maxsegs = minsegs
+    if minsegs > maxsegs:
+        raise ValueError('minsegs > maxsegs: %d > %d' % (minsegs, maxsegs))
+    if rest_with_last:
+        segs = path.split('/', maxsegs)
+        minsegs += 1
+        maxsegs += 1
+        count = len(segs)
+        if (segs[0] or count < minsegs or count > maxsegs or
+                '' in segs[1:minsegs]):
+            raise ValueError('Invalid path: %s' % urllib.quote(path))
+    else:
+        minsegs += 1
+        maxsegs += 1
+        segs = path.split('/', maxsegs)
+        count = len(segs)
+        if (segs[0] or count < minsegs or count > maxsegs + 1 or
+                '' in segs[1:minsegs] or
+                (count == maxsegs + 1 and segs[maxsegs])):
+            raise ValueError('Invalid path: %s' % urllib.quote(path))
+    segs = segs[1:maxsegs]
+    segs.extend([None] * (maxsegs - 1 - len(segs)))
+    return segs
+
+
+class ServiceError(Exception):
+    pass
+
+
+class S3Token(object):
+    """Auth Middleware that handles S3 authenticating client calls."""
+
+    def __init__(self, app, conf):
+        """Common initialization code."""
+        self.app = app
+        self.logger = logging.getLogger(conf.get('log_name', __name__))
+        self.logger.debug('Starting the %s component' % PROTOCOL_NAME)
+        self.reseller_prefix = conf.get('reseller_prefix', 'AUTH_')
+        # where to find the auth service (we use this to validate tokens)
+        self.auth_host = conf.get('auth_host')
+        self.auth_port = int(conf.get('auth_port', 35357))
+        self.auth_protocol = conf.get('auth_protocol', 'https')
+        if self.auth_protocol == 'http':
+            self.http_client_class = httplib.HTTPConnection
+        else:
+            self.http_client_class = httplib.HTTPSConnection
+        # SSL
+        self.cert_file = conf.get('certfile')
+        self.key_file = conf.get('keyfile')
+
+    def deny_request(self, code):
+        error_table = {
+            'AccessDenied': (401, 'Access denied'),
+            'InvalidURI': (400, 'Could not parse the specified URI'),
+        }
+        resp = webob.Response(content_type='text/xml')
+        resp.status = error_table[code][0]
+        resp.body = error_table[code][1]
+        resp.body = ('<?xml version="1.0" encoding="UTF-8"?>\r\n'
+                     '<Error>\r\n  <Code>%s</Code>\r\n  '
+                     '<Message>%s</Message>\r\n</Error>\r\n' %
+                     (code, error_table[code][1]))
+        return resp
+
+    def _json_request(self, creds_json):
+        headers = {'Content-Type': 'application/json'}
+        if self.auth_protocol == 'http':
+            conn = self.http_client_class(self.auth_host, self.auth_port)
+        else:
+            conn = self.http_client_class(self.auth_host,
+                                          self.auth_port,
+                                          self.key_file,
+                                          self.cert_file)
+        try:
+            conn.request('POST', '/v2.0/s3tokens',
+                         body=creds_json,
+                         headers=headers)
+            response = conn.getresponse()
+            output = response.read()
+        except Exception as e:
+            self.logger.info('HTTP connection exception: %s' % e)
+            resp = self.deny_request('InvalidURI')
+            raise ServiceError(resp)
+        finally:
+            conn.close()
+
+        if response.status < 200 or response.status >= 300:
+            self.logger.debug('Keystone reply error: status=%s reason=%s' %
+                              (response.status, response.reason))
+            resp = self.deny_request('AccessDenied')
+            raise ServiceError(resp)
+
+        return (response, output)
+
+    def __call__(self, environ, start_response):
+        """Handle incoming request. authenticate and send downstream."""
+        req = webob.Request(environ)
+        self.logger.debug('Calling S3Token middleware.')
+
+        try:
+            parts = split_path(req.path, 1, 4, True)
+            version, account, container, obj = parts
+        except ValueError:
+            msg = 'Not a path query, skipping.'
+            self.logger.debug(msg)
+            return self.app(environ, start_response)
+
+        # Read request signature and access id.
+        if 'Authorization' not in req.headers:
+            msg = 'No Authorization header. skipping.'
+            self.logger.debug(msg)
+            return self.app(environ, start_response)
+
+        token = req.headers.get('X-Auth-Token',
+                                req.headers.get('X-Storage-Token'))
+        if not token:
+            msg = 'You did not specify a auth or a storage token. skipping.'
+            self.logger.debug(msg)
+            return self.app(environ, start_response)
+
+        auth_header = req.headers['Authorization']
+        try:
+            access, signature = auth_header.split(' ')[-1].rsplit(':', 1)
+        except ValueError:
+            msg = 'You have an invalid Authorization header: %s'
+            self.logger.debug(msg % (auth_header))
+            return self.deny_request('InvalidURI')(environ, start_response)
+
+        # NOTE(chmou): This is to handle the special case with nova
+        # when we have the option s3_affix_tenant. We will force it to
+        # connect to another account than the one
+        # authenticated. Before people start getting worried about
+        # security, I should point that we are connecting with
+        # username/token specified by the user but instead of
+        # connecting to its own account we will force it to go to an
+        # another account. In a normal scenario if that user don't
+        # have the reseller right it will just fail but since the
+        # reseller account can connect to every account it is allowed
+        # by the swift_auth middleware.
+        force_tenant = None
+        if ':' in access:
+            access, force_tenant = access.split(':')
+
+        # Authenticate request.
+        creds = {'credentials': {'access': access,
+                                 'token': token,
+                                 'signature': signature}}
+        creds_json = jsonutils.dumps(creds)
+        self.logger.debug('Connecting to Keystone sending this JSON: %s' %
+                          creds_json)
+        # NOTE(vish): We could save a call to keystone by having
+        #             keystone return token, tenant, user, and roles
+        #             from this call.
+        #
+        # NOTE(chmou): We still have the same problem we would need to
+        #              change token_auth to detect if we already
+        #              identified and not doing a second query and just
+        #              pass it through to swiftauth in this case.
+        try:
+            resp, output = self._json_request(creds_json)
+        except ServiceError as e:
+            resp = e.args[0]
+            msg = 'Received error, exiting middleware with error: %s'
+            self.logger.debug(msg % (resp.status))
+            return resp(environ, start_response)
+
+        self.logger.debug('Keystone Reply: Status: %d, Output: %s' % (
+                          resp.status, output))
+
+        try:
+            identity_info = jsonutils.loads(output)
+            token_id = str(identity_info['access']['token']['id'])
+            tenant = identity_info['access']['token']['tenant']
+        except (ValueError, KeyError):
+            error = 'Error on keystone reply: %d %s'
+            self.logger.debug(error % (resp.status, str(output)))
+            return self.deny_request('InvalidURI')(environ, start_response)
+
+        req.headers['X-Auth-Token'] = token_id
+        tenant_to_connect = force_tenant or tenant['id']
+        self.logger.debug('Connecting with tenant: %s' % (tenant_to_connect))
+        new_tenant_name = '%s%s' % (self.reseller_prefix, tenant_to_connect)
+        environ['PATH_INFO'] = environ['PATH_INFO'].replace(account,
+                                                            new_tenant_name)
+        return self.app(environ, start_response)
+
+
+def filter_factory(global_conf, **local_conf):
+    """Returns a WSGI filter app for use with paste.deploy."""
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def auth_filter(app):
+        return S3Token(app, conf)
+    return auth_filter
diff --git a/keystoneclient/tests/test_s3_token_middleware.py b/keystoneclient/tests/test_s3_token_middleware.py
new file mode 100644
index 0000000..bea359a
--- /dev/null
+++ b/keystoneclient/tests/test_s3_token_middleware.py
@@ -0,0 +1,233 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import testtools
+import webob
+
+from keystoneclient.middleware import s3_token
+from keystoneclient.openstack.common import jsonutils
+
+
+class FakeHTTPResponse(object):
+    def __init__(self, status, body):
+        self.status = status
+        self.body = body
+        self.reason = ""
+
+    def read(self):
+        return self.body
+
+
+class FakeApp(object):
+    """This represents a WSGI app protected by the auth_token middleware."""
+    def __call__(self, env, start_response):
+        resp = webob.Response()
+        resp.environ = env
+        return resp(env, start_response)
+
+
+class FakeHTTPConnection(object):
+    def __init__(self, *args):
+        return
+
+    def getresponse(self):
+        return self.resp
+
+    def close(self):
+        pass
+
+    def request(self, method, path, **kwargs):
+        pass
+
+
+class S3TokenMiddlewareTestBase(testtools.TestCase):
+    def setUp(self):
+        super(S3TokenMiddlewareTestBase, self).setUp()
+
+    def start_fake_response(self, status, headers):
+        self.response_status = int(status.split(' ', 1)[0])
+        self.response_headers = dict(headers)
+
+
+def good_request(cls, method, path, **kwargs):
+    cls.status = 201
+    ret = {'access': {'token':
+                      {'id': 'TOKEN_ID',
+                       'tenant': {'id': 'TENANT_ID'}}}}
+    body = jsonutils.dumps(ret)
+    cls.resp = FakeHTTPResponse(cls.status, body)
+
+
+class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
+    def setup_middleware_fake(self):
+        self.middleware.http_client_class = FakeHTTPConnection
+        self.middleware.http_client_class.request = good_request
+
+    def setUp(self):
+        self.middleware = s3_token.S3Token(FakeApp(), {})
+        self.setup_middleware_fake()
+        super(S3TokenMiddlewareTestGood, self).setUp()
+
+    # Ignore the request and pass to the next middleware in the
+    # pipeline if no path has been specified.
+    def test_no_path_request(self):
+        req = webob.Request.blank('/')
+        self.middleware(req.environ, self.start_fake_response)
+        self.assertEqual(self.response_status, 200)
+
+    # Ignore the request and pass to the next middleware in the
+    # pipeline if no Authorization header has been specified
+    def test_without_authorization(self):
+        req = webob.Request.blank('/v1/AUTH_cfa/c/o')
+        self.middleware(req.environ, self.start_fake_response)
+        self.assertEqual(self.response_status, 200)
+
+    def test_without_auth_storage_token(self):
+        req = webob.Request.blank('/v1/AUTH_cfa/c/o')
+        req.headers['Authorization'] = 'badboy'
+        self.middleware(req.environ, self.start_fake_response)
+        self.assertEqual(self.response_status, 200)
+
+    def test_authorized(self):
+        req = webob.Request.blank('/v1/AUTH_cfa/c/o')
+        req.headers['Authorization'] = 'access:signature'
+        req.headers['X-Storage-Token'] = 'token'
+        req.get_response(self.middleware)
+        self.assertTrue(req.path.startswith('/v1/AUTH_TENANT_ID'))
+        self.assertEqual(req.headers['X-Auth-Token'], 'TOKEN_ID')
+
+    def test_authorized_http(self):
+        self.middleware = (
+            s3_token.filter_factory({'auth_protocol': 'http'})(FakeApp()))
+        self.setup_middleware_fake()
+        req = webob.Request.blank('/v1/AUTH_cfa/c/o')
+        req.headers['Authorization'] = 'access:signature'
+        req.headers['X-Storage-Token'] = 'token'
+        req.get_response(self.middleware)
+        self.assertTrue(req.path.startswith('/v1/AUTH_TENANT_ID'))
+        self.assertEqual(req.headers['X-Auth-Token'], 'TOKEN_ID')
+
+    def test_authorization_nova_toconnect(self):
+        req = webob.Request.blank('/v1/AUTH_swiftint/c/o')
+        req.headers['Authorization'] = 'access:FORCED_TENANT_ID:signature'
+        req.headers['X-Storage-Token'] = 'token'
+        req.get_response(self.middleware)
+        path = req.environ['PATH_INFO']
+        self.assertTrue(path.startswith('/v1/AUTH_FORCED_TENANT_ID'))
+
+
+class S3TokenMiddlewareTestBad(S3TokenMiddlewareTestBase):
+    def setUp(self):
+        self.middleware = s3_token.S3Token(FakeApp(), {})
+        self.middleware.http_client_class = FakeHTTPConnection
+        super(S3TokenMiddlewareTestBad, self).setUp()
+
+    def test_unauthorized_token(self):
+        def request(self, method, path, **kwargs):
+            ret = {"error":
+                   {"message": "EC2 access key not found.",
+                    "code": 401,
+                    "title": "Unauthorized"}}
+            body = jsonutils.dumps(ret)
+            self.status = 403
+            self.resp = FakeHTTPResponse(self.status, body)
+
+        req = webob.Request.blank('/v1/AUTH_cfa/c/o')
+        req.headers['Authorization'] = 'access:signature'
+        req.headers['X-Storage-Token'] = 'token'
+        self.middleware.http_client_class.request = request
+        resp = req.get_response(self.middleware)
+        s3_denied_req = self.middleware.deny_request('AccessDenied')
+        self.assertEqual(resp.body, s3_denied_req.body)
+        self.assertEqual(resp.status_int, s3_denied_req.status_int)
+
+    def test_bogus_authorization(self):
+        req = webob.Request.blank('/v1/AUTH_cfa/c/o')
+        req.headers['Authorization'] = 'badboy'
+        req.headers['X-Storage-Token'] = 'token'
+        resp = req.get_response(self.middleware)
+        self.assertEqual(resp.status_int, 400)
+        s3_invalid_req = self.middleware.deny_request('InvalidURI')
+        self.assertEqual(resp.body, s3_invalid_req.body)
+        self.assertEqual(resp.status_int, s3_invalid_req.status_int)
+
+    def test_fail_to_connect_to_keystone(self):
+        def request(self, method, path, **kwargs):
+            raise s3_token.ServiceError
+        self.middleware.http_client_class.request = request
+
+        req = webob.Request.blank('/v1/AUTH_cfa/c/o')
+        req.headers['Authorization'] = 'access:signature'
+        req.headers['X-Storage-Token'] = 'token'
+        self.middleware.http_client_class.status = 503
+        resp = req.get_response(self.middleware)
+        s3_invalid_req = self.middleware.deny_request('InvalidURI')
+        self.assertEqual(resp.body, s3_invalid_req.body)
+        self.assertEqual(resp.status_int, s3_invalid_req.status_int)
+
+    def test_bad_reply(self):
+        def request(self, method, path, **kwargs):
+            body = "<badreply>"
+            self.status = 201
+            self.resp = FakeHTTPResponse(self.status, body)
+
+        req = webob.Request.blank('/v1/AUTH_cfa/c/o')
+        req.headers['Authorization'] = 'access:signature'
+        req.headers['X-Storage-Token'] = 'token'
+        self.middleware.http_client_class.request = request
+        resp = req.get_response(self.middleware)
+        s3_invalid_req = self.middleware.deny_request('InvalidURI')
+        self.assertEqual(resp.body, s3_invalid_req.body)
+        self.assertEqual(resp.status_int, s3_invalid_req.status_int)
+
+
+class S3TokenMiddlewareTestUtil(testtools.TestCase):
+    def test_split_path_failed(self):
+        self.assertRaises(ValueError, s3_token.split_path, '')
+        self.assertRaises(ValueError, s3_token.split_path, '/')
+        self.assertRaises(ValueError, s3_token.split_path, '//')
+        self.assertRaises(ValueError, s3_token.split_path, '//a')
+        self.assertRaises(ValueError, s3_token.split_path, '/a/c')
+        self.assertRaises(ValueError, s3_token.split_path, '//c')
+        self.assertRaises(ValueError, s3_token.split_path, '/a/c/')
+        self.assertRaises(ValueError, s3_token.split_path, '/a//')
+        self.assertRaises(ValueError, s3_token.split_path, '/a', 2)
+        self.assertRaises(ValueError, s3_token.split_path, '/a', 2, 3)
+        self.assertRaises(ValueError, s3_token.split_path, '/a', 2, 3, True)
+        self.assertRaises(ValueError, s3_token.split_path, '/a/c/o/r', 3, 3)
+        self.assertRaises(ValueError, s3_token.split_path, '/a', 5, 4)
+
+    def test_split_path_success(self):
+        self.assertEqual(s3_token.split_path('/a'), ['a'])
+        self.assertEqual(s3_token.split_path('/a/'), ['a'])
+        self.assertEqual(s3_token.split_path('/a/c', 2), ['a', 'c'])
+        self.assertEqual(s3_token.split_path('/a/c/o', 3), ['a', 'c', 'o'])
+        self.assertEqual(s3_token.split_path('/a/c/o/r', 3, 3, True),
+                         ['a', 'c', 'o/r'])
+        self.assertEqual(s3_token.split_path('/a/c', 2, 3, True),
+                         ['a', 'c', None])
+        self.assertEqual(s3_token.split_path('/a/c/', 2), ['a', 'c'])
+        self.assertEqual(s3_token.split_path('/a/c/', 2, 3), ['a', 'c', ''])
+
+    def test_split_path_invalid_path(self):
+        try:
+            s3_token.split_path('o\nn e', 2)
+        except ValueError as err:
+            self.assertEqual(str(err), 'Invalid path: o%0An%20e')
+        try:
+            s3_token.split_path('o\nn e', 2, 3, True)
+        except ValueError as err:
+            self.assertEqual(str(err), 'Invalid path: o%0An%20e')