Expand support for command extensions

Allows client libraries to have complete access to the rest of the
OSC ClientManager.  In addition, extension libraries can define
global options (for API version options/env vars) and define
versioned API entry points similar to the in-repo commands.

The changes to ClientManager exposed some issues in the existing
object api tests that needed to be cleaned up.

Change-Id: Ic9662edf34c5dd130a2f1a69d2454adefc1f8a95

13 files changed, 165 insertions, 118 deletions

diff --git a/openstackclient/common/clientmanager.py b/openstackclient/common/clientmanager.py
index 85f544e4..a0224064 100644
--- a/openstackclient/common/clientmanager.py
+++ b/openstackclient/common/clientmanager.py
@@ -16,12 +16,10 @@
 """Manage access to the clients, including authenticating when needed."""
 
 import logging
+import pkg_resources
+import sys
 
-from openstackclient.compute import client as compute_client
 from openstackclient.identity import client as identity_client
-from openstackclient.image import client as image_client
-from openstackclient.object import client as object_client
-from openstackclient.volume import client as volume_client
 
 
 LOG = logging.getLogger(__name__)
@@ -42,11 +40,7 @@ class ClientCache(object):
 
 class ClientManager(object):
     """Manages access to API clients, including authentication."""
-    compute = ClientCache(compute_client.make_client)
     identity = ClientCache(identity_client.make_client)
-    image = ClientCache(image_client.make_client)
-    object = ClientCache(object_client.make_client)
-    volume = ClientCache(volume_client.make_client)
 
     def __init__(self, token=None, url=None, auth_url=None, project_name=None,
                  project_id=None, username=None, password=None,
@@ -93,3 +87,26 @@ class ClientManager(object):
             # Hope we were given the correct URL.
             endpoint = self._url
         return endpoint
+
+
+def get_extension_modules(group):
+    """Add extension clients"""
+    mod_list = []
+    for ep in pkg_resources.iter_entry_points(group):
+        LOG.debug('found extension %r' % ep.name)
+
+        __import__(ep.module_name)
+        module = sys.modules[ep.module_name]
+        mod_list.append(module)
+        init_func = getattr(module, 'Initialize', None)
+        if init_func:
+            init_func('x')
+
+        setattr(
+            ClientManager,
+            ep.name,
+            ClientCache(
+                getattr(sys.modules[ep.module_name], 'make_client', None)
+            ),
+        )
+    return mod_list
diff --git a/openstackclient/compute/client.py b/openstackclient/compute/client.py
index 4d3b1b71..4ccb2f6d 100644
--- a/openstackclient/compute/client.py
+++ b/openstackclient/compute/client.py
@@ -19,6 +19,8 @@ from openstackclient.common import utils
 
 LOG = logging.getLogger(__name__)
 
+DEFAULT_COMPUTE_API_VERSION = '2'
+API_VERSION_OPTION = 'os_compute_api_version'
 API_NAME = 'compute'
 API_VERSIONS = {
     '1.1': 'novaclient.v1_1.client.Client',
@@ -60,3 +62,17 @@ def make_client(instance):
         client.client.service_catalog = instance._service_catalog
     client.client.auth_token = instance._token
     return client
+
+
+def build_option_parser(parser):
+    """Hook to add global options"""
+    parser.add_argument(
+        '--os-compute-api-version',
+        metavar='<compute-api-version>',
+        default=utils.env(
+            'OS_COMPUTE_API_VERSION',
+            default=DEFAULT_COMPUTE_API_VERSION),
+        help='Compute API version, default=' +
+             DEFAULT_COMPUTE_API_VERSION +
+             ' (Env: OS_COMPUTE_API_VERSION)')
+    return parser
diff --git a/openstackclient/identity/client.py b/openstackclient/identity/client.py
index 4814bc3e..305d4cc4 100644
--- a/openstackclient/identity/client.py
+++ b/openstackclient/identity/client.py
@@ -21,6 +21,8 @@ from openstackclient.common import utils
 
 LOG = logging.getLogger(__name__)
 
+DEFAULT_IDENTITY_API_VERSION = '2.0'
+API_VERSION_OPTION = 'os_identity_api_version'
 API_NAME = 'identity'
 API_VERSIONS = {
     '2.0': 'openstackclient.identity.client.IdentityClientv2_0',
diff --git a/openstackclient/image/client.py b/openstackclient/image/client.py
index d56ca3b2..9edffded 100644
--- a/openstackclient/image/client.py
+++ b/openstackclient/image/client.py
@@ -23,6 +23,8 @@ from openstackclient.common import utils
 
 LOG = logging.getLogger(__name__)
 
+DEFAULT_IMAGE_API_VERSION = '1'
+API_VERSION_OPTION = 'os_image_api_version'
 API_NAME = "image"
 API_VERSIONS = {
     "1": "openstackclient.image.client.Client_v1",
@@ -48,6 +50,20 @@ def make_client(instance):
     )
 
 
+def build_option_parser(parser):
+    """Hook to add global options"""
+    parser.add_argument(
+        '--os-image-api-version',
+        metavar='<image-api-version>',
+        default=utils.env(
+            'OS_IMAGE_API_VERSION',
+            default=DEFAULT_IMAGE_API_VERSION),
+        help='Image API version, default=' +
+             DEFAULT_IMAGE_API_VERSION +
+             ' (Env: OS_IMAGE_API_VERSION)')
+    return parser
+
+
 # NOTE(dtroyer): glanceclient.v1.image.ImageManager() doesn't have a find()
 #                method so add one here until the common client libs arrive
 #                A similar subclass will be required for v2
diff --git a/openstackclient/object/client.py b/openstackclient/object/client.py
index a83a5c0a..1a5363b1 100644
--- a/openstackclient/object/client.py
+++ b/openstackclient/object/client.py
@@ -21,7 +21,9 @@ from openstackclient.common import utils
 
 LOG = logging.getLogger(__name__)
 
-API_NAME = 'object-store'
+DEFAULT_OBJECT_API_VERSION = '1'
+API_VERSION_OPTION = 'os_object_api_version'
+API_NAME = 'object'
 API_VERSIONS = {
     '1': 'openstackclient.object.client.ObjectClientv1',
 }
@@ -45,6 +47,20 @@ def make_client(instance):
     return client
 
 
+def build_option_parser(parser):
+    """Hook to add global options"""
+    parser.add_argument(
+        '--os-object-api-version',
+        metavar='<object-api-version>',
+        default=utils.env(
+            'OS_OBJECT_API_VERSION',
+            default=DEFAULT_OBJECT_API_VERSION),
+        help='Object API version, default=' +
+             DEFAULT_OBJECT_API_VERSION +
+             ' (Env: OS_OBJECT_API_VERSION)')
+    return parser
+
+
 class ObjectClientv1(object):
 
     def __init__(
diff --git a/openstackclient/shell.py b/openstackclient/shell.py
index 78e16cd6..f8a47ca6 100644
--- a/openstackclient/shell.py
+++ b/openstackclient/shell.py
@@ -33,15 +33,11 @@ from openstackclient.common import exceptions as exc
 from openstackclient.common import openstackkeyring
 from openstackclient.common import restapi
 from openstackclient.common import utils
+from openstackclient.identity import client as identity_client
 
 
 KEYRING_SERVICE = 'openstack'
 
-DEFAULT_COMPUTE_API_VERSION = '2'
-DEFAULT_IDENTITY_API_VERSION = '2.0'
-DEFAULT_IMAGE_API_VERSION = '1'
-DEFAULT_OBJECT_API_VERSION = '1'
-DEFAULT_VOLUME_API_VERSION = '1'
 DEFAULT_DOMAIN = 'default'
 
 
@@ -86,6 +82,15 @@ class OpenStackShell(app.App):
         # Assume TLS host certificate verification is enabled
         self.verify = True
 
+        # Get list of extension modules
+        self.ext_modules = clientmanager.get_extension_modules(
+            'openstack.cli.extension',
+        )
+
+        # Loop through extensions to get parser additions
+        for mod in self.ext_modules:
+            self.parser = mod.build_option_parser(self.parser)
+
         # NOTE(dtroyer): This hack changes the help action that Cliff
         #                automatically adds to the parser so we can defer
         #                its execution until after the api-versioned commands
@@ -203,51 +208,6 @@ class OpenStackShell(app.App):
                  DEFAULT_DOMAIN +
                  ' (Env: OS_DEFAULT_DOMAIN)')
         parser.add_argument(
-            '--os-identity-api-version',
-            metavar='<identity-api-version>',
-            default=env(
-                'OS_IDENTITY_API_VERSION',
-                default=DEFAULT_IDENTITY_API_VERSION),
-            help='Identity API version, default=' +
-                 DEFAULT_IDENTITY_API_VERSION +
-                 ' (Env: OS_IDENTITY_API_VERSION)')
-        parser.add_argument(
-            '--os-compute-api-version',
-            metavar='<compute-api-version>',
-            default=env(
-                'OS_COMPUTE_API_VERSION',
-                default=DEFAULT_COMPUTE_API_VERSION),
-            help='Compute API version, default=' +
-                 DEFAULT_COMPUTE_API_VERSION +
-                 ' (Env: OS_COMPUTE_API_VERSION)')
-        parser.add_argument(
-            '--os-image-api-version',
-            metavar='<image-api-version>',
-            default=env(
-                'OS_IMAGE_API_VERSION',
-                default=DEFAULT_IMAGE_API_VERSION),
-            help='Image API version, default=' +
-                 DEFAULT_IMAGE_API_VERSION +
-                 ' (Env: OS_IMAGE_API_VERSION)')
-        parser.add_argument(
-            '--os-object-api-version',
-            metavar='<object-api-version>',
-            default=env(
-                'OS_OBJECT_API_VERSION',
-                default=DEFAULT_OBJECT_API_VERSION),
-            help='Object API version, default=' +
-                 DEFAULT_OBJECT_API_VERSION +
-                 ' (Env: OS_OBJECT_API_VERSION)')
-        parser.add_argument(
-            '--os-volume-api-version',
-            metavar='<volume-api-version>',
-            default=env(
-                'OS_VOLUME_API_VERSION',
-                default=DEFAULT_VOLUME_API_VERSION),
-            help='Volume API version, default=' +
-                 DEFAULT_VOLUME_API_VERSION +
-                 ' (Env: OS_VOLUME_API_VERSION)')
-        parser.add_argument(
             '--os-token',
             metavar='<token>',
             default=env('OS_TOKEN'),
@@ -270,6 +230,16 @@ class OpenStackShell(app.App):
                             help='Use keyring to store password, '
                                  'default=False (Env: OS_USE_KEYRING)')
 
+        parser.add_argument(
+            '--os-identity-api-version',
+            metavar='<identity-api-version>',
+            default=env(
+                'OS_IDENTITY_API_VERSION',
+                default=identity_client.DEFAULT_IDENTITY_API_VERSION),
+            help='Identity API version, default=' +
+                 identity_client.DEFAULT_IDENTITY_API_VERSION +
+                 ' (Env: OS_IDENTITY_API_VERSION)')
+
         return parser
 
     def authenticate_user(self):
@@ -391,17 +361,20 @@ class OpenStackShell(app.App):
 
         # Stash selected API versions for later
         self.api_version = {
-            'compute': self.options.os_compute_api_version,
             'identity': self.options.os_identity_api_version,
-            'image': self.options.os_image_api_version,
-            'object-store': self.options.os_object_api_version,
-            'volume': self.options.os_volume_api_version,
         }
+        # Loop through extensions to get API versions
+        for mod in self.ext_modules:
+            ver = getattr(self.options, mod.API_VERSION_OPTION, None)
+            if ver:
+                self.api_version[mod.API_NAME] = ver
+                self.log.debug('%s API version %s' % (mod.API_NAME, ver))
 
         # Add the API version-specific commands
         for api in self.api_version.keys():
             version = '.v' + self.api_version[api].replace('.', '_')
             cmd_group = 'openstack.' + api.replace('-', '_') + version
+            self.log.debug('command group %s' % cmd_group)
             self.command_manager.add_command_group(cmd_group)
 
         # Commands that span multiple APIs
@@ -420,6 +393,8 @@ class OpenStackShell(app.App):
         # }
         self.command_manager.add_command_group(
             'openstack.extension')
+        # call InitializeXxx() here
+        # set up additional clients to stuff in to client_manager??
 
         # Handle deferred help and exit
         if self.options.deferred_help:
diff --git a/openstackclient/tests/fakes.py b/openstackclient/tests/fakes.py
index 8ab45465..bb89f762 100644
--- a/openstackclient/tests/fakes.py
+++ b/openstackclient/tests/fakes.py
@@ -49,6 +49,7 @@ class FakeClientManager(object):
         self.compute = None
         self.identity = None
         self.image = None
+        self.object = None
         self.volume = None
         self.auth_ref = None
 
diff --git a/openstackclient/tests/object/v1/fakes.py b/openstackclient/tests/object/v1/fakes.py
index fbc784aa..37c35d3b 100644
--- a/openstackclient/tests/object/v1/fakes.py
+++ b/openstackclient/tests/object/v1/fakes.py
@@ -13,6 +13,10 @@
 #   under the License.
 #
 
+from openstackclient.tests import fakes
+from openstackclient.tests import utils
+
+
 container_name = 'bit-bucket'
 container_bytes = 1024
 container_count = 1
@@ -65,3 +69,19 @@ OBJECT_2 = {
     'content_type': object_content_type_2,
     'last_modified': object_modified_2,
 }
+
+
+class FakeObjectv1Client(object):
+    def __init__(self, **kwargs):
+        self.endpoint = kwargs['endpoint']
+        self.token = kwargs['token']
+
+
+class TestObjectv1(utils.TestCommand):
+    def setUp(self):
+        super(TestObjectv1, self).setUp()
+
+        self.app.client_manager.object = FakeObjectv1Client(
+            endpoint=fakes.AUTH_URL,
+            token=fakes.AUTH_TOKEN,
+        )
diff --git a/openstackclient/tests/object/v1/lib/test_container.py b/openstackclient/tests/object/v1/lib/test_container.py
index 3b9976a1..c3fdea72 100644
--- a/openstackclient/tests/object/v1/lib/test_container.py
+++ b/openstackclient/tests/object/v1/lib/test_container.py
@@ -19,8 +19,7 @@ import mock
 
 from openstackclient.object.v1.lib import container as lib_container
 from openstackclient.tests.common import test_restapi as restapi
-from openstackclient.tests import fakes
-from openstackclient.tests import utils
+from openstackclient.tests.object.v1 import fakes as object_fakes
 
 
 fake_account = 'q12we34r'
@@ -36,12 +35,10 @@ class FakeClient(object):
         self.token = fake_auth
 
 
-class TestContainer(utils.TestCommand):
+class TestContainer(object_fakes.TestObjectv1):
 
     def setUp(self):
         super(TestContainer, self).setUp()
-        self.app.client_manager = fakes.FakeClientManager()
-        self.app.client_manager.object = FakeClient()
         self.app.restapi = mock.MagicMock()
 
 
@@ -53,7 +50,7 @@ class TestContainerList(TestContainer):
 
         data = lib_container.list_containers(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
         )
 
         # Check expected values
@@ -69,7 +66,7 @@ class TestContainerList(TestContainer):
 
         data = lib_container.list_containers(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             marker='next',
         )
 
@@ -86,7 +83,7 @@ class TestContainerList(TestContainer):
 
         data = lib_container.list_containers(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             limit=5,
         )
 
@@ -103,7 +100,7 @@ class TestContainerList(TestContainer):
 
         data = lib_container.list_containers(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             end_marker='last',
         )
 
@@ -120,7 +117,7 @@ class TestContainerList(TestContainer):
 
         data = lib_container.list_containers(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             prefix='foo/',
         )
 
@@ -147,7 +144,7 @@ class TestContainerList(TestContainer):
 
         data = lib_container.list_containers(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             full_listing=True,
         )
 
@@ -171,7 +168,7 @@ class TestContainerShow(TestContainer):
 
         data = lib_container.show_container(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             'is-name',
         )
 
diff --git a/openstackclient/tests/object/v1/lib/test_object.py b/openstackclient/tests/object/v1/lib/test_object.py
index 0104183e..ef93877a 100644
--- a/openstackclient/tests/object/v1/lib/test_object.py
+++ b/openstackclient/tests/object/v1/lib/test_object.py
@@ -19,8 +19,7 @@ import mock
 
 from openstackclient.object.v1.lib import object as lib_object
 from openstackclient.tests.common import test_restapi as restapi
-from openstackclient.tests import fakes
-from openstackclient.tests import utils
+from openstackclient.tests.object.v1 import fakes as object_fakes
 
 
 fake_account = 'q12we34r'
@@ -37,12 +36,10 @@ class FakeClient(object):
         self.token = fake_auth
 
 
-class TestObject(utils.TestCommand):
+class TestObject(object_fakes.TestObjectv1):
 
     def setUp(self):
         super(TestObject, self).setUp()
-        self.app.client_manager = fakes.FakeClientManager()
-        self.app.client_manager.object = FakeClient()
         self.app.restapi = mock.MagicMock()
 
 
@@ -54,7 +51,7 @@ class TestObjectListObjects(TestObject):
 
         data = lib_object.list_objects(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             fake_container,
         )
 
@@ -71,7 +68,7 @@ class TestObjectListObjects(TestObject):
 
         data = lib_object.list_objects(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             fake_container,
             marker='next',
         )
@@ -89,7 +86,7 @@ class TestObjectListObjects(TestObject):
 
         data = lib_object.list_objects(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             fake_container,
             limit=5,
         )
@@ -107,7 +104,7 @@ class TestObjectListObjects(TestObject):
 
         data = lib_object.list_objects(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             fake_container,
             end_marker='last',
         )
@@ -125,7 +122,7 @@ class TestObjectListObjects(TestObject):
 
         data = lib_object.list_objects(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             fake_container,
             delimiter='|',
         )
@@ -146,7 +143,7 @@ class TestObjectListObjects(TestObject):
 
         data = lib_object.list_objects(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             fake_container,
             prefix='foo/',
         )
@@ -164,7 +161,7 @@ class TestObjectListObjects(TestObject):
 
         data = lib_object.list_objects(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             fake_container,
             path='next',
         )
@@ -192,7 +189,7 @@ class TestObjectListObjects(TestObject):
 
         data = lib_object.list_objects(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             fake_container,
             full_listing=True,
         )
@@ -216,7 +213,7 @@ class TestObjectShowObjects(TestObject):
 
         data = lib_object.show_object(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             fake_container,
             fake_object,
         )
@@ -250,7 +247,7 @@ class TestObjectShowObjects(TestObject):
 
         data = lib_object.show_object(
             self.app.restapi,
-            self.app.client_manager.object.endpoint,
+            fake_url,
             fake_container,
             fake_object,
         )
diff --git a/openstackclient/tests/object/v1/test_container.py b/openstackclient/tests/object/v1/test_container.py
index 2090b880..e4d90fd9 100644
--- a/openstackclient/tests/object/v1/test_container.py
+++ b/openstackclient/tests/object/v1/test_container.py
@@ -16,10 +16,8 @@
 import copy
 import mock
 
-from openstackclient.common import clientmanager
 from openstackclient.object.v1 import container
 from openstackclient.tests.object.v1 import fakes as object_fakes
-from openstackclient.tests import utils
 
 
 AUTH_TOKEN = "foobar"
@@ -32,18 +30,10 @@ class FakeClient(object):
         self.token = AUTH_TOKEN
 
 
-class TestObject(utils.TestCommand):
+class TestObject(object_fakes.TestObjectv1):
     def setUp(self):
         super(TestObject, self).setUp()
 
-        api_version = {"object-store": "1"}
-        self.app.client_manager = clientmanager.ClientManager(
-            token=AUTH_TOKEN,
-            url=AUTH_URL,
-            auth_url=AUTH_URL,
-            api_version=api_version,
-        )
-
 
 class TestObjectClient(TestObject):
 
diff --git a/openstackclient/tests/object/v1/test_object.py b/openstackclient/tests/object/v1/test_object.py
index 52b5ebd0..9987c2d4 100644
--- a/openstackclient/tests/object/v1/test_object.py
+++ b/openstackclient/tests/object/v1/test_object.py
@@ -16,34 +16,18 @@
 import copy
 import mock
 
-from openstackclient.common import clientmanager
 from openstackclient.object.v1 import object as obj
 from openstackclient.tests.object.v1 import fakes as object_fakes
-from openstackclient.tests import utils
 
 
 AUTH_TOKEN = "foobar"
 AUTH_URL = "http://0.0.0.0"
 
 
-class FakeClient(object):
-    def __init__(self, endpoint=None, **kwargs):
-        self.endpoint = AUTH_URL
-        self.token = AUTH_TOKEN
-
-
-class TestObject(utils.TestCommand):
+class TestObject(object_fakes.TestObjectv1):
     def setUp(self):
         super(TestObject, self).setUp()
 
-        api_version = {"object-store": "1"}
-        self.app.client_manager = clientmanager.ClientManager(
-            token=AUTH_TOKEN,
-            url=AUTH_URL,
-            auth_url=AUTH_URL,
-            api_version=api_version,
-        )
-
 
 class TestObjectClient(TestObject):
 
diff --git a/openstackclient/volume/client.py b/openstackclient/volume/client.py
index 626b23f1..e04e8cd7 100644
--- a/openstackclient/volume/client.py
+++ b/openstackclient/volume/client.py
@@ -20,6 +20,8 @@ from openstackclient.common import utils
 
 LOG = logging.getLogger(__name__)
 
+DEFAULT_VOLUME_API_VERSION = '1'
+API_VERSION_OPTION = 'os_volume_api_version'
 API_NAME = "volume"
 API_VERSIONS = {
     "1": "cinderclient.v1.client.Client"
@@ -45,3 +47,17 @@ def make_client(instance):
     )
 
     return client
+
+
+def build_option_parser(parser):
+    """Hook to add global options"""
+    parser.add_argument(
+        '--os-volume-api-version',
+        metavar='<volume-api-version>',
+        default=utils.env(
+            'OS_VOLUME_API_VERSION',
+            default=DEFAULT_VOLUME_API_VERSION),
+        help='Volume API version, default=' +
+             DEFAULT_VOLUME_API_VERSION +
+             ' (Env: OS_VOLUME_API_VERSION)')
+    return parser