Merge "Add copy object method"

3 files changed, 396 insertions, 2 deletions

diff --git a/swiftclient/client.py b/swiftclient/client.py
index 35c9f7a..602489d 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -1330,6 +1330,70 @@ def post_object(url, token, container, name, headers, http_conn=None,
         raise ClientException.from_response(resp, 'Object POST failed', body)
 
 
+def copy_object(url, token, container, name, destination=None,
+                headers=None, fresh_metadata=None, http_conn=None,
+                response_dict=None, service_token=None):
+    """
+    Copy object
+
+    :param url: storage URL
+    :param token: auth token; if None, no token will be sent
+    :param container: container name that the source object is in
+    :param name: source object name
+    :param destination: The container and object name of the destination object
+                        in the form of /container/object; if None, the copy
+                        will use the source as the destination.
+    :param headers: additional headers to include in the request
+    :param fresh_metadata: Enables object creation that omits existing user
+                           metadata, default None
+    :param http_conn: HTTP connection object (If None, it will create the
+                      conn object)
+    :param response_dict: an optional dictionary into which to place
+                     the response - status, reason and headers
+    :param service_token: service auth token
+    :raises ClientException: HTTP COPY request failed
+    """
+    if http_conn:
+        parsed, conn = http_conn
+    else:
+        parsed, conn = http_connection(url)
+
+    path = parsed.path
+    container = quote(container)
+    name = quote(name)
+    path = '%s/%s/%s' % (path.rstrip('/'), container, name)
+
+    headers = dict(headers) if headers else {}
+
+    if destination is not None:
+        headers['Destination'] = quote(destination)
+    elif container and name:
+        headers['Destination'] = '/%s/%s' % (container, name)
+
+    if token is not None:
+        headers['X-Auth-Token'] = token
+    if service_token is not None:
+        headers['X-Service-Token'] = service_token
+
+    if fresh_metadata is not None:
+        # remove potential fresh metadata headers
+        for fresh_hdr in [hdr for hdr in headers.keys()
+                          if hdr.lower() == 'x-fresh-metadata']:
+            headers.pop(fresh_hdr)
+        headers['X-Fresh-Metadata'] = 'true' if fresh_metadata else 'false'
+
+    conn.request('COPY', path, '', headers)
+    resp = conn.getresponse()
+    body = resp.read()
+    http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'COPY',),
+             {'headers': headers}, resp, body)
+
+    store_response(resp, response_dict)
+
+    if resp.status < 200 or resp.status >= 300:
+        raise ClientException.from_response(resp, 'Object COPY failed', body)
+
+
 def delete_object(url, token=None, container=None, name=None, http_conn=None,
                   headers=None, proxy=None, query_string=None,
                   response_dict=None, service_token=None):
@@ -1727,6 +1791,13 @@ class Connection(object):
         return self._retry(None, post_object, container, obj, headers,
                            response_dict=response_dict)
 
+    def copy_object(self, container, obj, destination=None, headers=None,
+                    fresh_metadata=None, response_dict=None):
+        """Wrapper for :func:`copy_object`"""
+        return self._retry(None, copy_object, container, obj, destination,
+                           headers, fresh_metadata,
+                           response_dict=response_dict)
+
     def delete_object(self, container, obj, query_string=None,
                       response_dict=None):
         """Wrapper for :func:`delete_object`"""
diff --git a/swiftclient/service.py b/swiftclient/service.py
index 6ccba55..af412d1 100644
--- a/swiftclient/service.py
+++ b/swiftclient/service.py
@@ -200,7 +200,9 @@ _default_local_options = {
     'human': False,
     'dir_marker': False,
     'checksum': True,
-    'shuffle': False
+    'shuffle': False,
+    'destination': None,
+    'fresh_metadata': False,
 }
 
 POLICY = 'X-Storage-Policy'
@@ -330,6 +332,42 @@ class SwiftPostObject(object):
             self.options = options
 
 
+class SwiftCopyObject(object):
+    """
+    Class for specifying an object copy,
+    allowing the destination/headers/metadata/fresh_metadata to be specified
+    separately for each individual object.
+    destination and fresh_metadata should be set in options
+    """
+    def __init__(self, object_name, options=None):
+        if not isinstance(object_name, string_types) or not object_name:
+            raise SwiftError(
+                "Object names must be specified as non-empty strings"
+            )
+
+        self.object_name = object_name
+        self.options = options
+
+        if self.options is None:
+            self.destination = None
+            self.fresh_metadata = False
+        else:
+            self.destination = self.options.get('destination')
+            self.fresh_metadata = self.options.get('fresh_metadata', False)
+
+        if self.destination is not None:
+            destination_components = self.destination.split('/')
+            if destination_components[0] or len(destination_components) < 2:
+                raise SwiftError("destination must be in format /cont[/obj]")
+            if not destination_components[-1]:
+                raise SwiftError("destination must not end in a slash")
+            if len(destination_components) == 2:
+                # only container set in destination
+                self.destination = "{0}/{1}".format(
+                    self.destination, object_name
+                )
+
+
 class _SwiftReader(object):
     """
     Class for downloading objects from swift and raising appropriate
@@ -2391,6 +2429,191 @@ class SwiftService(object):
 
         return res
 
+    # Copy related methods
+    #
+    def copy(self, container, objects, options=None):
+        """
+        Copy operations on a list of objects in a container. Destination
+        containers will be created.
+
+        :param container: The container from which to copy the objects.
+        :param objects: A list of object names (strings) or SwiftCopyObject
+                        instances containing an object name and an
+                        options dict (can be None) to override the options for
+                        that individual copy operation::
+
+                            [
+                                'object_name',
+                                SwiftCopyObject(
+                                    'object_name',
+                                     options={
+                                        'destination': '/container/object',
+                                        'fresh_metadata': False,
+                                        ...
+                                        }),
+                                ...
+                            ]
+
+                        The options dict is described below.
+        :param options: A dictionary containing options to override the global
+                        options specified during the service object creation.
+                        These options are applied to all copy operations
+                        performed by this call, unless overridden on a per
+                        object basis.
+                        The options "destination" and "fresh_metadata" do
+                        not need to be set, in this case objects will be
+                        copied onto themselves and metadata will not be
+                        refreshed.
+                        The option "destination" can also be specified in the
+                        format '/container', in which case objects without an
+                        explicit destination will be copied to the destination
+                        /container/original_object_name. Combinations of
+                        multiple objects and a destination in the format
+                        '/container/object' is invalid. Possible options are
+                        given below::
+
+                            {
+                                'meta': [],
+                                'header': [],
+                                'destination': '/container/object',
+                                'fresh_metadata': False,
+                            }
+
+        :returns: A generator returning the results of copying the given list
+                  of objects.
+
+        :raises: SwiftError
+        """
+        if options is not None:
+            options = dict(self._options, **options)
+        else:
+            options = self._options
+
+        # Try to create the container, just in case it doesn't exist. If this
+        # fails, it might just be because the user doesn't have container PUT
+        # permissions, so we'll ignore any error. If there's really a problem,
+        # it'll surface on the first object COPY.
+        containers = set(
+            next(p for p in obj.destination.split("/") if p)
+            for obj in objects
+            if isinstance(obj, SwiftCopyObject) and obj.destination
+        )
+        if options.get('destination'):
+            destination_split = options['destination'].split('/')
+            if destination_split[0]:
+                raise SwiftError("destination must be in format /cont[/obj]")
+            _str_objs = [
+                o for o in objects if not isinstance(o, SwiftCopyObject)
+            ]
+            if len(destination_split) > 2 and len(_str_objs) > 1:
+                # TODO (clayg): could be useful to copy multiple objects into
+                # a destination like "/container/common/prefix/for/objects/"
+                # where the trailing "/" indicates the destination option is a
+                # prefix!
+                raise SwiftError("Combination of multiple objects and "
+                                 "destination including object is invalid")
+            if destination_split[-1] == '':
+                # N.B. this protects the above case
+                raise SwiftError("destination can not end in a slash")
+            containers.add(destination_split[1])
+
+        policy_header = {}
+        _header = split_headers(options["header"])
+        if POLICY in _header:
+            policy_header[POLICY] = _header[POLICY]
+        create_containers = [
+            self.thread_manager.container_pool.submit(
+                self._create_container_job, cont, headers=policy_header)
+            for cont in containers
+        ]
+
+        # wait for container creation jobs to complete before any COPY
+        for r in interruptable_as_completed(create_containers):
+            res = r.result()
+            yield res
+
+        copy_futures = []
+        copy_objects = self._make_copy_objects(objects, options)
+        for copy_object in copy_objects:
+            obj = copy_object.object_name
+            obj_options = copy_object.options
+            destination = copy_object.destination
+            fresh_metadata = copy_object.fresh_metadata
+            headers = split_headers(
+                options['meta'], 'X-Object-Meta-')
+            # add header options to the headers object for the request.
+            headers.update(
+                split_headers(options['header'], ''))
+            if obj_options is not None:
+                if 'meta' in obj_options:
+                    headers.update(
+                        split_headers(
+                            obj_options['meta'], 'X-Object-Meta-'
+                        )
+                    )
+                if 'header' in obj_options:
+                    headers.update(
+                        split_headers(obj_options['header'], '')
+                    )
+
+            copy = self.thread_manager.object_uu_pool.submit(
+                self._copy_object_job, container, obj, destination,
+                headers, fresh_metadata
+            )
+            copy_futures.append(copy)
+
+        for r in interruptable_as_completed(copy_futures):
+            res = r.result()
+            yield res
+
+    @staticmethod
+    def _make_copy_objects(objects, options):
+        copy_objects = []
+
+        for o in objects:
+            if isinstance(o, string_types):
+                obj = SwiftCopyObject(o, options)
+                copy_objects.append(obj)
+            elif isinstance(o, SwiftCopyObject):
+                copy_objects.append(o)
+            else:
+                raise SwiftError(
+                    "The copy operation takes only strings or "
+                    "SwiftCopyObjects as input",
+                    obj=o)
+
+        return copy_objects
+
+    @staticmethod
+    def _copy_object_job(conn, container, obj, destination, headers,
+                         fresh_metadata):
+        response_dict = {}
+        res = {
+            'success': True,
+            'action': 'copy_object',
+            'container': container,
+            'object': obj,
+            'destination': destination,
+            'headers': headers,
+            'fresh_metadata': fresh_metadata,
+            'response_dict': response_dict
+        }
+        try:
+            conn.copy_object(
+                container, obj, destination=destination, headers=headers,
+                fresh_metadata=fresh_metadata, response_dict=response_dict)
+        except Exception as err:
+            traceback, err_time = report_traceback()
+            logger.exception(err)
+            res.update({
+                'success': False,
+                'error': err,
+                'traceback': traceback,
+                'error_timestamp': err_time
+            })
+
+        return res
+
     # Capabilities related methods
     #
     def capabilities(self, url=None, refresh_cache=False):
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
index cc34b7f..be1888d 100755
--- a/swiftclient/shell.py
+++ b/swiftclient/shell.py
@@ -47,7 +47,7 @@ except ImportError:
     from pipes import quote as sh_quote
 
 BASENAME = 'swift'
-commands = ('delete', 'download', 'list', 'post', 'stat', 'upload',
+commands = ('delete', 'download', 'list', 'post', 'copy', 'stat', 'upload',
             'capabilities', 'info', 'tempurl', 'auth')
 
 
@@ -751,6 +751,105 @@ def st_post(parser, args, output_manager):
             output_manager.error(e.value)
 
 
+st_copy_options = '''[--destination </container/object>] [--fresh-metadata]
+                  [--meta <name:value>] [--header <header>] container object
+'''
+
+st_copy_help = '''
+Copies object to new destination, optionally updates objects metadata.
+If destination is not set, will update metadata of object
+
+Positional arguments:
+  container             Name of container to copy from.
+  object                Name of object to copy. Specify multiple times
+                        for multiple objects
+
+Optional arguments:
+  -d, --destination </container[/object]>
+                        The container and name of the destination object. Name
+                        of destination object can be ommited, then will be
+                        same as name of source object. Supplying multiple
+                        objects and destination with object name is invalid.
+  -M, --fresh-metadata  Copy the object without any existing metadata,
+                        If not set, metadata will be preserved or appended
+  -m, --meta <name:value>
+                        Sets a meta data item. This option may be repeated.
+                        Example: -m Color:Blue -m Size:Large
+  -H, --header <header:value>
+                        Adds a customized request header.
+                        This option may be repeated. Example
+                        -H "content-type:text/plain" -H "Content-Length: 4000"
+'''.strip('\n')
+
+
+def st_copy(parser, args, output_manager):
+    parser.add_argument(
+        '-d', '--destination', help='The container and name of the '
+        'destination object')
+    parser.add_argument(
+        '-M', '--fresh-metadata', action='store_true',
+        help='Copy the object without any existing metadata', default=False)
+    parser.add_argument(
+        '-m', '--meta', action='append', dest='meta', default=[],
+        help='Sets a meta data item. This option may be repeated. '
+        'Example: -m Color:Blue -m Size:Large')
+    parser.add_argument(
+        '-H', '--header', action='append', dest='header',
+        default=[], help='Adds a customized request header. '
+        'This option may be repeated. '
+        'Example: -H "content-type:text/plain" '
+        '-H "Content-Length: 4000"')
+    (options, args) = parse_args(parser, args)
+    args = args[1:]
+
+    with SwiftService(options=options) as swift:
+        try:
+            if len(args) >= 2:
+                container = args[0]
+                if '/' in container:
+                    output_manager.error(
+                        'WARNING: / in container name; you might have '
+                        "meant '%s' instead of '%s'." %
+                        (args[0].replace('/', ' ', 1), args[0]))
+                    return
+                objects = [arg for arg in args[1:]]
+
+                for r in swift.copy(
+                        container=container, objects=objects,
+                        options=options):
+                    if r['success']:
+                        if options['verbose']:
+                            if r['action'] == 'copy_object':
+                                output_manager.print_msg(
+                                    '%s/%s copied to %s' % (
+                                        r['container'],
+                                        r['object'],
+                                        r['destination'] or '<self>'))
+                            if r['action'] == 'create_container':
+                                output_manager.print_msg(
+                                    'created container %s' % r['container']
+                                )
+                    else:
+                        error = r['error']
+                        if 'action' in r and r['action'] == 'create_container':
+                            # it is not an error to be unable to create the
+                            # container so print a warning and carry on
+                            output_manager.warning(
+                                'Warning: failed to create container '
+                                "'%s': %s", container, error
+                            )
+                        else:
+                            output_manager.error("%s" % error)
+            else:
+                output_manager.error(
+                    'Usage: %s copy %s\n%s', BASENAME,
+                    st_copy_options, st_copy_help)
+                return
+
+        except SwiftError as e:
+            output_manager.error(e.value)
+
+
 st_upload_options = '''[--changed] [--skip-identical] [--segment-size <size>]
                     [--segment-container <container>] [--leave-segments]
                     [--object-threads <thread>] [--segment-threads <threads>]
@@ -1283,6 +1382,7 @@ Positional arguments:
                          for a container.
     post                 Updates meta information for the account, container,
                          or object; creates containers if not present.
+    copy                 Copies object, optionally adds meta
     stat                 Displays information for the account, container,
                          or object.
     upload               Uploads files or directories to the given container.