subversion-1.9.7HEADsubversion-1.9.7master

1 files changed, 1065 insertions, 0 deletions

diff --git a/tools/client-side/svn-vendor.py b/tools/client-side/svn-vendor.py
new file mode 100755
index 0000000..d0c862c
--- /dev/null
+++ b/tools/client-side/svn-vendor.py
@@ -0,0 +1,1065 @@
+#!/usr/bin/python3
+# vim: set sw=4 expandtab :
+# ====================================================================
+#    Licensed to the Apache Software Foundation (ASF) under one
+#    or more contributor license agreements.  See the NOTICE file
+#    distributed with this work for additional information
+#    regarding copyright ownership.  The ASF licenses this file
+#    to you under the Apache License, Version 2.0 (the
+#    "License"); you may not use this file except in compliance
+#    with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing,
+#    software distributed under the License is distributed on an
+#    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#    KIND, either express or implied.  See the License for the
+#    specific language governing permissions and limitations
+#    under the License.
+# ====================================================================
+#
+##############################################################################
+# svn-vendor.py
+#
+# Overview
+# --------
+#   Replacement for svn_load_dirs.pl (included as a 'contributed utility' in
+#   Subversion sources). Main difference is some heuristics in detection of
+#   the renames. Note that this script does not attempt to automate remote
+#   SVN operations (check-out, check-in and tagging), so it is possible to
+#   review the state of sources that are about to be checked in. Another
+#   difference is an ability to save the detected renames, review/re-apply
+#   them.
+#
+#   This script requires Python 3.3.x or higher. Sorry, I was too lazy
+#   to write shell quoting routines that are already available in recent
+#   Python versions.
+#
+# Using this script
+# -----------------
+#   First, it is necessary to check out the working copy from the URL that
+#   will host the imported sources. E.g., if the versions of FOO are being
+#   imported into svn://example.com/vendor/FOO/current:
+#
+#     svn co svn://example.com/vendor/FOO/current wc
+#
+#   Then, unpack the sources of the version to be imported:
+#
+#     tar xzf foo-1.1.tar.gz
+#
+#   Examples below assume the command above created a `foo-1.1' directory.
+#   After that, there are three different modes of operation:
+#
+#   1. Fully automatic
+#
+#     svn-vendor.py --auto wc foo-1.1
+#     svn st wc
+#     svn ci wc
+#
+#   In this mode, the script fully relies on its heuristics in detection of
+#   renames. In many cases, it "just works". There can be spurious moves
+#   detected in this mode, though. For example, consider a deleted header
+#   that consists of 50 lines of GPL text,  1 line of copyright, and
+#   3 lines of declarations, and a similar unrelated header in the imported
+#   sources. From the script's point of view, the files are nearly identical
+#   (4 lines removed, 4 lines added, 50 lines unchanged).
+#
+#   After the script completes, examine the working copy by doing 'svn diff'
+#   and/or 'svn status', paying particular attention to renames. If all the
+#   moves are detected correctly, check in the changes in the working copy.
+#
+#   2. Semi-automatic
+#
+#     svn-vendor.py --detect moves-foo-1.1.txt wc foo-1.1
+#     vi moves-foo-1.1.txt
+#     svn-vendor.py --apply moves-foo-1.1.txt wc foo-1.1
+#     svn ci wc
+#
+#   If the fully automatic mode mis-detected some spurious moves, or did not
+#   detect some renames you want to be performed, it is still possible to
+#   leverage what the script has detected automatically. First command above
+#   does the automatic detection, just as it does in fully automatic mode,
+#   but stops short of performing any modification of the working copy.
+#   The list of detected copies and renames is saved into a text file,
+#   `moves-foo-1.1.txt'.
+#
+#   That file can be inspected after the script finishes. Spurious moves can
+#   be deleted from the file, and new copies/renames can be added. Then the
+#   changes can be applied to the working copy.
+#
+#   3. Manual
+#
+#     svn-vendor.py wc foo-1.1
+#     (svn-vendor) detect
+#     (svn-vendor) move x.c y.c
+#     (svn-vendor) move include/1.h include/2.h
+#     (svn-vendor) copy include/3.h include/3-copy.h
+#     (svn-vendor) lsprep
+#     (svn-vendor) save /tmp/renames-to-be-applied.txt
+#     (svn-vendor) apply
+#
+#   If the automatic detection does not help, it is possible to do the renames
+#   manually (similarly to svn_load_dirs.pl). Use the 'help' command to get
+#   the list of supported commands and their description. Feel free to play
+#   around - since the script does not perform any remote SVN operation,
+#   there is no chance to commit the changes accidentally.
+#
+# Notes
+# -----
+#   I. The time for rename detection O(Fs*Fd) + O(Ds*Dd), where Fs is
+#   the number of files removed from current directory, Fd is number of files
+#   added in imported sources, and Ds/Dd is the same for directories. That is,
+#   the running time may become an issue if the numbers of added/removed files
+#   go into a few thousands (e.g. if updating Linux kernel 2.6.35 to 3.10).
+#   As a workaround, import interim releases first so that the number of
+#   renames remains sane at each step. That makes reviewing the renames
+#   performed by the script much easier.
+#
+#   Enjoy!
+#
+##############################################################################
+
+import argparse
+import cmd
+import difflib
+import filecmp
+import os
+import readline
+import shlex
+import shutil
+import subprocess
+import sys
+
+def name_similarity(n1, n2):
+    '''
+    Function to be used as a key for sorting dirs/files by name matching
+    '''
+    sm = difflib.SequenceMatcher(a=n1, b=n2)
+    return 1.0 - sm.ratio()
+
+
+def filename_sort_key(s):
+    '''
+    Function to sort filenames so that parent directory is always followed
+    by its children. Without it, [ "/a", "/a-b", "/a/b", "/a-b/c" ] would
+    not be sorted correctly.
+    '''
+    return s.replace('/', '\001')
+
+
+def descendant_or_self(path, ancestor):
+    '''
+    Check if path is somewhere in hierarchy under ancestor.
+    '''
+    return path == ancestor or path.startswith(ancestor + os.sep)
+
+def path_rebase(path, old_base, new_base):
+    '''
+    Return a path name that has the same relative path to new_base as path
+    had to old_base. Assumes path is a descendant of old_base.
+    '''
+    if path == old_base:
+        return new_base
+    return os.path.normpath(os.path.join(new_base,
+        os.path.relpath(path, old_base)))
+
+
+def for_all_parents(path, func):
+    '''
+    Invoke func for each parent path.
+    '''
+    d = os.path.dirname(path)
+    while d != "":
+        func(d)
+        d = os.path.dirname(d)
+
+class InvalidUsageException(Exception):
+    '''
+    Raised if command line arguments are invalid
+    '''
+    def __init__(self, cmd, msg):
+        Exception.__init__(self, msg)
+        self.cmd = cmd
+
+
+class NotImplementedException(Exception):
+    '''
+    Raised if some code path is not implemented
+    '''
+    pass
+
+
+# Indexes into FSO.state
+S_WC = 0
+S_IM = 1
+
+class FSO(object):
+    '''
+    File system object (file/dir either in imported dir or in WC)
+    '''
+    def __init__(self):
+        self.wc_path = None
+        self.state = [ "-", "-" ] # '-': absent, 'F': file, 'D': dir
+
+    def status(self):
+        return "[%s%s]" % (self.state[S_WC], self.state[S_IM])
+
+    def orig_reference(self, curpath):
+        if self.wc_path and self.wc_path != curpath:
+            return " (original: %s)" % shlex.quote(self.wc_path)
+        return ""
+
+
+class FSOCollection(dict):
+    '''
+    Collection of FSOs
+    '''
+    def print(self):
+        print(" / Status in working copy (-:absent, F:file, D:dir)")
+        print(" |/ Status in imported sources (-:absent, F:file, D:dir)")
+        for k in sorted(self.keys(), key=filename_sort_key):
+            e = self[k]
+            print("%s %s%s" % (e.status(), shlex.quote(k),
+                e.orig_reference(k)))
+
+    def get(self, path):
+        'Get existing FSO or create a new one'
+        if path in self:
+            return self[path]
+        e = FSO()
+        self[path] = e
+        return e
+
+    def add(self, path, where, kind):
+        'Adding entries during initial scan'
+        path = os.path.normpath(path)
+        e = self.get(path)
+        e.state[where] = kind
+        if where == S_WC:
+            e.wc_path = path
+
+    def wc_copy(self, src, dst):
+        'Handle move in a working copy'
+        keys = list(self.keys())
+        for k in keys:
+            if descendant_or_self(k, src):
+                esrc = self[k]
+                if esrc.state[S_WC] == "-":
+                    continue
+                kn = path_rebase(k, src, dst)
+                edst = self.get(kn)
+                if edst.state[S_WC] != "-":
+                    # Copying into existing destination.
+                    # Caller should've checked this.
+                    raise NotImplementedException
+                edst.wc_path = esrc.wc_path
+                edst.state[S_WC] = esrc.state[S_WC]
+
+    def wc_remove(self, path):
+        'Handle removal in a working copy'
+        keys = list(self.keys())
+        for k in keys:
+            if descendant_or_self(k, path):
+                self[k].state[S_WC] = "-"
+
+
+class ConfigOpt(object):
+    'Helper class - single option (string)'
+    def __init__(self, value, helpmsg):
+        self.value = value
+        self.helpmsg = helpmsg
+
+    def set(self, new_value):
+        self.value = new_value
+
+    def __str__(self):
+        return "<none>" if self.value is None else "`%s'" % self.value
+
+
+class ConfigOptInt(ConfigOpt):
+    'Helper class - single option (integer)'
+    def set(self, new_value):
+        try:
+            self.value = int(new_value)
+        except ValueError:
+            raise InvalidUsageException(None, "Value must be integer")
+
+    def __str__(self):
+        return "%d" % self.value
+
+
+class Config(dict):
+    '''
+    Store configuration options.
+    '''
+    def add_option(self, name, cfgopt):
+        self[name] = cfgopt
+
+    def set(self, name, value):
+        if name not in self:
+            raise InvalidUsageException(None,
+                    "Unknown config variable '%s'" % name)
+        self[name].set(value)
+
+    def get(self, name):
+        if name not in self:
+            raise NotImplementedException()
+        return self[name].value
+
+    def print(self):
+        for k in sorted(self):
+            o = self[k]
+            print("# %s" % o.helpmsg)
+            print("%-20s: %s" % (k, str(o)))
+            print()
+
+
+class SvnVndImport(cmd.Cmd):
+    '''
+    Main driving class.
+    '''
+    intro = "Welcome to SVN vendor import helper. " + \
+            "Type help or ? to list commands.\n"
+    prompt = "(svn-vendor) "
+    prepare_ops = []
+
+    def __init__(self, wcdir, importdir, svninfo):
+        cmd.Cmd.__init__(self)
+        self.wcdir = wcdir
+        self.importdir = importdir
+        self.svninfo = svninfo
+        self.config = Config()
+        self.config.add_option('save-diff-copied',
+                ConfigOpt(None, "Save 'svn diff' output on the " +
+                    "moved/copied files and directories to this " +
+                    "file as part of 'apply'"))
+        self.config.add_option('dir-similarity',
+                ConfigOptInt(600, "Similarity between dirs to assume " +
+                    "a copy/move [0..1000]"))
+        self.config.add_option('file-similarity',
+                ConfigOptInt(600, "Similarity between files to assume a " +
+                    "copy/move [0..1000]"))
+        self.config.add_option('file-min-lines',
+                ConfigOptInt(10, "Minimal number of lines in a file for " +
+                    "meaningful comparison"))
+        self.config.add_option('verbose',
+                ConfigOptInt(3, "Verbosity of the output [0..5]"))
+        try:
+            self.termwidth = os.get_terminal_size()[0]
+        except OSError:
+            # Not running in a terminal - probably redirected to file
+            self.termwidth = 150 # arbitrary number
+
+    def info(self, level, msg):
+        'Print message with specified verbosity'
+        if level <= self.config.get('verbose'):
+            print(msg, flush=True)
+
+    def scan(self):
+        self.items = FSOCollection()
+        self.info(1, "Scanning working copy directory...")
+        self.get_lists(self.wcdir, S_WC)
+        self.info(1, "Scanning imported directory...")
+        self.get_lists(self.importdir, S_IM)
+
+    def get_lists(self, top, where):
+        for d, dn, fn in os.walk(top, followlinks=True):
+            dr = os.path.relpath(d, top)
+            # If under .svn directory at the top (SVN 1.7+) or has .svn
+            # in the path (older SVN), ignore
+            if descendant_or_self(dr, '.svn') or \
+                    os.path.basename(dr) == '.svn' or \
+                    (os.sep + '.svn' + os.sep) in dr:
+                continue
+            if dr != '.':
+                self.items.add(dr, where, "D")
+            for f in fn:
+                fr = os.path.normpath(os.path.join(dr, f))
+                self.items.add(fr, where, "F")
+
+    def onecmd(self, str):
+        'Override for checking number of arguments'
+        try:
+            return cmd.Cmd.onecmd(self, str)
+        except InvalidUsageException as e:
+            if e.cmd is not None:
+                print("!!! Invalid usage of `%s' command: %s" % (e.cmd, e))
+                print()
+                self.onecmd("help " + e.cmd)
+            else:
+                print("!!! %s" % e)
+
+    def parse_args(self, line, nargs, cmd):
+        'Parse arguments for a command'
+        args = shlex.split(line)
+        if len(args) != nargs:
+            raise InvalidUsageException(cmd, "expect %d arguments" % nargs)
+        return args
+
+    def run_svn(self, args_fixed, args_split=[]):
+        'Run SVN command(s), potentially splitting long argument lists'
+        rv = True
+        pos = 0
+        atatime = 100
+        output = ""
+        while pos < len(args_split) or (pos == 0 and len(args_split) == 0):
+            svnargs = ['svn'] + args_fixed + args_split[pos : pos + atatime]
+            pos += atatime
+            self.info(5, "Running: " + " ".join(map(shlex.quote, svnargs)))
+            p = subprocess.Popen(args=svnargs, stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE, cwd=self.wcdir)
+            so, se = p.communicate()
+            if p.returncode != 0:
+                print("`%s' exited with %d status:" %
+                        (" ".join(map(shlex.quote, svnargs)), p.returncode))
+                print(se.decode())
+                rv = False
+            else:
+                output += so.decode()
+        return rv, output
+
+    def copy_or_move(self, op, src, dst):
+        'Handle copy or move operation'
+        if src not in self.items or self.items[src].state[S_WC] == "-":
+            raise InvalidUsageException(None,
+                    "Nothing known about `%s'" % src)
+        if dst in self.items and self.items[dst].state[S_WC] != "-":
+            raise InvalidUsageException(None,
+                    "Destination path `%s' already exists" % dst)
+        # Check that we're not creating dst under a file (not a dir)
+        new_dirs = []
+        def check_parent(d):
+            if d not in self.items or self.items[d].state[S_WC] == "-":
+                new_dirs.append(d)
+            elif self.items[d].state[S_WC] == "F":
+                raise InvalidUsageException(None,
+                        "Destination path `%s' created under `%s' " +
+                        "which is a file" % (dst, d))
+        for_all_parents(dst, check_parent)
+        # All ok, record new directories that may be created
+        for d in new_dirs:
+            self.items.get(d).state[S_WC] = "D"
+        # Record the operation and update the FSO collection
+        self.prepare_ops.append((op, src, dst))
+        self.items.wc_copy(src, dst)
+        if op == "mv":
+            self.items.wc_remove(src)
+
+    def remove(self, path):
+        if path not in self.items or self.items[path].state[S_WC] == "-":
+            raise InvalidUsageException(None,
+                    "Nothing known about `%s'" % path)
+        self.prepare_ops.append(("rm", path))
+        self.items.wc_remove(path)
+
+    def similarity_file(self, src, dst, threshold, lst_removal):
+        'Compare two files, return similarity ratio on 0..1000 scale'
+        if self.items[src].state[S_WC] != "F":
+            return 0
+        # Source is in working copy
+        fn1 = os.path.join(self.wcdir, self.items[src].wc_path)
+        # Destination is in imported dir
+        fn2 = os.path.join(self.importdir, dst)
+        minlines = self.config.get('file-min-lines')
+        try:
+            f1 = open(fn1, 'r')
+            l1 = f1.readlines()
+            f1.close()
+            if len(l1) < minlines:
+                return 0
+            f2 = open(fn2, 'r')
+            l2 = f2.readlines()
+            f2.close()
+            if len(l2) < minlines:
+                return 0
+            sm = difflib.SequenceMatcher(a=l1, b=l2)
+            return int(1000 * sm.quick_ratio())
+        except UnicodeDecodeError:
+            # Oops, file seems to be binary. Fall back to comparing whole
+            # file contents.
+            if filecmp.cmp(fn1, fn2, shallow=False):
+                return 1000
+            return 0
+
+    def _similarity_dir(self, src, dst, get_file_similarity, lst_removal):
+        'Iterate over FSOs, using callback to compare file entries'
+        common = 0
+        total = 0
+        for xsrc in self.items:
+            if xsrc.startswith(src + os.sep):
+                esrc = self.items[xsrc]
+                if esrc.state[S_WC] == "-":
+                    # Source not in WC - ignore for similarity calculation
+                    continue
+                skip = False
+                if lst_removal is not None:
+                    for i in lst_removal:
+                        if descendant_or_self(xsrc, i):
+                            skip = True
+                if skip:
+                    # Moved to another place, do not consider in score
+                    continue
+                total += 1000
+                xdst = path_rebase(xsrc, src, dst)
+                if xdst not in self.items:
+                    # Destination not in imported sources - non-similar item
+                    continue
+                edst = self.items[xdst]
+                if edst.state[S_IM] == esrc.state[S_WC]:
+                    if esrc.state[S_WC] == "D":
+                        common += 1000
+                    else:
+                        common += get_file_similarity(xsrc, xdst)
+        if total == 0:
+            # No files/subdirs in source directory - avoid copying empty dirs
+            return 0
+        return 1000 * common / total
+
+    def similarity_dir(self, src, dst, threshold, lst_removal):
+        '''
+        Compare two dirs recursively, return similarity ratio on
+        0..1000 scale.
+        '''
+        common = 0
+        total = 0
+        # Quickly estimate upper boundary by comparing file names. Only
+        # concern ourselves with files in source directory. I.e., if
+        # files were added after the move in the destination directory,
+        # it's ok. If most of the files from the source directory were
+        # removed, the directory is not considered similar - instead,
+        # file move detection would move files one by one.
+        upper = self._similarity_dir(src, dst, lambda s, d: 1000, lst_removal)
+        if upper <= threshold:
+            # Even the best estimate is worse than current cut-off
+            return 0
+        # Okay, looks roughly similar. Now redo the above procedure, but also
+        # compare the file content.
+        return self._similarity_dir(src, dst,
+                lambda s, d: self.similarity_file(s, d, 0, lst_removal),
+                lst_removal)
+
+    def similar(self, src, dst, threshold=0, lst_removal=None):
+        'Compare two FSOs, source in WC and destination in imported dir'
+        if src not in self.items:
+            print("Source `%s' not in the working copy" % src)
+            return
+        xsrc = self.items[src]
+        if xsrc.state[S_WC] == "-":
+            print("Source `%s' not in the working copy" % src)
+            return
+        if dst not in self.items:
+            print("Destination `%s' not in imported sources" % dst)
+            return
+        xdst = self.items[dst]
+        if xdst.state[S_IM] == "-":
+            print("Destination `%s' not in imported sources" % dst)
+            return
+        if xsrc.state[S_WC] != xdst.state[S_IM]:
+            # Different kinds - definitely not the same object
+            return 0
+        if xsrc.state[S_WC] == "D":
+            return self.similarity_dir(src, dst, threshold, lst_removal)
+        else:
+            return self.similarity_file(src, dst, threshold, lst_removal)
+
+    def handle_op(self, op_tuple):
+        'Handle one SVN operation, recorded as a tuple'
+        def x_mv(src, dst):
+            self.info(2, "  Move `%s' to `%s'" % (src, dst))
+            self.copy_or_move("mv", src, dst)
+        def x_cp(src, dst):
+            self.info(2, "  Copy `%s' to `%s'" % (src, dst))
+            self.copy_or_move("cp", src, dst)
+        def x_rm(path):
+            self.info(2, "  Remove `%s'" % path)
+            self.remove(path)
+        known_ops = {
+                # key: (nargs, handler)
+                'cp' : (3, x_cp),
+                'mv' : (3, x_mv),
+                'rm' : (2, x_rm),
+                }
+        if len(op_tuple) == 0:
+            raise InvalidUsageException
+        op = op_tuple[0]
+        if op not in known_ops:
+            return False
+        nargs, func = known_ops[op]
+        if nargs != len(op_tuple):
+            return False
+        func(*op_tuple[1:])
+        return True
+
+    def detect(self, thresholds):
+        'Helper for finding copy/move destinations'
+        ilst = []
+        wlst = {}
+        ilst_map = {}
+        for p in self.items:
+            e = self.items[p]
+            if e.state[S_WC] != "-" and e.state[S_IM] == "-":
+                wlst[p] = [] # wlst hash stores copy destinations
+            elif e.state[S_WC] == "-" and e.state[S_IM] != "-":
+                # ilst just lists destination paths as tuples with node kind
+                ilst.append((e.state[S_IM], p))
+        iteration = 0
+        # Do not apply operations immediately - we'll need to post-process
+        # them to account for files/dirs moved inside a moved parent dir.
+        ops = []
+        to_be_removed = []
+        def get_renamed_name(path, rename_ops):
+            '''
+            Check if path was renamed/removed in the recorded operations,
+            return new name.
+            '''
+            for op_tuple in rename_ops:
+                # Since copies do not remove the source file, ignore them.
+                # We push no 'rm' ops in this function
+                if op_tuple[0] == "mv":
+                    src = op_tuple[1]
+                    dst = op_tuple[2]
+                    if descendant_or_self(path, src):
+                        path = path_rebase(path, src, dst)
+            return path
+
+        while len(wlst):
+            iteration += 1
+            self.info(2, ("Iteration %d: Possible sources: %d, " +
+                    "possible destinations: %d") %
+                    (iteration, len(wlst), len(ilst)))
+            ndst = len(ilst)
+            for idx, (nk, dst) in enumerate(sorted(ilst,
+                    key=lambda s: filename_sort_key(s[1]))):
+                class SkipDestFile(Exception):
+                    pass
+                # Check if moved as a part of a parent directory.
+                def check_moved_parent(xdst):
+                    if xdst in ilst_map:
+                        src = path_rebase(dst, xdst, ilst_map[xdst])
+                        # Did it exist in copied directory?
+                        if src in self.items and \
+                                self.items[src].state[S_WC] == nk:
+                            sim = self.similar(src, dst, thresholds[nk],
+                                    to_be_removed)
+                            if sim > thresholds[nk]:
+                                self.info(2, ("  [%04d/%04d] Skipping `%s' " +
+                                        "(copied as part of `%s')") %
+                                        (idx, ndst, dst, xdst))
+                                raise SkipDestFile
+                            # Copied, not similar - search for other sources
+                            raise StopIteration
+                try:
+                    for_all_parents(dst, check_moved_parent)
+                except SkipDestFile:
+                    continue
+                except StopIteration:
+                    pass
+                self.info(2, ("  [%04d/%04d] Looking for possible source " +
+                        "for `%s'") % (idx, ndst, dst))
+                bestsrc = None
+                # Won't even consider those lower than threshold
+                bestsim = thresholds[nk]
+                for src in sorted(wlst.keys(),
+                        key=lambda x: name_similarity(x, dst)):
+                    sim = self.similar(src, dst, bestsim, to_be_removed)
+                    if sim > bestsim:
+                        self.info(3, "    [similarity %4d] %s" % (sim, src))
+                        bestsim = sim
+                        bestsrc = src
+                    if bestsim == 1000:
+                        # No chance we're finding anything better
+                        break
+                if bestsrc is not None:
+                    wlst[bestsrc].append(dst)
+                    ilst_map[dst] = bestsrc
+
+            # Discovered all copies/moves, now record them.
+            new_wlst = {}
+            for src in sorted(wlst.keys(), key=filename_sort_key):
+                dlist = wlst[src]
+                if len(dlist) == 0:
+                    continue
+                if len(dlist) == 1:
+                    ops.append(("mv", src, dlist[0]))
+                    to_be_removed.append(src)
+                else:
+                    # We don't remove the source here, it will be done when
+                    # the changes are applied (it will remove all the WC files
+                    # not found in imported sources). Avoiding removal here
+                    # simplifies operation sorting below, since we would not
+                    # be concerned with source file/dir disappearing before
+                    # it is copied to its destination.
+                    to_be_removed.append(src)
+                    for d in dlist:
+                        ops.append(("cp", src, d))
+                # If we copied something - recheck parent source directories.
+                # Since some source file/dir was scheduled to be removed,
+                # this may have increased the similarity to some destination.
+                def recheck_parent(x):
+                    if x in wlst and len(wlst) == 0:
+                        new_wlst[x] = []
+                for_all_parents(src, recheck_parent)
+
+            # At this point, if we're going to have the next iteration, we
+            # are only concerned about directories (by the way new_wlst is
+            # created above). So, filter out all files from ilst as well.
+            wlst = new_wlst
+            ilst = list(filter(lambda t: t[0] == 'D', ilst))
+
+        # Finished collecting the operations - now can post-process and
+        # apply them. First, sort copies/moves by destination (so that
+        # parent directories are created before files/subdirs are
+        # copied/renamed inside)
+        ops = sorted(ops, key=lambda op: filename_sort_key(op[2]))
+        for i, op_tuple in enumerate(ops):
+            # For each operation, go over its precedents to see if the source
+            # has been renamed. If it is, find out new name.
+            op = op_tuple[0]
+            src = get_renamed_name(op_tuple[1], reversed(ops[:i]))
+            if src != op_tuple[2]:
+                # Unless it became the same file after renames
+                try:
+                    # Try to remove the destination, if it existed
+                    self.remove(op_tuple[2])
+                except InvalidUsageException:
+                    # Okay, it didn't exist
+                    pass
+                self.handle_op((op, src, op_tuple[2]))
+
+    def do_detect(self, arg):
+        '''
+        detect : auto-detect possible moves (where source/destination name
+                 is unique). If not all moves are applicable, save move list,
+                 edit and load.
+        '''
+        self.parse_args(arg, 0, "detect")
+        self.detect({ "D": self.config.get('dir-similarity'),
+            "F": self.config.get('file-similarity')})
+
+    def do_apply(self, arg):
+        '''
+        apply : Perform copies/renames; then copy imported sources into
+                the working copy. Modifies working copy. Exits after
+                completion.
+        '''
+        self.info(1, "Copying imported sources into working copy...")
+        # Perform the recorded copies/moves/removals
+        self.info(2, "  Preparatory operations (copies/renames/removals)")
+        to_be_diffed = []
+        for o in self.prepare_ops:
+            op = o[0]
+            if op == "mv":
+                self.run_svn(["mv", "--parents", o[1], o[2]])
+                to_be_diffed.append(o[2])
+            elif op == "cp":
+                self.run_svn(["cp", "--parents", o[1], o[2]])
+                to_be_diffed.append(o[2])
+            elif op == "rm":
+                # --force, as the removed path is likely created as a result
+                # of previous copy/rename
+                self.run_svn(["rm", "--force", o[1]])
+        dirs_added = []
+        dirs_removed = []
+        files_added = []
+        files_removed = []
+        self.info(2, "  Creating dirs and copying files...")
+        for i in sorted(self.items.keys()):
+            e = self.items[i]
+            nk_wc = e.state[S_WC]
+            nk_im = e.state[S_IM]
+            flg = None
+            if nk_wc == "-":
+                # Absent in working copy
+                if nk_im == "D":
+                    # Directory added
+                    os.mkdir(os.path.join(self.wcdir, i))
+                    dirs_added.append(i)
+                    flg = "(added dir)"
+                elif nk_im == "F":
+                    # New file added
+                    shutil.copyfile(os.path.join(self.importdir, i),
+                            os.path.join(self.wcdir, i))
+                    files_added.append(i)
+                    flg = "(added file)"
+                else:
+                    # Not in imported sources, not in WC (moved
+                    # away/removed) - nothing to do
+                    pass
+            elif nk_wc == "F":
+                # File in a working copy
+                if nk_im == "D":
+                    # File replaced with a directory. See comment above.
+                    self.run_svn(["rm", "--force", i])
+                    os.mkdir(os.path.join(self.wcdir, i))
+                    dirs_added.append(i)
+                    flg = "(replaced file with dir)"
+                elif nk_im == "F":
+                    # Was a file, is a file - just copy contents
+                    shutil.copyfile(os.path.join(self.importdir, i),
+                            os.path.join(self.wcdir, i))
+                    flg = "(copied)"
+                else:
+                    # Was a file, removed
+                    files_removed.append(i)
+                    flg = "(removed file)"
+            elif nk_wc == "D":
+                # Directory in a working copy
+                if nk_im == "D":
+                    # Was a directory, is a directory - nothing to do
+                    pass
+                elif nk_im == "F":
+                    # Directory replaced with file. Need to remove dir
+                    # immediately, as bulk removals/additions assume new files
+                    # and dirs already in place.
+                    self.run_svn(["rm", "--force", i])
+                    shutil.copyfile(os.path.join(self.importdir, i),
+                            os.path.join(self.wcdir, i))
+                    files_added.append(i)
+                    flg = "(replaced dir with file)"
+                else:
+                    # Directory removed
+                    dirs_removed.append(i)
+                    flg = "(removed dir)"
+            if flg is not None:
+                self.info(4, "    %s %s %s" % (e.status(), i, flg))
+        # Filter files/directories removed as a part of parent directory
+        files_removed = list(filter(lambda x: os.path.dirname(x) not in
+                dirs_removed, files_removed))
+        dirs_removed = list(filter(lambda x: os.path.dirname(x) not in
+            dirs_removed, dirs_removed))
+        files_added = list(filter(lambda x: os.path.dirname(x) not in
+            dirs_added, files_added))
+        dirs_added = list(filter(lambda x: os.path.dirname(x) not in
+            dirs_added, dirs_added))
+        self.info(2, "  Running SVN add/rm commands");
+        if len(dirs_added):
+            self.run_svn(["add"], dirs_added)
+        if len(files_added):
+            self.run_svn(["add"], files_added)
+        if len(dirs_removed):
+            self.run_svn(["rm"], dirs_removed)
+        if len(files_removed):
+            self.run_svn(["rm"], files_removed)
+        # Save the diff for the copied/moved items
+        diff_save = self.config.get('save-diff-copied')
+        if diff_save is not None:
+            self.info(2, "  Saving 'svn diff' on copied files/dirs to `%s'" %
+                    diff_save)
+            to_be_diffed = list(filter(lambda x: os.path.dirname(x) not in
+                to_be_diffed, to_be_diffed))
+            if len(to_be_diffed):
+                try:
+                    rv, out = self.run_svn(["diff"], to_be_diffed)
+                except UnicodeDecodeError:
+                    # Some binary files not marked with appropriate MIME type,
+                    # or broken text files
+                    rv, out = (True, "WARNING: diff contained binary files\n")
+            else:
+                rv, out = (True, "")
+            if rv:
+                f = open(diff_save, "w")
+                f.write(out)
+                f.close()
+        # Exiting, as the resulting working copy can no longer be used
+        # for move analysis
+        self.info(1, "Done. Exiting; please examine the working copy " +
+                "and commit.")
+        return True
+
+    def do_similarity(self, arg):
+        '''
+        similarity SRD DST : estimate whether SRC could be potential source
+                             for DST (0=no match, 1000=perfect match)
+        '''
+        src, dst = self.parse_args(arg, 2, "similarity")
+        sim = self.similar(src, dst)
+        if sim is not None:
+            print("Similarity between source `%s' and destination `%s': %4d" %
+                    (src, dst, sim))
+
+    def do_set(self, arg):
+        '''
+        set         : display current settings
+        set CFG VAL : set a config variable
+        '''
+        if arg.strip() == '':
+            self.config.print()
+        else:
+            cfg, val = self.parse_args(arg, 2, "set")
+            self.config.set(cfg, val)
+
+    def do_move(self, arg):
+        '''
+        move SRC DST : Perform a move from source to destination
+        '''
+        src, dst = self.parse_args(arg, 2, "move")
+        self.copy_or_move("mv", src, dst)
+
+    def do_copy(self, arg):
+        '''
+        copy SRC DST : Perform a copy from source to destination
+        '''
+        src, dst = self.parse_args(arg, 2, "copy")
+        self.copy_or_move("cp", src, dst)
+
+    def do_remove(self, arg):
+        '''
+        remove PATH : Remove a path
+        '''
+        path = self.parse_args(arg, 1, "remove")[0]
+        self.copy_or_move("rm", path)
+
+    def do_lsprep(self, arg):
+        '''
+        lsprep : List the currently recorded moves/copies/removals
+        '''
+        self.parse_args(arg, 0, "lsprep")
+        colsz = int((self.termwidth - 14) / 2)
+        if len(self.prepare_ops):
+            print("Currently recorded preparatory operations:")
+            print()
+            print("%5s  %s  %-*s  %-*s" %
+                    ("#", "Op", colsz, "Source", colsz, "Destination"))
+            for id, o in enumerate(self.prepare_ops):
+                if id % 10 == 0:
+                    print("%5s  %s  %*s  %*s" %
+                            ("-"*5, "--", colsz, "-"*colsz, colsz, "-"*colsz))
+                if len(o) == 3:
+                    print("%5d  %s  %-*s  %-*s" %
+                            (id, o[0], colsz, o[1], colsz, o[2]))
+                else:
+                    print("%5d  %s  %-*s" % (id, o[0], colsz, o[1]))
+            print()
+        else:
+            print("No copies/moves/removals recorded")
+            print()
+
+    def do_save(self, arg):
+        '''
+        save FILENAME : Save current preparation operations to a file
+        '''
+        fn = self.parse_args(arg, 1, "save")[0]
+        f = open(fn, 'w')
+        longestname = 0
+        for o in self.prepare_ops:
+            if len(o[1]) > longestname:
+                longestname = len(o[1])
+            if len(o) == 3 and len(o[2]) > longestname:
+                longestname = len(o[2])
+        for o in self.prepare_ops:
+            if len(o) == 2:
+                f.write("svn %s %-*s\n" %
+                        (o[0], longestname, shlex.quote(o[1])))
+            else:
+                f.write("svn %s %-*s %-*s\n" %
+                        (o[0], longestname, shlex.quote(o[1]),
+                            longestname, shlex.quote(o[2])))
+            pass
+        f.close()
+
+    def do_load(self, arg):
+        '''
+        load FILENAME : Load/append preparation operations from a file
+        '''
+        fn = self.parse_args(arg, 1, "load")[0]
+        self.info(1, "Performing operations from `%s'" % fn)
+        f = open(fn, 'r')
+        for l in f.readlines():
+            if l[0] == '#':
+                continue
+            args = shlex.split(l)
+            try:
+                if len(args) < 2 or args[0] != 'svn':
+                    raise InvalidUsageException(None, "")
+                self.handle_op(args[1:])
+            except InvalidUsageException as e:
+                # Rethrow
+                raise InvalidUsageException(None,
+                        "Invalid line in file: %s(%s)" % (l, e))
+        f.close()
+
+    def do_svninfo(self, arg):
+        '''
+        svninfo : Display SVN info on the working copy (debug)
+        '''
+        self.parse_args(arg, 0, "svninfo")
+        print(str(self.svninfo))
+
+    def do_printlst(self, arg):
+        '''
+        printlst WHAT : Print list of files; WHAT is one of {dir,file} (debug)
+        '''
+        self.parse_args(arg, 0, "printlst")
+        self.items.print()
+
+    def do_help(self, arg):
+        '''
+        help [COMMAND] : Print the help message
+        '''
+        cmd.Cmd.do_help(self, arg)
+
+    def do_EOF(self, arg):
+        '''
+        Quit the script
+        '''
+        return True
+
+    def do_quit(self, arg):
+        '''
+        quit : Quit the script
+        '''
+        return True
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+            description="Prepare a working copy for SVN vendor import.")
+    parser.add_argument('wcdir',
+            help="Path to working copy (destination of import)")
+    parser.add_argument('importdir',
+            help="Path to imported sources (source of import)")
+    grp = parser.add_mutually_exclusive_group()
+    grp.add_argument('--auto', action='store_true',
+            help="Automatic mode: detect moves, apply them and copy sources")
+    grp.add_argument('--detect', metavar='FILE',
+            help="Semi-automatic mode: detect moves and save them to FILE")
+    grp.add_argument('--apply', metavar='FILE',
+            help="Semi-automatic mode: apply the moves from FILE " +
+            "and copy the sources")
+    parser.add_argument('--save', metavar='FILE',
+            help="Automatic mode: save moves to FILE after detection, " +
+            "then proceed to apply the changes")
+    parser.add_argument('--config', metavar=('OPT','VALUE'), action='append',
+            nargs=2, help="Set configuration option OPT to VALUE")
+    args = parser.parse_args()
+    p = subprocess.Popen(args=['svn', 'info', args.wcdir],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    so, se = p.communicate()
+    if p.returncode != 0:
+        print("%s: does not appear to be SVN working copy." % args.wcdir)
+        print("`svn info' exited with status %d and returned:" % p.returncode)
+        print()
+        print(se.decode())
+        sys.exit(1)
+    imp = SvnVndImport(args.wcdir, args.importdir, so.decode())
+    if args.config:
+        try:
+            for o, v in args.config:
+                imp.config.set(o, v)
+        except InvalidUsageException as e:
+            parser.error(e)
+    imp.scan()
+    if args.auto:
+        imp.onecmd("detect")
+        if args.save:
+            imp.onecmd("save " + shlex.quote(args.save))
+        imp.onecmd("apply")
+    elif args.detect:
+        imp.onecmd("detect")
+        imp.onecmd("save " + shlex.quote(args.detect))
+    elif args.apply:
+        imp.onecmd("load " + shlex.quote(args.apply))
+        imp.onecmd("apply")
+    else:
+        imp.cmdloop()