Merge branch 'py3' into 0.3

Conflicts:
	git/refs/log.py

3 files changed, 43 insertions, 37 deletions

diff --git a/git/index/base.py b/git/index/base.py
index fdcfcd12..cc883469 100644
--- a/git/index/base.py
+++ b/git/index/base.py
@@ -8,16 +8,16 @@ import os
 import sys
 import subprocess
 import glob
-from cStringIO import StringIO
+from io import BytesIO
 
 from stat import S_ISLNK
 
-from typ import (
+from .typ import (
     BaseIndexEntry,
     IndexEntry,
 )
 
-from util import (
+from .util import (
     TemporaryFileSwap,
     post_clear_cache,
     default_index,
@@ -25,7 +25,6 @@ from util import (
 )
 
 import git.diff as diff
-
 from git.exc import (
     GitCommandError,
     CheckoutError
@@ -40,6 +39,14 @@ from git.objects import (
 )
 
 from git.objects.util import Serializable
+from git.compat import (
+    izip,
+    xrange,
+    string_types,
+    force_bytes,
+    defenc,
+    mviter
+)
 
 from git.util import (
     LazyMixin,
@@ -49,7 +56,7 @@ from git.util import (
     to_native_path_linux,
 )
 
-from fun import (
+from .fun import (
     entry_key,
     write_cache,
     read_cache,
@@ -62,7 +69,6 @@ from fun import (
 from gitdb.base import IStream
 from gitdb.db import MemoryDB
 from gitdb.util import to_bin_sha
-from itertools import izip
 
 __all__ = ('IndexFile', 'CheckoutError')
 
@@ -101,7 +107,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
         repository's index on demand."""
         self.repo = repo
         self.version = self._VERSION
-        self._extension_data = ''
+        self._extension_data = b''
         self._file_path = file_path or self._index_path()
 
     def _set_cache_(self, attr):
@@ -161,9 +167,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
 
     def _entries_sorted(self):
         """:return: list of entries, in a sorted fashion, first by path, then by stage"""
-        entries_sorted = self.entries.values()
-        entries_sorted.sort(key=lambda e: (e.path, e.stage))        # use path/stage as sort key
-        return entries_sorted
+        return sorted(self.entries.values(), key=lambda e: (e.path, e.stage))
 
     def _serialize(self, stream, ignore_tree_extension_data=False):
         entries = self._entries_sorted()
@@ -395,7 +399,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
         fprogress(filepath, False, item)
         rval = None
         try:
-            proc.stdin.write("%s\n" % filepath)
+            proc.stdin.write(("%s\n" % filepath).encode(defenc))
         except IOError:
             # pipe broke, usually because some error happend
             raise fmakeexc()
@@ -414,7 +418,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
             Function(t) returning True if tuple(stage, Blob) should be yielded by the
             iterator. A default filter, the BlobFilter, allows you to yield blobs
             only if they match a given list of paths. """
-        for entry in self.entries.itervalues():
+        for entry in mviter(self.entries):
             blob = entry.to_blob(self.repo)
             blob.size = entry.size
             output = (entry.stage, blob)
@@ -439,7 +443,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
         for stage, blob in self.iter_blobs(is_unmerged_blob):
             path_map.setdefault(blob.path, list()).append((stage, blob))
         # END for each unmerged blob
-        for l in path_map.itervalues():
+        for l in mviter(path_map):
             l.sort()
         return path_map
 
@@ -542,7 +546,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
         entries = list()
 
         for item in items:
-            if isinstance(item, basestring):
+            if isinstance(item, string_types):
                 paths.append(self._to_relative_path(item))
             elif isinstance(item, (Blob, Submodule)):
                 entries.append(BaseIndexEntry.from_blob(item))
@@ -559,7 +563,8 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
         st = os.lstat(filepath)     # handles non-symlinks as well
         stream = None
         if S_ISLNK(st.st_mode):
-            stream = StringIO(os.readlink(filepath))
+            # in PY3, readlink is string, but we need bytes. In PY2, it's just OS encoded bytes, we assume UTF-8
+            stream = BytesIO(force_bytes(os.readlink(filepath), encoding='utf-8'))
         else:
             stream = open(filepath, 'rb')
         # END handle stream
@@ -753,7 +758,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
         for item in items:
             if isinstance(item, (BaseIndexEntry, (Blob, Submodule))):
                 paths.append(self._to_relative_path(item.path))
-            elif isinstance(item, basestring):
+            elif isinstance(item, string_types):
                 paths.append(self._to_relative_path(item))
             else:
                 raise TypeError("Invalid item type: %r" % item)
@@ -855,7 +860,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
 
         # parse result - first 0:n/2 lines are 'checking ', the remaining ones
         # are the 'renaming' ones which we parse
-        for ln in xrange(len(mvlines) / 2, len(mvlines)):
+        for ln in xrange(int(len(mvlines) / 2), len(mvlines)):
             tokens = mvlines[ln].split(' to ')
             assert len(tokens) == 2, "Too many tokens in %s" % mvlines[ln]
 
@@ -953,6 +958,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
             if not stderr:
                 return
             # line contents:
+            stderr = stderr.decode(defenc)
             # git-checkout-index: this already exists
             failed_files = list()
             failed_reasons = list()
@@ -1001,11 +1007,11 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
             proc = self.repo.git.checkout_index(*args, **kwargs)
             proc.wait()
             fprogress(None, True, None)
-            rval_iter = (e.path for e in self.entries.itervalues())
+            rval_iter = (e.path for e in mviter(self.entries))
             handle_stderr(proc, rval_iter)
             return rval_iter
         else:
-            if isinstance(paths, basestring):
+            if isinstance(paths, string_types):
                 paths = [paths]
 
             # make sure we have our entries loaded before we start checkout_index
@@ -1031,7 +1037,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
                     dir = co_path
                     if not dir.endswith('/'):
                         dir += '/'
-                    for entry in self.entries.itervalues():
+                    for entry in mviter(self.entries):
                         if entry.path.startswith(dir):
                             p = entry.path
                             self._write_path_to_stdin(proc, p, p, make_exc,
@@ -1141,7 +1147,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
         # index against anything but None is a reverse diff with the respective
         # item. Handle existing -R flags properly. Transform strings to the object
         # so that we can call diff on it
-        if isinstance(other, basestring):
+        if isinstance(other, string_types):
             other = self.repo.rev_parse(other)
         # END object conversion
 
diff --git a/git/index/fun.py b/git/index/fun.py
index eec90519..f0dee961 100644
--- a/git/index/fun.py
+++ b/git/index/fun.py
@@ -12,7 +12,7 @@ from stat import (
 
 S_IFGITLINK = S_IFLNK | S_IFDIR     # a submodule
 
-from cStringIO import StringIO
+from io import BytesIO
 
 from git.util import IndexFileSHA1Writer
 from git.exc import UnmergedEntriesError
@@ -22,7 +22,7 @@ from git.objects.fun import (
     traverse_trees_recursive
 )
 
-from typ import (
+from .typ import (
     BaseIndexEntry,
     IndexEntry,
     CE_NAMEMASK,
@@ -30,13 +30,14 @@ from typ import (
 )
 CE_NAMEMASK_INV = ~CE_NAMEMASK
 
-from util import (
+from .util import (
     pack,
     unpack
 )
 
 from gitdb.base import IStream
 from gitdb.typ import str_tree_type
+from git.compat import defenc
 
 __all__ = ('write_cache', 'read_cache', 'write_tree_from_cache', 'entry_key',
            'stat_mode_to_index_mode', 'S_IFGITLINK')
@@ -49,7 +50,7 @@ def stat_mode_to_index_mode(mode):
         return S_IFLNK
     if S_ISDIR(mode) or S_IFMT(mode) == S_IFGITLINK:    # submodules
         return S_IFGITLINK
-    return S_IFREG | 0644 | (mode & 0100)       # blobs with or without executable bit
+    return S_IFREG | 0o644 | (mode & 0o100)       # blobs with or without executable bit
 
 
 def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1Writer):
@@ -72,7 +73,7 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1
 
     # header
     version = 2
-    write("DIRC")
+    write(b"DIRC")
     write(pack(">LL", version, len(entries)))
 
     # body
@@ -86,9 +87,9 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1
         flags = plen | (entry[2] & CE_NAMEMASK_INV)     # clear possible previous values
         write(pack(">LLLLLL20sH", entry[6], entry[7], entry[0],
                    entry[8], entry[9], entry[10], entry[1], flags))
-        write(path)
+        write(path.encode(defenc))
         real_size = ((tell() - beginoffset + 8) & ~7)
-        write("\0" * ((beginoffset + real_size) - tell()))
+        write(b"\0" * ((beginoffset + real_size) - tell()))
     # END for each entry
 
     # write previously cached extensions data
@@ -102,7 +103,7 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1
 def read_header(stream):
     """Return tuple(version_long, num_entries) from the given stream"""
     type_id = stream.read(4)
-    if type_id != "DIRC":
+    if type_id != b"DIRC":
         raise AssertionError("Invalid index file header: %r" % type_id)
     version, num_entries = unpack(">LL", stream.read(4 * 2))
 
@@ -142,7 +143,7 @@ def read_cache(stream):
         (dev, ino, mode, uid, gid, size, sha, flags) = \
             unpack(">LLLLLL20sH", read(20 + 4 * 6 + 2))
         path_size = flags & CE_NAMEMASK
-        path = read(path_size)
+        path = read(path_size).decode(defenc)
 
         real_size = ((tell() - beginoffset + 8) & ~7)
         read((beginoffset + real_size) - tell())
@@ -218,7 +219,7 @@ def write_tree_from_cache(entries, odb, sl, si=0):
     # END for each entry
 
     # finally create the tree
-    sio = StringIO()
+    sio = BytesIO()
     tree_to_stream(tree_items, sio.write)
     sio.seek(0)
 
diff --git a/git/index/typ.py b/git/index/typ.py
index 222252c5..0998ecb0 100644
--- a/git/index/typ.py
+++ b/git/index/typ.py
@@ -1,15 +1,14 @@
 """Module with additional types used by the index"""
 
-from util import (
+from binascii import b2a_hex
+
+from .util import (
     pack,
     unpack
 )
+from git.objects import Blob
 
-from binascii import (
-    b2a_hex,
-)
 
-from git.objects import Blob
 __all__ = ('BlobFilter', 'BaseIndexEntry', 'IndexEntry')
 
 #{ Invariants
@@ -76,7 +75,7 @@ class BaseIndexEntry(tuple):
     @property
     def hexsha(self):
         """hex version of our sha"""
-        return b2a_hex(self[1])
+        return b2a_hex(self[1]).decode('ascii')
 
     @property
     def stage(self):