fix(encoding): in untracked_files() and index

* untracked_files could, if there were spaces in the path returned,
  re-rencode the previously decoded unicode string thanks to a
  `decode("string_escape")` call. Now re-encode into utf-8 afterwards
  - added test to assure this works indeed
* IndexFile.add() didn't handle unicode correctly and would write
  broken index files. The solution was to compute the path length after
  encoding it into utf-8 bytes, not before ... .

Closes #320

2 files changed, 4 insertions, 3 deletions

diff --git a/git/index/base.py b/git/index/base.py
index b955dae4..4317d46a 100644
--- a/git/index/base.py
+++ b/git/index/base.py
@@ -583,7 +583,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
         stream = None
         if S_ISLNK(st.st_mode):
             # in PY3, readlink is string, but we need bytes. In PY2, it's just OS encoded bytes, we assume UTF-8
-            stream = BytesIO(force_bytes(os.readlink(filepath), encoding='utf-8'))
+            stream = BytesIO(force_bytes(os.readlink(filepath), encoding=defenc))
         else:
             stream = open(filepath, 'rb')
         # END handle stream
@@ -610,7 +610,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable):
 
                 blob = Blob(self.repo, Blob.NULL_BIN_SHA,
                             stat_mode_to_index_mode(os.stat(abspath).st_mode),
-                            to_native_path_linux(gitrelative_path))
+                            to_native_path_linux(gitrelative_path), encoding=defenc)
                 # TODO: variable undefined
                 entries.append(BaseIndexEntry.from_blob(blob))
             # END for each path
diff --git a/git/index/fun.py b/git/index/fun.py
index c1188ccb..9ae46861 100644
--- a/git/index/fun.py
+++ b/git/index/fun.py
@@ -124,12 +124,13 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1
         write(entry[4])         # ctime
         write(entry[5])         # mtime
         path = entry[3]
+        path = path.encode(defenc)
         plen = len(path) & CE_NAMEMASK      # path length
         assert plen == len(path), "Path %s too long to fit into index" % entry[3]
         flags = plen | (entry[2] & CE_NAMEMASK_INV)     # clear possible previous values
         write(pack(">LLLLLL20sH", entry[6], entry[7], entry[0],
                    entry[8], entry[9], entry[10], entry[1], flags))
-        write(path.encode(defenc))
+        write(path)
         real_size = ((tell() - beginoffset + 8) & ~7)
         write(b"\0" * ((beginoffset + real_size) - tell()))
     # END for each entry