io, dif: #519: FIX DIFF freeze when reading from GIL

+ CAUSE: In Windows, Diffs freeze while reading Popen streams,
probably buffers smaller; good-thin(TM) in this case because reading a
Popen-proc from the launching-thread freezes GIL.  The alternative to
use `proc.communicate()` also relies on big buffers.
+ SOLUTION: Use `cmd.handle_process_output()` to consume Diff-proc
streams.
+ Retroffited `handle_process_output()` code to support also
byte-streams, both Threading(Windows) and Select/Poll (Posix) paths
updated.

- TODO: Unfortunately, `Diff._index_from_patch_format()` still slurps
input; need to re-phrase header-regexes linewise to resolve it.

1 files changed, 21 insertions, 11 deletions

diff --git a/git/diff.py b/git/diff.py
index fb8faaf6..54804c45 100644
--- a/git/diff.py
+++ b/git/diff.py
@@ -15,6 +15,8 @@ from git.compat import (
     defenc,
     PY3
 )
+from git.cmd import handle_process_output
+from git.util import finalize_process
 
 __all__ = ('Diffable', 'DiffIndex', 'Diff', 'NULL_TREE')
 
@@ -145,10 +147,10 @@ class Diffable(object):
         kwargs['as_process'] = True
         proc = diff_cmd(*self._process_diff_args(args), **kwargs)
 
-        diff_method = Diff._index_from_raw_format
-        if create_patch:
-            diff_method = Diff._index_from_patch_format
-        index = diff_method(self.repo, proc.stdout)
+        diff_method = (Diff._index_from_patch_format
+                       if create_patch
+                       else Diff._index_from_raw_format)
+        index = diff_method(self.repo, proc)
 
         proc.wait()
         return index
@@ -397,13 +399,18 @@ class Diff(object):
         return None
 
     @classmethod
-    def _index_from_patch_format(cls, repo, stream):
+    def _index_from_patch_format(cls, repo, proc):
         """Create a new DiffIndex from the given text which must be in patch format
         :param repo: is the repository we are operating on - it is required
         :param stream: result of 'git diff' as a stream (supporting file protocol)
         :return: git.DiffIndex """
+
+        ## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise.
+        text = []
+        handle_process_output(proc, text.append, None, finalize_process, decode_stdout=False)
+
         # for now, we have to bake the stream
-        text = stream.read()
+        text = b''.join(text)
         index = DiffIndex()
         previous_header = None
         for header in cls.re_header.finditer(text):
@@ -450,17 +457,19 @@ class Diff(object):
         return index
 
     @classmethod
-    def _index_from_raw_format(cls, repo, stream):
+    def _index_from_raw_format(cls, repo, proc):
         """Create a new DiffIndex from the given stream which must be in raw format.
         :return: git.DiffIndex"""
         # handles
         # :100644 100644 687099101... 37c5e30c8... M    .gitignore
+
         index = DiffIndex()
-        for line in stream.readlines():
+
+        def handle_diff_line(line):
             line = line.decode(defenc)
             if not line.startswith(":"):
-                continue
-            # END its not a valid diff line
+                return
+
             meta, _, path = line[1:].partition('\t')
             old_mode, new_mode, a_blob_id, b_blob_id, change_type = meta.split(None, 4)
             path = path.strip()
@@ -489,6 +498,7 @@ class Diff(object):
             diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode,
                         new_file, deleted_file, rename_from, rename_to, '', change_type)
             index.append(diff)
-        # END for each line
+
+        handle_process_output(proc, handle_diff_line, None, finalize_process, decode_stdout=False)
 
         return index