From a5db3d3c49ebe559cb80983d7bb855d4adf1b887 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 28 Sep 2016 01:05:38 +0200 Subject: io, dif: #519: FIX DIFF freeze when reading from GIL + CAUSE: In Windows, Diffs freeze while reading Popen streams, probably buffers smaller; good-thin(TM) in this case because reading a Popen-proc from the launching-thread freezes GIL. The alternative to use `proc.communicate()` also relies on big buffers. + SOLUTION: Use `cmd.handle_process_output()` to consume Diff-proc streams. + Retroffited `handle_process_output()` code to support also byte-streams, both Threading(Windows) and Select/Poll (Posix) paths updated. - TODO: Unfortunately, `Diff._index_from_patch_format()` still slurps input; need to re-phrase header-regexes linewise to resolve it. --- git/diff.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) (limited to 'git/diff.py') diff --git a/git/diff.py b/git/diff.py index fb8faaf6..54804c45 100644 --- a/git/diff.py +++ b/git/diff.py @@ -15,6 +15,8 @@ from git.compat import ( defenc, PY3 ) +from git.cmd import handle_process_output +from git.util import finalize_process __all__ = ('Diffable', 'DiffIndex', 'Diff', 'NULL_TREE') @@ -145,10 +147,10 @@ class Diffable(object): kwargs['as_process'] = True proc = diff_cmd(*self._process_diff_args(args), **kwargs) - diff_method = Diff._index_from_raw_format - if create_patch: - diff_method = Diff._index_from_patch_format - index = diff_method(self.repo, proc.stdout) + diff_method = (Diff._index_from_patch_format + if create_patch + else Diff._index_from_raw_format) + index = diff_method(self.repo, proc) proc.wait() return index @@ -397,13 +399,18 @@ class Diff(object): return None @classmethod - def _index_from_patch_format(cls, repo, stream): + def _index_from_patch_format(cls, repo, proc): """Create a new DiffIndex from the given text which must be in patch format :param repo: is the repository we are operating on - it is required :param stream: result of 'git diff' as a stream (supporting file protocol) :return: git.DiffIndex """ + + ## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. + text = [] + handle_process_output(proc, text.append, None, finalize_process, decode_stdout=False) + # for now, we have to bake the stream - text = stream.read() + text = b''.join(text) index = DiffIndex() previous_header = None for header in cls.re_header.finditer(text): @@ -450,17 +457,19 @@ class Diff(object): return index @classmethod - def _index_from_raw_format(cls, repo, stream): + def _index_from_raw_format(cls, repo, proc): """Create a new DiffIndex from the given stream which must be in raw format. :return: git.DiffIndex""" # handles # :100644 100644 687099101... 37c5e30c8... M .gitignore + index = DiffIndex() - for line in stream.readlines(): + + def handle_diff_line(line): line = line.decode(defenc) if not line.startswith(":"): - continue - # END its not a valid diff line + return + meta, _, path = line[1:].partition('\t') old_mode, new_mode, a_blob_id, b_blob_id, change_type = meta.split(None, 4) path = path.strip() @@ -489,6 +498,7 @@ class Diff(object): diff = Diff(repo, a_path, b_path, a_blob_id, b_blob_id, old_mode, new_mode, new_file, deleted_file, rename_from, rename_to, '', change_type) index.append(diff) - # END for each line + + handle_process_output(proc, handle_diff_line, None, finalize_process, decode_stdout=False) return index -- cgit v1.2.1 From 44c6d0b368bc1ec6cd0a97b01678b38788c9bd9c Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 28 Sep 2016 05:46:50 +0200 Subject: Proc, #519: Rework error-exc msgs & log thread-pumps errors + No WindowsError exception. + Add `test_exc.py` for unicode issues. + Single-arg for decoding-streams in pump-func. --- git/diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'git/diff.py') diff --git a/git/diff.py b/git/diff.py index 54804c45..35c7ff86 100644 --- a/git/diff.py +++ b/git/diff.py @@ -407,7 +407,7 @@ class Diff(object): ## FIXME: Here SLURPING raw, need to re-phrase header-regexes linewise. text = [] - handle_process_output(proc, text.append, None, finalize_process, decode_stdout=False) + handle_process_output(proc, text.append, None, finalize_process, decode_streams=False) # for now, we have to bake the stream text = b''.join(text) @@ -499,6 +499,6 @@ class Diff(object): new_file, deleted_file, rename_from, rename_to, '', change_type) index.append(diff) - handle_process_output(proc, handle_diff_line, None, finalize_process, decode_stdout=False) + handle_process_output(proc, handle_diff_line, None, finalize_process, decode_streams=False) return index -- cgit v1.2.1