From b343718cc1290c8d5fd5b1217724b077153262a8 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 26 Sep 2016 02:37:38 +0200 Subject: test, #519: Popen() pump: remove WaitGroup --- git/util.py | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index f5c69231..b56b96da 100644 --- a/git/util.py +++ b/git/util.py @@ -12,7 +12,6 @@ import stat import shutil import platform import getpass -import threading import logging # NOTE: Some of the unused imports might be used/imported by others. @@ -39,7 +38,7 @@ from gitdb.util import ( # NOQA __all__ = ("stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux", "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList", "BlockingLockFile", "LockFile", 'Actor', 'get_user_id', 'assure_directory_exists', - 'RemoteProgress', 'CallableRemoteProgress', 'rmtree', 'WaitGroup', 'unbare_repo') + 'RemoteProgress', 'CallableRemoteProgress', 'rmtree', 'unbare_repo') #{ Utility Methods @@ -324,12 +323,12 @@ class RemoteProgress(object): You may read the contents of the current line in self._cur_line""" pass - + class CallableRemoteProgress(RemoteProgress): """An implementation forwarding updates to any callable""" __slots__ = ('_callable') - + def __init__(self, fn): self._callable = fn super(CallableRemoteProgress, self).__init__() @@ -754,35 +753,6 @@ class Iterable(object): #} END classes -class WaitGroup(object): - """WaitGroup is like Go sync.WaitGroup. - - Without all the useful corner cases. - By Peter Teichman, taken from https://gist.github.com/pteichman/84b92ae7cef0ab98f5a8 - """ - def __init__(self): - self.count = 0 - self.cv = threading.Condition() - - def add(self, n): - self.cv.acquire() - self.count += n - self.cv.release() - - def done(self): - self.cv.acquire() - self.count -= 1 - if self.count == 0: - self.cv.notify_all() - self.cv.release() - - def wait(self, stderr=b''): - self.cv.acquire() - while self.count > 0: - self.cv.wait() - self.cv.release() - - class NullHandler(logging.Handler): def emit(self, record): pass -- cgit v1.2.1 From f495e94028bfddc264727ffc464cd694ddd05ab8 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 26 Sep 2016 20:41:41 +0200 Subject: src, #519: collect all is_() calls --- git/util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index b56b96da..31ff94fa 100644 --- a/git/util.py +++ b/git/util.py @@ -6,7 +6,6 @@ import os import re -import sys import time import stat import shutil @@ -26,7 +25,7 @@ from .compat import ( # Most of these are unused here, but are for use by git-python modules so these # don't see gitdb all the time. Flake of course doesn't like it. -from gitdb.util import ( # NOQA +from gitdb.util import (# NOQA make_sha, LockedFD, file_contents_ro, @@ -34,6 +33,7 @@ from gitdb.util import ( # NOQA to_hex_sha, to_bin_sha ) +from git.compat import is_win __all__ = ("stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux", "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList", @@ -106,7 +106,7 @@ def join_path(a, *p): return path -if sys.platform.startswith('win'): +if is_win(): def to_native_path_windows(path): return path.replace('/', '\\') @@ -587,7 +587,7 @@ class LockFile(object): try: # on bloody windows, the file needs write permissions to be removable. # Why ... - if os.name == 'nt': + if is_win(): os.chmod(lfp, 0o777) # END handle win32 os.remove(lfp) -- cgit v1.2.1 From e61439b3018b0b9a8eb43e59d0d7cf32041e2fed Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 27 Sep 2016 16:05:58 +0200 Subject: src: constify is_() calls + TCs: unittest-asserts for git-tests. --- git/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index 31ff94fa..f931abe2 100644 --- a/git/util.py +++ b/git/util.py @@ -106,7 +106,7 @@ def join_path(a, *p): return path -if is_win(): +if is_win: def to_native_path_windows(path): return path.replace('/', '\\') @@ -587,7 +587,7 @@ class LockFile(object): try: # on bloody windows, the file needs write permissions to be removable. # Why ... - if is_win(): + if is_win: os.chmod(lfp, 0o777) # END handle win32 os.remove(lfp) -- cgit v1.2.1 From 137ee6ef22c4e6480f95972ef220d1832cdc709a Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Tue, 27 Sep 2016 22:07:19 +0200 Subject: Win, #519: FIX with_rw_directory() to remove read-only dirs + Stop using gitdb's respective helper. + Fix files chmod(555) which CANNOT DELETE on Windows (but do on Linux). --- git/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index f931abe2..eb5a6ac1 100644 --- a/git/util.py +++ b/git/util.py @@ -68,7 +68,7 @@ def rmtree(path): os.chmod(path, stat.S_IWUSR) func(path) else: - raise + raise FileExistsError("Cannot delete '%s'", path) # END end onerror return shutil.rmtree(path, False, onerror) -- cgit v1.2.1 From cf2335af23fb693549d6c4e72b65f97afddc5f64 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 28 Sep 2016 01:47:49 +0200 Subject: Win, hook, #519: Consume Hook Popen-proc out of GIL + HookException thrown on Popen, and were missed on Windows. + No SHELL on Popen?? + Minor fixes: + Try harder to delete trees - no remorses. + Simplify exception reprs. + Unittest-ize test_index assertions. --- git/util.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index eb5a6ac1..9faa8eff 100644 --- a/git/util.py +++ b/git/util.py @@ -62,14 +62,12 @@ def rmtree(path): :note: we use shutil rmtree but adjust its behaviour to see whether files that couldn't be deleted are read-only. Windows will not remove them in that case""" + def onerror(func, path, exc_info): - if not os.access(path, os.W_OK): - # Is the error an access error ? - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise FileExistsError("Cannot delete '%s'", path) - # END end onerror + # Is the error an access error ? + os.chmod(path, stat.S_IWUSR) + func(path) # Will scream if still not possible to delete. + return shutil.rmtree(path, False, onerror) @@ -151,6 +149,7 @@ def get_user_id(): def finalize_process(proc, **kwargs): """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly""" + ## TODO: No close proc-streams?? proc.wait(**kwargs) #} END utilities -- cgit v1.2.1 From f11fdf1d9d22a198511b02f3ca90146cfa5deb5c Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Wed, 28 Sep 2016 14:43:47 +0200 Subject: remote, #519: FIX1-of-2 double-decoding push-infos + When `universal_lines==True` (515a6b9ccf8) must tel `handle_process_output` to stop decoding strings. --- git/util.py | 1 + 1 file changed, 1 insertion(+) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index 9faa8eff..f6f6dea9 100644 --- a/git/util.py +++ b/git/util.py @@ -3,6 +3,7 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php +from __future__ import unicode_literals import os import re -- cgit v1.2.1 From 13d399f4460ecb17cecc59d7158a4159010b2ac5 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 12:12:19 +0200 Subject: ci: restore ci log-level to normal, coverage on Win-Appveyor + Extract util-method to delete lock-files, also on Windows (will be needed by TCs). --- git/util.py | 53 +++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 24 deletions(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index f6f6dea9..87ef38d3 100644 --- a/git/util.py +++ b/git/util.py @@ -5,37 +5,39 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php from __future__ import unicode_literals +import getpass +import logging import os +import platform import re -import time -import stat import shutil -import platform -import getpass -import logging +import stat +import time -# NOTE: Some of the unused imports might be used/imported by others. -# Handle once test-cases are back up and running. -from .exc import InvalidGitRepositoryError +from git.compat import is_win +from gitdb.util import ( # NOQA + make_sha, + LockedFD, + file_contents_ro, + LazyMixin, + to_hex_sha, + to_bin_sha +) + +import os.path as osp from .compat import ( MAXSIZE, defenc, PY3 ) +from .exc import InvalidGitRepositoryError + +# NOTE: Some of the unused imports might be used/imported by others. +# Handle once test-cases are back up and running. # Most of these are unused here, but are for use by git-python modules so these # don't see gitdb all the time. Flake of course doesn't like it. -from gitdb.util import (# NOQA - make_sha, - LockedFD, - file_contents_ro, - LazyMixin, - to_hex_sha, - to_bin_sha -) -from git.compat import is_win - __all__ = ("stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux", "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList", "BlockingLockFile", "LockFile", 'Actor', 'get_user_id', 'assure_directory_exists', @@ -72,6 +74,14 @@ def rmtree(path): return shutil.rmtree(path, False, onerror) +def rmfile(path): + """Ensure file deleted also on *Windows* where read-only files need special treatment.""" + if osp.isfile(path): + if is_win: + os.chmod(path, 0o777) + os.remove(path) + + def stream_copy(source, destination, chunk_size=512 * 1024): """Copy all data from the source stream into the destination stream in chunks of size chunk_size @@ -585,12 +595,7 @@ class LockFile(object): # instead of failing, to make it more usable. lfp = self._lock_file_path() try: - # on bloody windows, the file needs write permissions to be removable. - # Why ... - if is_win: - os.chmod(lfp, 0o777) - # END handle win32 - os.remove(lfp) + rmfile(lfp) except OSError: pass self._owns_lock = False -- cgit v1.2.1 From a79cf677744e2c1721fa55f934fa07034bc54b0a Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 12:58:54 +0200 Subject: repo-TCs, #519: FIX config resource leaks + Modify lock/read-config-file code to ensure files closed. + Use `with GitConfigarser()` more systematically in TCs. + Clear any locks left hanging from prev Tcs. + Util: mark lock-files as SHORT_LIVED; save some SSDs... --- git/util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index 87ef38d3..a6c5a100 100644 --- a/git/util.py +++ b/git/util.py @@ -574,7 +574,10 @@ class LockFile(object): (self._file_path, lock_file)) try: - fd = os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if is_win: + flags |= getattr(os, 'O_SHORT_LIVED') + fd = os.open(lock_file, flags, 0) os.close(fd) except OSError as e: raise IOError(str(e)) -- cgit v1.2.1 From 9a521681ff8614beb8e2c566cf3c475baca22169 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 1 Oct 2016 18:20:13 +0200 Subject: io, #519: ALL open() --> with open() + Some cases had restructuring of code. --- git/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index a6c5a100..814cd7f4 100644 --- a/git/util.py +++ b/git/util.py @@ -576,7 +576,7 @@ class LockFile(object): try: flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if is_win: - flags |= getattr(os, 'O_SHORT_LIVED') + flags |= os.O_SHORT_LIVED fd = os.open(lock_file, flags, 0) os.close(fd) except OSError as e: -- cgit v1.2.1 From 51f4a1407ef12405e16f643f5f9d2002b4b52ab9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 2 Oct 2016 11:17:36 -0400 Subject: RF: use @functools.wraps within decorators instead of manual __name__ reassignment @wraps does more and does it right ;) --- git/util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index 814cd7f4..9640a74f 100644 --- a/git/util.py +++ b/git/util.py @@ -14,6 +14,8 @@ import shutil import stat import time +from functools import wraps + from git.compat import is_win from gitdb.util import ( # NOQA make_sha, @@ -50,13 +52,13 @@ def unbare_repo(func): """Methods with this decorator raise InvalidGitRepositoryError if they encounter a bare repository""" + @wraps(func) def wrapper(self, *args, **kwargs): if self.repo.bare: raise InvalidGitRepositoryError("Method '%s' cannot operate on bare repositories" % func.__name__) # END bare method return func(self, *args, **kwargs) # END wrapper - wrapper.__name__ = func.__name__ return wrapper -- cgit v1.2.1 From be44602b633cfb49a472e192f235ba6de0055d38 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 3 Oct 2016 12:25:09 +0200 Subject: hidden win-errs: Let leaking TCs run till end, then hide + Detect code breaking the body of TCs eventually hidden win-errors by raising SkipTest ALAP. + submodule.base.py: import classes from `git.objects` instead of `utils`. + had to ++ ulimit 100->110 for the extra code tested (more leaks :-) + Centralize is_win detection. --- git/util.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index 9640a74f..1fa080a0 100644 --- a/git/util.py +++ b/git/util.py @@ -34,6 +34,7 @@ from .compat import ( PY3 ) from .exc import InvalidGitRepositoryError +from unittest.case import SkipTest # NOTE: Some of the unused imports might be used/imported by others. @@ -71,7 +72,15 @@ def rmtree(path): def onerror(func, path, exc_info): # Is the error an access error ? os.chmod(path, stat.S_IWUSR) - func(path) # Will scream if still not possible to delete. + + try: + func(path) # Will scream if still not possible to delete. + except Exception as ex: + from git.test.lib.helper import HIDE_WINDOWS_KNOWN_ERRORS + if HIDE_WINDOWS_KNOWN_ERRORS: + raise SkipTest("FIXME: fails with: PermissionError\n %s", ex) + else: + raise return shutil.rmtree(path, False, onerror) -- cgit v1.2.1 From a469af892b3e929cbe9d29e414b6fcd59bec246e Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Mon, 3 Oct 2016 23:35:55 +0200 Subject: src: No PyDev warnings + Mark all unused vars and other non-pep8 (PyDev) warnings + test_utils: + enable & fix forgotten IterableList looped path. + unittestize all assertions. + remote: minor fix progress dispatching unknown err-lines --- git/util.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'git/util.py') diff --git a/git/util.py b/git/util.py index 1fa080a0..c96a6b08 100644 --- a/git/util.py +++ b/git/util.py @@ -17,13 +17,13 @@ import time from functools import wraps from git.compat import is_win -from gitdb.util import ( # NOQA +from gitdb.util import ( # NOQA make_sha, - LockedFD, - file_contents_ro, - LazyMixin, - to_hex_sha, - to_bin_sha + LockedFD, # @UnusedImport + file_contents_ro, # @UnusedImport + LazyMixin, # @UnusedImport + to_hex_sha, # @UnusedImport + to_bin_sha # @UnusedImport ) import os.path as osp @@ -251,7 +251,7 @@ class RemoteProgress(object): # END could not get match op_code = 0 - remote, op_name, percent, cur_count, max_count, message = match.groups() + remote, op_name, percent, cur_count, max_count, message = match.groups() # @UnusedVariable # get operation id if op_name == "Counting objects": -- cgit v1.2.1