diff options
Diffstat (limited to 'git')
49 files changed, 708 insertions, 443 deletions
@@ -7,18 +7,24 @@ import os import sys import logging -from util import ( - LazyMixin, - stream_copy -) -from exc import GitCommandError - from subprocess import ( call, Popen, PIPE ) + +from .util import ( + LazyMixin, + stream_copy +) +from .exc import GitCommandError +from git.compat import ( + string_types, + defenc, + PY3 +) + execute_kwargs = ('istream', 'with_keep_cwd', 'with_extended_output', 'with_exceptions', 'as_process', 'output_stream') @@ -114,7 +120,7 @@ class Git(LazyMixin): :raise GitCommandError: if the return status is not 0""" status = self.proc.wait() if status != 0: - raise GitCommandError(self.args, status, self.proc.stderr.read()) + raise GitCommandError(self.args, status, self.proc.stderr.read().decode(defenc)) # END status handling return status # END auto interrupt @@ -314,6 +320,7 @@ class Git(LazyMixin): always be created with a pipe due to issues with subprocess. This merely is a workaround as data will be copied from the output pipe to the given output stream directly. + Judging from the implementation, you shouldn't use this flag ! :param subprocess_kwargs: Keyword arguments to be passed to subprocess.Popen. Please note that @@ -365,15 +372,15 @@ class Git(LazyMixin): # Wait for the process to return status = 0 - stdout_value = '' - stderr_value = '' + stdout_value = b'' + stderr_value = b'' try: if output_stream is None: stdout_value, stderr_value = proc.communicate() # strip trailing "\n" - if stdout_value.endswith("\n"): + if stdout_value.endswith(b"\n"): stdout_value = stdout_value[:-1] - if stderr_value.endswith("\n"): + if stderr_value.endswith(b"\n"): stderr_value = stderr_value[:-1] status = proc.returncode else: @@ -381,7 +388,7 @@ class Git(LazyMixin): stdout_value = output_stream stderr_value = proc.stderr.read() # strip trailing "\n" - if stderr_value.endswith("\n"): + if stderr_value.endswith(b"\n"): stderr_value = stderr_value[:-1] status = proc.wait() # END stdout handling @@ -392,9 +399,10 @@ class Git(LazyMixin): if self.GIT_PYTHON_TRACE == 'full': cmdstr = " ".join(command) if stderr_value: - log.info("%s -> %d; stdout: '%s'; stderr: '%s'", cmdstr, status, stdout_value, stderr_value) + log.info("%s -> %d; stdout: '%s'; stderr: '%s'", + cmdstr, status, stdout_value.decode(defenc), stderr_value.decode(defenc)) elif stdout_value: - log.info("%s -> %d; stdout: '%s'", cmdstr, status, stdout_value) + log.info("%s -> %d; stdout: '%s'", cmdstr, status, stdout_value.decode(defenc)) else: log.info("%s -> %d", cmdstr, status) # END handle debug printing @@ -405,9 +413,12 @@ class Git(LazyMixin): else: raise GitCommandError(command, status, stderr_value) + if isinstance(stdout_value, bytes): # could also be output_stream + stdout_value = stdout_value.decode(defenc) + # Allow access to the command's status code if with_extended_output: - return (status, stdout_value, stderr_value) + return (status, stdout_value, stderr_value.decode(defenc)) else: return stdout_value @@ -433,16 +444,18 @@ class Git(LazyMixin): @classmethod def __unpack_args(cls, arg_list): if not isinstance(arg_list, (list, tuple)): - if isinstance(arg_list, unicode): - return [arg_list.encode('utf-8')] + # This is just required for unicode conversion, as subprocess can't handle it + # However, in any other case, passing strings (usually utf-8 encoded) is totally fine + if not PY3 and isinstance(arg_list, unicode): + return [arg_list.encode(defenc)] return [str(arg_list)] outlist = list() for arg in arg_list: if isinstance(arg_list, (list, tuple)): outlist.extend(cls.__unpack_args(arg)) - elif isinstance(arg_list, unicode): - outlist.append(arg_list.encode('utf-8')) + elif not PY3 and isinstance(arg_list, unicode): + outlist.append(arg_list.encode(defenc)) # END recursion else: outlist.append(str(arg)) @@ -498,8 +511,8 @@ class Git(LazyMixin): # Prepare the argument list opt_args = self.transform_kwargs(**kwargs) - ext_args = self.__unpack_args([a for a in args if a is not None]) + args = opt_args + ext_args def make_call(): @@ -567,14 +580,20 @@ class Git(LazyMixin): raise ValueError("Failed to parse header: %r" % header_line) return (tokens[0], tokens[1], int(tokens[2])) - def __prepare_ref(self, ref): - # required for command to separate refs on stdin - refstr = str(ref) # could be ref-object - if refstr.endswith("\n"): - return refstr - return refstr + "\n" + def _prepare_ref(self, ref): + # required for command to separate refs on stdin, as bytes + refstr = ref + if isinstance(ref, bytes): + # Assume 40 bytes hexsha - bin-to-ascii for some reason returns bytes, not text + refstr = ref.decode('ascii') + elif not isinstance(ref, string_types): + refstr = str(ref) # could be ref-object + + if not refstr.endswith("\n"): + refstr += "\n" + return refstr.encode(defenc) - def __get_persistent_cmd(self, attr_name, cmd_name, *args, **kwargs): + def _get_persistent_cmd(self, attr_name, cmd_name, *args, **kwargs): cur_val = getattr(self, attr_name) if cur_val is not None: return cur_val @@ -587,7 +606,7 @@ class Git(LazyMixin): return cmd def __get_object_header(self, cmd, ref): - cmd.stdin.write(self.__prepare_ref(ref)) + cmd.stdin.write(self._prepare_ref(ref)) cmd.stdin.flush() return self._parse_object_header(cmd.stdout.readline()) @@ -599,7 +618,7 @@ class Git(LazyMixin): once and reuses the command in subsequent calls. :return: (hexsha, type_string, size_as_int)""" - cmd = self.__get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) + cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) return self.__get_object_header(cmd, ref) def get_object_data(self, ref): @@ -616,7 +635,7 @@ class Git(LazyMixin): :return: (hexsha, type_string, size_as_int, stream) :note: This method is not threadsafe, you need one independent Command instance per thread to be safe !""" - cmd = self.__get_persistent_cmd("cat_file_all", "cat_file", batch=True) + cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) hexsha, typename, size = self.__get_object_header(cmd, ref) return (hexsha, typename, size, self.CatFileContentStream(size, cmd.stdout)) diff --git a/git/compat.py b/git/compat.py new file mode 100644 index 00000000..5c330e5b --- /dev/null +++ b/git/compat.py @@ -0,0 +1,66 @@ +#-*-coding:utf-8-*- +# config.py +# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors +# +# This module is part of GitPython and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php +"""utilities to help provide compatibility with python 3""" +# flake8: noqa + +import sys + +from gitdb.utils.compat import ( + PY3, + xrange, + MAXSIZE, + izip, +) + +from gitdb.utils.encoding import ( + string_types, + text_type, + force_bytes, + force_text +) + +defenc = sys.getdefaultencoding() +if PY3: + import io + FileType = io.IOBase + def byte_ord(b): + return b + def bchr(n): + return bytes([n]) + def mviter(d): + return d.values() +else: + FileType = file + # usually, this is just ascii, which might not enough for our encoding needs + # Unless it's set specifically, we override it to be utf-8 + if defenc == 'ascii': + defenc = 'utf-8' + byte_ord = ord + bchr = chr + def mviter(d): + return d.itervalues() + + +def with_metaclass(meta, *bases): + """copied from https://github.com/Byron/bcore/blob/master/src/python/butility/future.py#L15""" + class metaclass(meta): + __call__ = type.__call__ + __init__ = type.__init__ + + def __new__(cls, name, nbases, d): + if nbases is None: + return type.__new__(cls, name, (), d) + # There may be clients who rely on this attribute to be set to a reasonable value, which is why + # we set the __metaclass__ attribute explicitly + if not PY3 and '___metaclass__' not in d: + d['__metaclass__'] = meta + # end + return meta(name, bases, d) + # end + # end metaclass + return metaclass(meta.__name__ + 'Helper', None, {}) + # end handle py2 diff --git a/git/config.py b/git/config.py index 6a85760c..eefab299 100644 --- a/git/config.py +++ b/git/config.py @@ -7,12 +7,23 @@ configuration files""" import re -import ConfigParser as cp +try: + import ConfigParser as cp +except ImportError: + # PY3 + import configparser as cp import inspect import logging +import abc from git.odict import OrderedDict from git.util import LockFile +from git.compat import ( + string_types, + FileType, + defenc, + with_metaclass +) __all__ = ('GitConfigParser', 'SectionConstraint') @@ -20,7 +31,7 @@ __all__ = ('GitConfigParser', 'SectionConstraint') log = logging.getLogger('git.config') -class MetaParserBuilder(type): +class MetaParserBuilder(abc.ABCMeta): """Utlity class wrapping base-class methods into decorators that assure read-only properties""" def __new__(metacls, name, bases, clsdict): @@ -31,7 +42,7 @@ class MetaParserBuilder(type): if kmm in clsdict: mutating_methods = clsdict[kmm] for base in bases: - methods = (t for t in inspect.getmembers(base, inspect.ismethod) if not t[0].startswith("_")) + methods = (t for t in inspect.getmembers(base, inspect.isroutine) if not t[0].startswith("_")) for name, method in methods: if name in clsdict: continue @@ -88,6 +99,12 @@ class SectionConstraint(object): self._config = config self._section_name = section + def __del__(self): + # Yes, for some reason, we have to call it explicitly for it to work in PY3 ! + # Apparently __del__ doesn't get call anymore if refcount becomes 0 + # Ridiculous ... . + self._config.release() + def __getattr__(self, attr): if attr in self._valid_attrs_: return lambda *args, **kwargs: self._call_config(attr, *args, **kwargs) @@ -103,8 +120,12 @@ class SectionConstraint(object): """return: Configparser instance we constrain""" return self._config + def release(self): + """Equivalent to GitConfigParser.release(), which is called on our underlying parser instance""" + return self._config.release() + -class GitConfigParser(cp.RawConfigParser, object): +class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, object)): """Implements specifics required to read git style configuration files. @@ -120,7 +141,6 @@ class GitConfigParser(cp.RawConfigParser, object): :note: The config is case-sensitive even when queried, hence section and option names must match perfectly.""" - __metaclass__ = MetaParserBuilder #{ Configuration # The lock type determines the type of lock to use in new configuration readers. @@ -142,7 +162,6 @@ class GitConfigParser(cp.RawConfigParser, object): # list of RawConfigParser methods able to change the instance _mutating_methods_ = ("add_section", "remove_section", "remove_option", "set") - __slots__ = ("_sections", "_defaults", "_file_or_files", "_read_only", "_is_initialized", '_lock') def __init__(self, file_or_files, read_only=True): """Initialize a configuration reader to read the given file_or_files and to @@ -154,11 +173,11 @@ class GitConfigParser(cp.RawConfigParser, object): :param read_only: If True, the ConfigParser may only read the data , but not change it. If False, only a single file path or file object may be given.""" - super(GitConfigParser, self).__init__() - # initialize base with ordered dictionaries to be sure we write the same - # file back - self._sections = OrderedDict() - self._defaults = OrderedDict() + cp.RawConfigParser.__init__(self, dict_type=OrderedDict) + + # Used in python 3, needs to stay in sync with sections for underlying implementation to work + if not hasattr(self, '_proxies'): + self._proxies = self._dict() self._file_or_files = file_or_files self._read_only = read_only @@ -171,7 +190,7 @@ class GitConfigParser(cp.RawConfigParser, object): "Write-ConfigParsers can operate on a single file only, multiple files have been passed") # END single file check - if not isinstance(file_or_files, basestring): + if not isinstance(file_or_files, string_types): file_or_files = file_or_files.name # END get filename from handle/stream # initialize lock base - we want to write @@ -182,9 +201,16 @@ class GitConfigParser(cp.RawConfigParser, object): def __del__(self): """Write pending changes if required and release locks""" + # NOTE: only consistent in PY2 + self.release() + + def release(self): + """Flush changes and release the configuration write lock. This instance must not be used anymore afterwards. + In Python 3, it's required to explicitly release locks and flush changes, as __del__ is not called + deterministically anymore.""" # checking for the lock here makes sure we do not raise during write() # in case an invalid parser was created who could not get a lock - if self.read_only or not self._lock._has_lock(): + if self.read_only or (self._lock and not self._lock._has_lock()): return try: @@ -192,6 +218,11 @@ class GitConfigParser(cp.RawConfigParser, object): self.write() except IOError: log.error("Exception during destruction of GitConfigParser", exc_info=True) + except ReferenceError: + # This happens in PY3 ... and usually means that some state cannot be written + # as the sections dict cannot be iterated + # Usually when shutting down the interpreter, don'y know how to fix this + pass finally: self._lock._release_lock() @@ -214,7 +245,8 @@ class GitConfigParser(cp.RawConfigParser, object): lineno = 0 e = None # None, or an exception while True: - line = fp.readline() + # we assume to read binary ! + line = fp.readline().decode(defenc) if not line: break lineno = lineno + 1 @@ -234,9 +266,9 @@ class GitConfigParser(cp.RawConfigParser, object): elif sectname == cp.DEFAULTSECT: cursect = self._defaults else: - # THE ONLY LINE WE CHANGED ! - cursect = OrderedDict((('__name__', sectname),)) + cursect = self._dict((('__name__', sectname),)) self._sections[sectname] = cursect + self._proxies[sectname] = None # So sections can't start with a continuation line optname = None # no section header in the file? @@ -287,7 +319,7 @@ class GitConfigParser(cp.RawConfigParser, object): # assume a path if it is not a file-object if not hasattr(file_object, "seek"): try: - fp = open(file_object) + fp = open(file_object, 'rb') close_fp = True except IOError: continue @@ -306,16 +338,17 @@ class GitConfigParser(cp.RawConfigParser, object): """Write an .ini-format representation of the configuration state in git compatible format""" def write_section(name, section_dict): - fp.write("[%s]\n" % name) + fp.write(("[%s]\n" % name).encode(defenc)) for (key, value) in section_dict.items(): if key != "__name__": - fp.write("\t%s = %s\n" % (key, str(value).replace('\n', '\n\t'))) + fp.write(("\t%s = %s\n" % (key, str(value).replace('\n', '\n\t'))).encode(defenc)) # END if key is not __name__ # END section writing if self._defaults: write_section(cp.DEFAULTSECT, self._defaults) - map(lambda t: write_section(t[0], t[1]), self._sections.items()) + for name, value in self._sections.items(): + write_section(name, value) @needs_values def write(self): @@ -329,12 +362,12 @@ class GitConfigParser(cp.RawConfigParser, object): close_fp = False # we have a physical file on disk, so get a lock - if isinstance(fp, (basestring, file)): + if isinstance(fp, string_types + (FileType, )): self._lock._obtain_lock() # END get lock for physical files if not hasattr(fp, "seek"): - fp = open(self._file_or_files, "w") + fp = open(self._file_or_files, "wb") close_fp = True else: fp.seek(0) @@ -363,8 +396,7 @@ class GitConfigParser(cp.RawConfigParser, object): @set_dirty_and_flush_changes def add_section(self, section): """Assures added options will stay in order""" - super(GitConfigParser, self).add_section(section) - self._sections[section] = OrderedDict() + return super(GitConfigParser, self).add_section(section) @property def read_only(self): @@ -387,7 +419,7 @@ class GitConfigParser(cp.RawConfigParser, object): return default raise - types = (long, float) + types = (int, float) for numtype in types: try: val = numtype(valuestr) @@ -408,7 +440,7 @@ class GitConfigParser(cp.RawConfigParser, object): if vl == 'true': return True - if not isinstance(valuestr, basestring): + if not isinstance(valuestr, string_types): raise TypeError("Invalid value type: only int, long, float and str are allowed", valuestr) return valuestr @@ -1,14 +1,8 @@ """Module with our own gitdb implementation - it uses the git command""" -from exc import ( - GitCommandError, - BadObject -) - from gitdb.base import ( OInfo, OStream ) - from gitdb.util import ( bin_to_hex, hex_to_bin @@ -16,6 +10,11 @@ from gitdb.util import ( from gitdb.db import GitDB from gitdb.db import LooseObjectDB +from .exc import ( + GitCommandError, + BadObject +) + __all__ = ('GitCmdObjectDB', 'GitDB') diff --git a/git/diff.py b/git/diff.py index 5325ad6b..3c4e8529 100644 --- a/git/diff.py +++ b/git/diff.py @@ -3,13 +3,15 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php - import re -from objects.blob import Blob -from objects.util import mode_str_to_int from gitdb.util import hex_to_bin +from .objects.blob import Blob +from .objects.util import mode_str_to_int + +from git.compat import defenc + __all__ = ('Diffable', 'DiffIndex', 'Diff') @@ -195,7 +197,7 @@ class Diff(object): """, re.VERBOSE | re.MULTILINE) # can be used for comparisons NULL_HEX_SHA = "0" * 40 - NULL_BIN_SHA = "\0" * 20 + NULL_BIN_SHA = b"\0" * 20 __slots__ = ("a_blob", "b_blob", "a_mode", "b_mode", "new_file", "deleted_file", "rename_from", "rename_to", "diff") @@ -294,7 +296,7 @@ class Diff(object): :param stream: result of 'git diff' as a stream (supporting file protocol) :return: git.DiffIndex """ # for now, we have to bake the stream - text = stream.read() + text = stream.read().decode(defenc) index = DiffIndex() diff_header = cls.re_header.match @@ -323,6 +325,7 @@ class Diff(object): # :100644 100644 687099101... 37c5e30c8... M .gitignore index = DiffIndex() for line in stream: + line = line.decode(defenc) if not line.startswith(":"): continue # END its not a valid diff line @@ -7,6 +7,8 @@ from gitdb.exc import * # NOQA +from git.compat import defenc + class InvalidGitRepositoryError(Exception): @@ -29,10 +31,12 @@ class GitCommandError(Exception): self.command = command def __str__(self): - ret = "'%s' returned exit status %i: %s" % \ - (' '.join(str(i) for i in self.command), self.status, self.stderr) - if self.stdout is not None: - ret += "\nstdout: %s" % self.stdout + ret = "'%s' returned with exit code %i" % \ + (' '.join(str(i) for i in self.command), self.status) + if self.stderr: + ret += "\nstderr: '%s'" % self.stderr.decode(defenc) + if self.stdout: + ret += "\nstdout: '%s'" % self.stdout.decode(defenc) return ret diff --git a/git/index/base.py b/git/index/base.py index fdcfcd12..cc883469 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -8,16 +8,16 @@ import os import sys import subprocess import glob -from cStringIO import StringIO +from io import BytesIO from stat import S_ISLNK -from typ import ( +from .typ import ( BaseIndexEntry, IndexEntry, ) -from util import ( +from .util import ( TemporaryFileSwap, post_clear_cache, default_index, @@ -25,7 +25,6 @@ from util import ( ) import git.diff as diff - from git.exc import ( GitCommandError, CheckoutError @@ -40,6 +39,14 @@ from git.objects import ( ) from git.objects.util import Serializable +from git.compat import ( + izip, + xrange, + string_types, + force_bytes, + defenc, + mviter +) from git.util import ( LazyMixin, @@ -49,7 +56,7 @@ from git.util import ( to_native_path_linux, ) -from fun import ( +from .fun import ( entry_key, write_cache, read_cache, @@ -62,7 +69,6 @@ from fun import ( from gitdb.base import IStream from gitdb.db import MemoryDB from gitdb.util import to_bin_sha -from itertools import izip __all__ = ('IndexFile', 'CheckoutError') @@ -101,7 +107,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): repository's index on demand.""" self.repo = repo self.version = self._VERSION - self._extension_data = '' + self._extension_data = b'' self._file_path = file_path or self._index_path() def _set_cache_(self, attr): @@ -161,9 +167,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): def _entries_sorted(self): """:return: list of entries, in a sorted fashion, first by path, then by stage""" - entries_sorted = self.entries.values() - entries_sorted.sort(key=lambda e: (e.path, e.stage)) # use path/stage as sort key - return entries_sorted + return sorted(self.entries.values(), key=lambda e: (e.path, e.stage)) def _serialize(self, stream, ignore_tree_extension_data=False): entries = self._entries_sorted() @@ -395,7 +399,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): fprogress(filepath, False, item) rval = None try: - proc.stdin.write("%s\n" % filepath) + proc.stdin.write(("%s\n" % filepath).encode(defenc)) except IOError: # pipe broke, usually because some error happend raise fmakeexc() @@ -414,7 +418,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): Function(t) returning True if tuple(stage, Blob) should be yielded by the iterator. A default filter, the BlobFilter, allows you to yield blobs only if they match a given list of paths. """ - for entry in self.entries.itervalues(): + for entry in mviter(self.entries): blob = entry.to_blob(self.repo) blob.size = entry.size output = (entry.stage, blob) @@ -439,7 +443,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): for stage, blob in self.iter_blobs(is_unmerged_blob): path_map.setdefault(blob.path, list()).append((stage, blob)) # END for each unmerged blob - for l in path_map.itervalues(): + for l in mviter(path_map): l.sort() return path_map @@ -542,7 +546,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): entries = list() for item in items: - if isinstance(item, basestring): + if isinstance(item, string_types): paths.append(self._to_relative_path(item)) elif isinstance(item, (Blob, Submodule)): entries.append(BaseIndexEntry.from_blob(item)) @@ -559,7 +563,8 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): st = os.lstat(filepath) # handles non-symlinks as well stream = None if S_ISLNK(st.st_mode): - stream = StringIO(os.readlink(filepath)) + # in PY3, readlink is string, but we need bytes. In PY2, it's just OS encoded bytes, we assume UTF-8 + stream = BytesIO(force_bytes(os.readlink(filepath), encoding='utf-8')) else: stream = open(filepath, 'rb') # END handle stream @@ -753,7 +758,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): for item in items: if isinstance(item, (BaseIndexEntry, (Blob, Submodule))): paths.append(self._to_relative_path(item.path)) - elif isinstance(item, basestring): + elif isinstance(item, string_types): paths.append(self._to_relative_path(item)) else: raise TypeError("Invalid item type: %r" % item) @@ -855,7 +860,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): # parse result - first 0:n/2 lines are 'checking ', the remaining ones # are the 'renaming' ones which we parse - for ln in xrange(len(mvlines) / 2, len(mvlines)): + for ln in xrange(int(len(mvlines) / 2), len(mvlines)): tokens = mvlines[ln].split(' to ') assert len(tokens) == 2, "Too many tokens in %s" % mvlines[ln] @@ -953,6 +958,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): if not stderr: return # line contents: + stderr = stderr.decode(defenc) # git-checkout-index: this already exists failed_files = list() failed_reasons = list() @@ -1001,11 +1007,11 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): proc = self.repo.git.checkout_index(*args, **kwargs) proc.wait() fprogress(None, True, None) - rval_iter = (e.path for e in self.entries.itervalues()) + rval_iter = (e.path for e in mviter(self.entries)) handle_stderr(proc, rval_iter) return rval_iter else: - if isinstance(paths, basestring): + if isinstance(paths, string_types): paths = [paths] # make sure we have our entries loaded before we start checkout_index @@ -1031,7 +1037,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): dir = co_path if not dir.endswith('/'): dir += '/' - for entry in self.entries.itervalues(): + for entry in mviter(self.entries): if entry.path.startswith(dir): p = entry.path self._write_path_to_stdin(proc, p, p, make_exc, @@ -1141,7 +1147,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): # index against anything but None is a reverse diff with the respective # item. Handle existing -R flags properly. Transform strings to the object # so that we can call diff on it - if isinstance(other, basestring): + if isinstance(other, string_types): other = self.repo.rev_parse(other) # END object conversion diff --git a/git/index/fun.py b/git/index/fun.py index eec90519..f0dee961 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -12,7 +12,7 @@ from stat import ( S_IFGITLINK = S_IFLNK | S_IFDIR # a submodule -from cStringIO import StringIO +from io import BytesIO from git.util import IndexFileSHA1Writer from git.exc import UnmergedEntriesError @@ -22,7 +22,7 @@ from git.objects.fun import ( traverse_trees_recursive ) -from typ import ( +from .typ import ( BaseIndexEntry, IndexEntry, CE_NAMEMASK, @@ -30,13 +30,14 @@ from typ import ( ) CE_NAMEMASK_INV = ~CE_NAMEMASK -from util import ( +from .util import ( pack, unpack ) from gitdb.base import IStream from gitdb.typ import str_tree_type +from git.compat import defenc __all__ = ('write_cache', 'read_cache', 'write_tree_from_cache', 'entry_key', 'stat_mode_to_index_mode', 'S_IFGITLINK') @@ -49,7 +50,7 @@ def stat_mode_to_index_mode(mode): return S_IFLNK if S_ISDIR(mode) or S_IFMT(mode) == S_IFGITLINK: # submodules return S_IFGITLINK - return S_IFREG | 0644 | (mode & 0100) # blobs with or without executable bit + return S_IFREG | 0o644 | (mode & 0o100) # blobs with or without executable bit def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1Writer): @@ -72,7 +73,7 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1 # header version = 2 - write("DIRC") + write(b"DIRC") write(pack(">LL", version, len(entries))) # body @@ -86,9 +87,9 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1 flags = plen | (entry[2] & CE_NAMEMASK_INV) # clear possible previous values write(pack(">LLLLLL20sH", entry[6], entry[7], entry[0], entry[8], entry[9], entry[10], entry[1], flags)) - write(path) + write(path.encode(defenc)) real_size = ((tell() - beginoffset + 8) & ~7) - write("\0" * ((beginoffset + real_size) - tell())) + write(b"\0" * ((beginoffset + real_size) - tell())) # END for each entry # write previously cached extensions data @@ -102,7 +103,7 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1 def read_header(stream): """Return tuple(version_long, num_entries) from the given stream""" type_id = stream.read(4) - if type_id != "DIRC": + if type_id != b"DIRC": raise AssertionError("Invalid index file header: %r" % type_id) version, num_entries = unpack(">LL", stream.read(4 * 2)) @@ -142,7 +143,7 @@ def read_cache(stream): (dev, ino, mode, uid, gid, size, sha, flags) = \ unpack(">LLLLLL20sH", read(20 + 4 * 6 + 2)) path_size = flags & CE_NAMEMASK - path = read(path_size) + path = read(path_size).decode(defenc) real_size = ((tell() - beginoffset + 8) & ~7) read((beginoffset + real_size) - tell()) @@ -218,7 +219,7 @@ def write_tree_from_cache(entries, odb, sl, si=0): # END for each entry # finally create the tree - sio = StringIO() + sio = BytesIO() tree_to_stream(tree_items, sio.write) sio.seek(0) diff --git a/git/index/typ.py b/git/index/typ.py index 222252c5..0998ecb0 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -1,15 +1,14 @@ """Module with additional types used by the index""" -from util import ( +from binascii import b2a_hex + +from .util import ( pack, unpack ) +from git.objects import Blob -from binascii import ( - b2a_hex, -) -from git.objects import Blob __all__ = ('BlobFilter', 'BaseIndexEntry', 'IndexEntry') #{ Invariants @@ -76,7 +75,7 @@ class BaseIndexEntry(tuple): @property def hexsha(self): """hex version of our sha""" - return b2a_hex(self[1]) + return b2a_hex(self[1]).decode('ascii') @property def stage(self): diff --git a/git/objects/__init__.py b/git/objects/__init__.py index 70fc52cb..ee642876 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -7,9 +7,10 @@ import inspect from .base import * # Fix import dependency - add IndexObject to the util module, so that it can be # imported by the submodule.base -from .submodule import util -util.IndexObject = IndexObject -util.Object = Object +from .submodule import util as smutil +smutil.IndexObject = IndexObject +smutil.Object = Object +del(smutil) from .submodule.base import * from .submodule.root import * diff --git a/git/objects/base.py b/git/objects/base.py index 20147e57..3f595d9d 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -3,8 +3,8 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php +from .util import get_object_type_by_name from git.util import LazyMixin, join_path_native, stream_copy -from util import get_object_type_by_name from gitdb.util import ( bin_to_hex, basename @@ -21,7 +21,7 @@ class Object(LazyMixin): """Implements an Object which may be Blobs, Trees, Commits and Tags""" NULL_HEX_SHA = '0' * 40 - NULL_BIN_SHA = '\0' * 20 + NULL_BIN_SHA = b'\0' * 20 TYPES = (dbtyp.str_blob_type, dbtyp.str_tree_type, dbtyp.str_commit_type, dbtyp.str_tag_type) __slots__ = ("repo", "binsha", "size") @@ -60,7 +60,7 @@ class Object(LazyMixin): :param sha1: 20 byte binary sha1""" if sha1 == cls.NULL_BIN_SHA: # the NULL binsha is always the root commit - return get_object_type_by_name('commit')(repo, sha1) + return get_object_type_by_name(b'commit')(repo, sha1) # END handle special case oinfo = repo.odb.info(sha1) inst = get_object_type_by_name(oinfo.type)(repo, oinfo.binsha) @@ -94,7 +94,7 @@ class Object(LazyMixin): def __str__(self): """:return: string of our SHA1 as understood by all git commands""" - return bin_to_hex(self.binsha) + return self.hexsha def __repr__(self): """:return: string with pythonic representation of our object""" @@ -103,7 +103,8 @@ class Object(LazyMixin): @property def hexsha(self): """:return: 40 byte hex version of our 20 byte binary sha""" - return bin_to_hex(self.binsha) + # b2a_hex produces bytes + return bin_to_hex(self.binsha).decode('ascii') @property def data_stream(self): diff --git a/git/objects/blob.py b/git/objects/blob.py index b05e5b84..322f6992 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -3,9 +3,8 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php - from mimetypes import guess_type -import base +from . import base __all__ = ('Blob', ) diff --git a/git/objects/commit.py b/git/objects/commit.py index 9c733695..8f93d1b9 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -4,6 +4,8 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php +from gitdb import IStream +from gitdb.util import hex_to_bin from git.util import ( Actor, Iterable, @@ -11,26 +13,24 @@ from git.util import ( finalize_process ) from git.diff import Diffable -from tree import Tree -from gitdb import IStream -from cStringIO import StringIO -import base -from gitdb.util import ( - hex_to_bin -) -from util import ( +from .tree import Tree +from . import base +from .util import ( Traversable, Serializable, parse_date, altz_to_utctz_str, parse_actor_and_date ) +from git.compat import text_type + from time import ( time, altzone ) import os +from io import BytesIO import logging log = logging.getLogger('git.objects.commit') @@ -62,7 +62,7 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): "author", "authored_date", "author_tz_offset", "committer", "committed_date", "committer_tz_offset", "message", "parents", "encoding", "gpgsig") - _id_attribute_ = "binsha" + _id_attribute_ = "hexsha" def __init__(self, repo, binsha, tree=None, author=None, authored_date=None, author_tz_offset=None, committer=None, committed_date=None, committer_tz_offset=None, @@ -133,7 +133,7 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): if attr in Commit.__slots__: # read the data in a chunk, its faster - then provide a file wrapper binsha, typename, self.size, stream = self.repo.odb.stream(self.binsha) - self._deserialize(StringIO(stream.read())) + self._deserialize(BytesIO(stream.read())) else: super(Commit, self)._set_cache_(attr) # END handle attrs @@ -345,7 +345,7 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): committer, committer_time, committer_offset, message, parent_commits, conf_encoding) - stream = StringIO() + stream = BytesIO() new_commit._serialize(stream) streamlen = stream.tell() stream.seek(0) @@ -373,43 +373,36 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): def _serialize(self, stream): write = stream.write - write("tree %s\n" % self.tree) + write(("tree %s\n" % self.tree).encode('ascii')) for p in self.parents: - write("parent %s\n" % p) + write(("parent %s\n" % p).encode('ascii')) a = self.author aname = a.name - if isinstance(aname, unicode): - aname = aname.encode(self.encoding) - # END handle unicode in name - c = self.committer fmt = "%s %s <%s> %s %s\n" - write(fmt % ("author", aname, a.email, - self.authored_date, - altz_to_utctz_str(self.author_tz_offset))) + write((fmt % ("author", aname, a.email, + self.authored_date, + altz_to_utctz_str(self.author_tz_offset))).encode(self.encoding)) # encode committer aname = c.name - if isinstance(aname, unicode): - aname = aname.encode(self.encoding) - # END handle unicode in name - write(fmt % ("committer", aname, c.email, - self.committed_date, - altz_to_utctz_str(self.committer_tz_offset))) + write((fmt % ("committer", aname, c.email, + self.committed_date, + altz_to_utctz_str(self.committer_tz_offset))).encode(self.encoding)) if self.encoding != self.default_encoding: - write("encoding %s\n" % self.encoding) + write(("encoding %s\n" % self.encoding).encode('ascii')) if self.gpgsig: - write("gpgsig") + write(b"gpgsig") for sigline in self.gpgsig.rstrip("\n").split("\n"): - write(" " + sigline + "\n") + write((" " + sigline + "\n").encode('ascii')) - write("\n") + write(b"\n") # write plain bytes, be sure its encoded according to our encoding - if isinstance(self.message, unicode): + if isinstance(self.message, text_type): write(self.message.encode(self.encoding)) else: write(self.message) @@ -426,23 +419,25 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): next_line = None while True: parent_line = readline() - if not parent_line.startswith('parent'): + if not parent_line.startswith(b'parent'): next_line = parent_line break # END abort reading parents - self.parents.append(type(self)(self.repo, hex_to_bin(parent_line.split()[-1]))) + self.parents.append(type(self)(self.repo, hex_to_bin(parent_line.split()[-1].decode('ascii')))) # END for each parent line self.parents = tuple(self.parents) - self.author, self.authored_date, self.author_tz_offset = parse_actor_and_date(next_line) - self.committer, self.committed_date, self.committer_tz_offset = parse_actor_and_date(readline()) + # we don't know actual author encoding before we have parsed it, so keep the lines around + author_line = next_line + committer_line = readline() # we might run into one or more mergetag blocks, skip those for now next_line = readline() - while next_line.startswith('mergetag '): + while next_line.startswith(b'mergetag '): next_line = readline() while next_line.startswith(' '): next_line = readline() + # end skip mergetags # now we can have the encoding line, or an empty line followed by the optional # message. @@ -451,39 +446,40 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): # read headers enc = next_line buf = enc.strip() - while buf != "": - if buf[0:10] == "encoding ": - self.encoding = buf[buf.find(' ') + 1:] - elif buf[0:7] == "gpgsig ": - sig = buf[buf.find(' ') + 1:] + "\n" + while buf: + if buf[0:10] == b"encoding ": + self.encoding = buf[buf.find(' ') + 1:].decode('ascii') + elif buf[0:7] == b"gpgsig ": + sig = buf[buf.find(b' ') + 1:] + b"\n" is_next_header = False while True: sigbuf = readline() - if sigbuf == "": + if not sigbuf: break - if sigbuf[0:1] != " ": + if sigbuf[0:1] != b" ": buf = sigbuf.strip() is_next_header = True break sig += sigbuf[1:] - self.gpgsig = sig.rstrip("\n") + # end read all signature + self.gpgsig = sig.rstrip(b"\n").decode('ascii') if is_next_header: continue buf = readline().strip() - # decode the authors name + try: - self.author.name = self.author.name.decode(self.encoding) + self.author, self.authored_date, self.author_tz_offset = \ + parse_actor_and_date(author_line.decode(self.encoding)) except UnicodeDecodeError: - log.error("Failed to decode author name '%s' using encoding %s", self.author.name, self.encoding, + log.error("Failed to decode author line '%s' using encoding %s", author_line, self.encoding, exc_info=True) - # END handle author's encoding - # decode committer name try: - self.committer.name = self.committer.name.decode(self.encoding) + self.committer, self.committed_date, self.committer_tz_offset = \ + parse_actor_and_date(committer_line.decode(self.encoding)) except UnicodeDecodeError: - log.error("Failed to decode committer name '%s' using encoding %s", self.committer.name, self.encoding, + log.error("Failed to decode committer line '%s' using encoding %s", committer_line, self.encoding, exc_info=True) # END handle author's encoding @@ -495,6 +491,7 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): except UnicodeDecodeError: log.error("Failed to decode message '%s' using encoding %s", self.message, self.encoding, exc_info=True) # END exception handling + return self #} END serializable implementation diff --git a/git/objects/fun.py b/git/objects/fun.py index 416a52e6..c04f80b5 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -1,5 +1,12 @@ """Module with functions which are supposed to be as fast as possible""" from stat import S_ISDIR +from git.compat import ( + byte_ord, + defenc, + xrange, + text_type, + bchr +) __all__ = ('tree_to_stream', 'tree_entries_from_data', 'traverse_trees_recursive', 'traverse_tree_recursive') @@ -13,13 +20,13 @@ def tree_to_stream(entries, write): bit_mask = 7 # 3 bits set for binsha, mode, name in entries: - mode_str = '' + mode_str = b'' for i in xrange(6): - mode_str = chr(((mode >> (i * 3)) & bit_mask) + ord_zero) + mode_str + mode_str = bchr(((mode >> (i * 3)) & bit_mask) + ord_zero) + mode_str # END for each 8 octal value # git slices away the first octal if its zero - if mode_str[0] == '0': + if byte_ord(mode_str[0]) == ord_zero: mode_str = mode_str[1:] # END save a byte @@ -28,17 +35,18 @@ def tree_to_stream(entries, write): # hence we must convert to an utf8 string for it to work properly. # According to my tests, this is exactly what git does, that is it just # takes the input literally, which appears to be utf8 on linux. - if isinstance(name, unicode): - name = name.encode("utf8") - write("%s %s\0%s" % (mode_str, name, binsha)) + if isinstance(name, text_type): + name = name.encode(defenc) + write(b''.join((mode_str, b' ', name, b'\0', binsha))) # END for each item def tree_entries_from_data(data): """Reads the binary representation of a tree and returns tuples of Tree items - :param data: data block with tree data + :param data: data block with tree data (as bytes) :return: list(tuple(binsha, mode, tree_relative_path), ...)""" ord_zero = ord('0') + space_ord = ord(' ') len_data = len(data) i = 0 out = list() @@ -48,10 +56,10 @@ def tree_entries_from_data(data): # read mode # Some git versions truncate the leading 0, some don't # The type will be extracted from the mode later - while data[i] != ' ': + while byte_ord(data[i]) != space_ord: # move existing mode integer up one level being 3 bits # and add the actual ordinal value of the character - mode = (mode << 3) + (ord(data[i]) - ord_zero) + mode = (mode << 3) + (byte_ord(data[i]) - ord_zero) i += 1 # END while reading mode @@ -61,7 +69,7 @@ def tree_entries_from_data(data): # parse name, it is NULL separated ns = i - while data[i] != '\0': + while byte_ord(data[i]) != 0: i += 1 # END while not reached NULL @@ -69,12 +77,9 @@ def tree_entries_from_data(data): # Only use the respective unicode object if the byte stream was encoded name = data[ns:i] try: - name_enc = name.decode("utf-8") + name = name.decode(defenc) except UnicodeDecodeError: pass - else: - if len(name) > len(name_enc): - name = name_enc # END handle encoding # byte is NULL, get next 20 diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index d6f8982b..0ec6f656 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1,5 +1,5 @@ -import util -from util import ( +from . import util +from .util import ( mkhead, sm_name, sm_section, @@ -8,7 +8,7 @@ from util import ( find_first_remote_branch ) from git.objects.util import Traversable -from StringIO import StringIO # need a dict to set bloody .name field +from io import BytesIO # need a dict to set bloody .name field from git.util import ( Iterable, join_path_native, @@ -17,11 +17,15 @@ from git.util import ( rmtree ) -from git.config import SectionConstraint +from git.config import ( + SectionConstraint, + cp +) from git.exc import ( InvalidGitRepositoryError, NoSuchPathError ) +from git.compat import string_types import stat import git @@ -93,7 +97,7 @@ class Submodule(util.IndexObject, Iterable, Traversable): if url is not None: self._url = url if branch_path is not None: - assert isinstance(branch_path, basestring) + assert isinstance(branch_path, string_types) self._branch_path = branch_path if name is not None: self._name = name @@ -186,8 +190,8 @@ class Submodule(util.IndexObject, Iterable, Traversable): @classmethod def _sio_modules(cls, parent_commit): - """:return: Configuration file as StringIO - we only access it through the respective blob's data""" - sio = StringIO(parent_commit.tree[cls.k_modules_file].data_stream.read()) + """:return: Configuration file as BytesIO - we only access it through the respective blob's data""" + sio = BytesIO(parent_commit.tree[cls.k_modules_file].data_stream.read()) sio.name = cls.k_modules_file return sio @@ -301,6 +305,7 @@ class Submodule(util.IndexObject, Iterable, Traversable): writer.set_value(cls.k_head_option, br.path) sm._branch_path = br.path # END handle path + writer.release() del(writer) # we deliberatly assume that our head matches our index ! @@ -418,7 +423,9 @@ class Submodule(util.IndexObject, Iterable, Traversable): # the default implementation will be offended and not update the repository # Maybe this is a good way to assure it doesn't get into our way, but # we want to stay backwards compatible too ... . Its so redundant ! - self.repo.config_writer().set_value(sm_section(self.name), 'url', self.url) + writer = self.repo.config_writer() + writer.set_value(sm_section(self.name), 'url', self.url) + writer.release() # END handle dry_run # END handle initalization @@ -575,6 +582,7 @@ class Submodule(util.IndexObject, Iterable, Traversable): writer = self.config_writer(index=index) # auto-write writer.set_value('path', module_path) self.path = module_path + writer.release() del(writer) # END handle configuration flag except Exception: @@ -699,8 +707,12 @@ class Submodule(util.IndexObject, Iterable, Traversable): # now git config - need the config intact, otherwise we can't query # inforamtion anymore - self.repo.config_writer().remove_section(sm_section(self.name)) - self.config_writer().remove_section() + writer = self.repo.config_writer() + writer.remove_section(sm_section(self.name)) + writer.release() + writer = self.config_writer() + writer.remove_section() + writer.release() # END delete configuration # void our data not to delay invalid access @@ -799,14 +811,18 @@ class Submodule(util.IndexObject, Iterable, Traversable): """ :return: True if the submodule exists, False otherwise. Please note that a submodule may exist (in the .gitmodules file) even though its module - doesn't exist""" + doesn't exist on disk""" # keep attributes for later, and restore them if we have no valid data # this way we do not actually alter the state of the object loc = locals() for attr in self._cache_attrs: - if hasattr(self, attr): - loc[attr] = getattr(self, attr) - # END if we have the attribute cache + try: + if hasattr(self, attr): + loc[attr] = getattr(self, attr) + # END if we have the attribute cache + except cp.NoSectionError: + # on PY3, this can happen apparently ... don't know why this doesn't happen on PY2 + pass # END for each attr self._clear_cache() diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index 708749c7..8c9afff1 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -1,5 +1,8 @@ -from base import Submodule, UpdateProgress -from util import ( +from .base import ( + Submodule, + UpdateProgress +) +from .util import ( find_first_remote_branch ) from git.exc import InvalidGitRepositoryError diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index 01bd03b3..5604dec7 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -1,7 +1,7 @@ import git from git.exc import InvalidGitRepositoryError from git.config import GitConfigParser -from StringIO import StringIO +from io import BytesIO import weakref __all__ = ('sm_section', 'sm_name', 'mkhead', 'unbare_repo', 'find_first_remote_branch', @@ -83,7 +83,7 @@ class SubmoduleConfigParser(GitConfigParser): """Flush changes in our configuration file to the index""" assert self._smref is not None # should always have a file here - assert not isinstance(self._file_or_files, StringIO) + assert not isinstance(self._file_or_files, BytesIO) sm = self._smref() if sm is not None: diff --git a/git/objects/tag.py b/git/objects/tag.py index 3c379579..c8684447 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -4,12 +4,13 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php """ Module containing all object based types. """ -import base -from gitdb.util import hex_to_bin -from util import ( +from . import base +from .util import ( get_object_type_by_name, parse_actor_and_date ) +from gitdb.util import hex_to_bin +from git.compat import defenc __all__ = ("TagObject", ) @@ -52,11 +53,12 @@ class TagObject(base.Object): """Cache all our attributes at once""" if attr in TagObject.__slots__: ostream = self.repo.odb.stream(self.binsha) - lines = ostream.read().splitlines() + lines = ostream.read().decode(defenc).splitlines() obj, hexsha = lines[0].split(" ") # object <hexsha> type_token, type_name = lines[1].split(" ") # type <type_name> - self.object = get_object_type_by_name(type_name)(self.repo, hex_to_bin(hexsha)) + self.object = \ + get_object_type_by_name(type_name.encode('ascii'))(self.repo, hex_to_bin(hexsha)) self.tag = lines[2][4:] # tag <tag name> diff --git a/git/objects/tree.py b/git/objects/tree.py index c77e6056..f9bee01e 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -3,22 +3,21 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -import util -from base import IndexObject from git.util import join_path -from blob import Blob -from submodule.base import Submodule import git.diff as diff +from gitdb.util import to_bin_sha -from fun import ( +from . import util +from .base import IndexObject +from .blob import Blob +from .submodule.base import Submodule +from git.compat import string_types + +from .fun import ( tree_entries_from_data, tree_to_stream ) -from gitdb.util import ( - to_bin_sha, -) - __all__ = ("TreeModifier", "Tree") @@ -160,7 +159,7 @@ class Tree(IndexObject, diff.Diffable, util.Traversable, util.Serializable): raise TypeError("Unknown mode %o found in tree data for path '%s'" % (mode, path)) # END for each item - def __div__(self, file): + def join(self, file): """Find the named object in this tree's contents :return: ``git.Blob`` or ``git.Tree`` or ``git.Submodule`` @@ -193,6 +192,14 @@ class Tree(IndexObject, diff.Diffable, util.Traversable, util.Serializable): raise KeyError(msg % file) # END handle long paths + def __div__(self, file): + """For PY2 only""" + return self.join(file) + + def __truediv__(self, file): + """For PY3 only""" + return self.join(file) + @property def trees(self): """:return: list(Tree, ...) list of trees directly below this tree""" @@ -234,9 +241,9 @@ class Tree(IndexObject, diff.Diffable, util.Traversable, util.Serializable): info = self._cache[item] return self._map_id_to_type[info[1] >> 12](self.repo, info[0], info[1], join_path(self.path, info[2])) - if isinstance(item, basestring): + if isinstance(item, string_types): # compatability - return self.__div__(item) + return self.join(item) # END index is basestring raise TypeError("Invalid index type: %r" % item) diff --git a/git/objects/util.py b/git/objects/util.py index fdf9622b..cefef862 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -46,17 +46,17 @@ def get_object_type_by_name(object_type_name): :param object_type_name: Member of TYPES :raise ValueError: In case object_type_name is unknown""" - if object_type_name == "commit": - import commit + if object_type_name == b"commit": + from . import commit return commit.Commit - elif object_type_name == "tag": - import tag + elif object_type_name == b"tag": + from . import tag return tag.TagObject - elif object_type_name == "blob": - import blob + elif object_type_name == b"blob": + from . import blob return blob.Blob - elif object_type_name == "tree": - import tree + elif object_type_name == b"tree": + from . import tree return tree.Tree else: raise ValueError("Cannot handle unknown object type: %s" % object_type_name) diff --git a/git/refs/head.py b/git/refs/head.py index 25c994a3..750d15b6 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -1,12 +1,10 @@ -from symbolic import SymbolicReference -from reference import Reference - from git.config import SectionConstraint - from git.util import join_path - from git.exc import GitCommandError +from .symbolic import SymbolicReference +from .reference import Reference + __all__ = ["HEAD", "Head"] @@ -150,6 +148,7 @@ class Head(Reference): writer.set_value(self.k_config_remote, remote_reference.remote_name) writer.set_value(self.k_config_remote_ref, Head.to_full_path(remote_reference.remote_head)) # END handle ref value + writer.release() return self diff --git a/git/refs/log.py b/git/refs/log.py index 07465e6f..7708dd73 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -17,6 +17,11 @@ from git.objects.util import ( Serializable, altz_to_utctz_str, ) +from git.compat import ( + xrange, + string_types, + defenc +) import time import re @@ -34,9 +39,8 @@ class RefLogEntry(tuple): def __repr__(self): """Representation of ourselves in git reflog format""" act = self.actor - name = act.name.encode('utf-8') time = self.time - return self._fmt % (self.oldhexsha, self.newhexsha, name, act.email, + return self._fmt % (self.oldhexsha, self.newhexsha, act.name, act.email, time[0], altz_to_utctz_str(time[1]), self.message) @property @@ -78,7 +82,7 @@ class RefLogEntry(tuple): @classmethod def from_line(cls, line): """:return: New RefLogEntry instance from the given revlog line. - :param line: line without trailing newline + :param line: line bytes without trailing newline :raise ValueError: If line could not be parsed""" fields = line.split('\t', 1) if len(fields) == 1: @@ -89,6 +93,7 @@ class RefLogEntry(tuple): raise ValueError("Line must have up to two TAB-separated fields." " Got %s" % repr(line)) # END handle first split + oldhexsha = info[:40] newhexsha = info[41:81] for hexsha in (oldhexsha, newhexsha): @@ -174,7 +179,7 @@ class RefLog(list, Serializable): :param stream: file-like object containing the revlog in its native format or basestring instance pointing to a file to read""" new_entry = RefLogEntry.from_line - if isinstance(stream, basestring): + if isinstance(stream, string_types): stream = file_contents_ro_filepath(stream) # END handle stream type while True: @@ -253,15 +258,18 @@ class RefLog(list, Serializable): # END handle sha type assure_directory_exists(filepath, is_file=True) committer = isinstance(config_reader, Actor) and config_reader or Actor.committer(config_reader) - entry = RefLogEntry( - (bin_to_hex(oldbinsha), bin_to_hex(newbinsha), committer, (int(time.time()), time.altzone), message)) + entry = RefLogEntry(( + bin_to_hex(oldbinsha).decode('ascii'), + bin_to_hex(newbinsha).decode('ascii'), + committer, (int(time.time()), time.altzone), message + )) lf = LockFile(filepath) lf._obtain_lock_or_raise() - fd = open(filepath, 'a') + fd = open(filepath, 'ab') try: - fd.write(repr(entry)) + fd.write(repr(entry).encode(defenc)) finally: fd.close() lf._release_lock() @@ -286,7 +294,7 @@ class RefLog(list, Serializable): # write all entries for e in self: - write(repr(e)) + write(repr(e).encode(defenc)) # END for each entry def _deserialize(self, stream): diff --git a/git/refs/reference.py b/git/refs/reference.py index b07ac0cd..8741ebb9 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -1,8 +1,9 @@ -from symbolic import SymbolicReference from git.util import ( LazyMixin, Iterable, ) +from .symbolic import SymbolicReference + __all__ = ["Reference"] diff --git a/git/refs/remote.py b/git/refs/remote.py index e3827ad9..b692e6df 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -1,7 +1,8 @@ -from head import Head from git.util import join_path from gitdb.util import join +from .head import Head + import os diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index e0f5531a..cbb129d4 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -1,4 +1,5 @@ import os + from git.objects import Object, Commit from git.util import ( join_path, @@ -18,8 +19,12 @@ from gitdb.util import ( hex_to_bin, LockedFD ) +from git.compat import ( + string_types, + defenc +) -from log import RefLog +from .log import RefLog __all__ = ["SymbolicReference"] @@ -77,10 +82,10 @@ class SymbolicReference(object): @classmethod def _iter_packed_refs(cls, repo): - """Returns an iterator yielding pairs of sha1/path pairs for the corresponding refs. + """Returns an iterator yielding pairs of sha1/path pairs (as bytes) for the corresponding refs. :note: The packed refs file will be kept open as long as we iterate""" try: - fp = open(cls._get_packed_refs_path(repo), 'rb') + fp = open(cls._get_packed_refs_path(repo), 'rt') for line in fp: line = line.strip() if not line: @@ -121,12 +126,12 @@ class SymbolicReference(object): @classmethod def _get_ref_info(cls, repo, ref_path): - """Return: (sha, target_ref_path) if available, the sha the file at + """Return: (str(sha), str(target_ref_path)) if available, the sha the file at rela_path points to, or None. target_ref_path is the reference we point to, or None""" tokens = None try: - fp = open(join(repo.git_dir, ref_path), 'r') + fp = open(join(repo.git_dir, ref_path), 'rt') value = fp.read().rstrip() fp.close() # Don't only split on spaces, but on whitespace, which allows to parse lines like @@ -139,7 +144,8 @@ class SymbolicReference(object): for sha, path in cls._iter_packed_refs(repo): if path != ref_path: continue - tokens = (sha, path) + # sha will be used + tokens = sha, path break # END for each packed ref # END handle packed refs @@ -273,7 +279,7 @@ class SymbolicReference(object): elif isinstance(ref, Object): obj = ref write_value = ref.hexsha - elif isinstance(ref, basestring): + elif isinstance(ref, string_types): try: obj = self.repo.rev_parse(ref + "^{}") # optionally deref tags write_value = obj.hexsha @@ -303,7 +309,7 @@ class SymbolicReference(object): lfd = LockedFD(fpath) fd = lfd.open(write=True, stream=True) - fd.write(write_value) + fd.write(write_value.encode('ascii')) lfd.commit() # Adjust the reflog @@ -424,6 +430,7 @@ class SymbolicReference(object): # in the line # If we deleted the last line and this one is a tag-reference object, # we drop it as well + line = line.decode(defenc) if (line.startswith('#') or full_ref_path not in line) and \ (not dropped_last_line or dropped_last_line and not line.startswith('^')): new_lines.append(line) @@ -441,7 +448,7 @@ class SymbolicReference(object): if made_change: # write-binary is required, otherwise windows will # open the file in text mode and change LF to CRLF ! - open(pack_file_path, 'wb').writelines(new_lines) + open(pack_file_path, 'wb').writelines(l.encode(defenc) for l in new_lines) # END write out file # END open exception handling # END handle deletion @@ -473,7 +480,7 @@ class SymbolicReference(object): target_data = target.path if not resolve: target_data = "ref: " + target_data - existing_data = open(abs_ref_path, 'rb').read().strip() + existing_data = open(abs_ref_path, 'rb').read().decode(defenc).strip() if existing_data != target_data: raise OSError("Reference at %r does already exist, pointing to %r, requested was %r" % (full_ref_path, existing_data, target_data)) diff --git a/git/refs/tag.py b/git/refs/tag.py index 6509c891..3334e53c 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -1,4 +1,4 @@ -from reference import Reference +from .reference import Reference __all__ = ["TagReference", "Tag"] diff --git a/git/remote.py b/git/remote.py index 44b7ffaa..484bc031 100644 --- a/git/remote.py +++ b/git/remote.py @@ -5,33 +5,35 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php # Module implementing a remote object allowing easy access to git remotes +import re +import os -from exc import GitCommandError -from ConfigParser import NoOptionError -from config import SectionConstraint - -from git.util import ( - LazyMixin, - Iterable, - IterableList, - RemoteProgress +from .exc import GitCommandError +from .config import ( + SectionConstraint, + cp, ) - -from refs import ( +from .refs import ( Reference, RemoteReference, SymbolicReference, TagReference ) + +from git.util import ( + LazyMixin, + Iterable, + IterableList, + RemoteProgress +) from git.util import ( join_path, finalize_process ) from gitdb.util import join +from git.compat import defenc -import re -import os __all__ = ('RemoteProgress', 'PushInfo', 'FetchInfo', 'Remote') @@ -45,16 +47,16 @@ def digest_process_messages(fh, progress): :param fh: File handle to read from :return: list(line, ...) list of lines without linebreaks that did not contain progress information""" - line_so_far = '' + line_so_far = b'' dropped_lines = list() while True: - char = fh.read(1) + char = fh.read(1) # reads individual single byte strings if not char: break - if char in ('\r', '\n') and line_so_far: - dropped_lines.extend(progress._parse_progress_line(line_so_far)) - line_so_far = '' + if char in (b'\r', b'\n') and line_so_far: + dropped_lines.extend(progress._parse_progress_line(line_so_far.decode(defenc))) + line_so_far = b'' else: line_so_far += char # END process parsed line @@ -136,7 +138,7 @@ class PushInfo(object): @classmethod def _from_line(cls, remote, line): """Create a new PushInfo instance as parsed from line which is expected to be like - refs/heads/master:refs/heads/master 05d2687..1d0568e""" + refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes""" control_character, from_to, summary = line.split('\t', 3) flags = 0 @@ -390,7 +392,7 @@ class Remote(LazyMixin, Iterable): # even though a slot of the same name exists try: return self._config_reader.get(attr) - except NoOptionError: + except cp.NoOptionError: return super(Remote, self).__getattr__(attr) # END handle exception @@ -520,6 +522,7 @@ class Remote(LazyMixin, Iterable): def _get_fetch_info_from_stderr(self, proc, progress): # skip first line as it is some remote info we are not interested in + # TODO: Use poll() to process stdout and stderr at same time output = IterableList('name') # lines which are no progress are fetch info lines @@ -542,8 +545,8 @@ class Remote(LazyMixin, Iterable): # END for each line # read head information - fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'r') - fetch_head_info = fp.readlines() + fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') + fetch_head_info = [l.decode(defenc) for l in fp.readlines()] fp.close() # NOTE: We assume to fetch at least enough progress lines to allow matching each fetch head line with it. @@ -560,10 +563,12 @@ class Remote(LazyMixin, Iterable): # we hope stdout can hold all the data, it should ... # read the lines manually as it will use carriage returns between the messages # to override the previous one. This is why we read the bytes manually + # TODO: poll() on file descriptors to know what to read next, process streams concurrently digest_process_messages(proc.stderr, progress) output = IterableList('name') for line in proc.stdout.readlines(): + line = line.decode(defenc) try: output.append(PushInfo._from_line(self, line)) except ValueError: @@ -571,7 +576,6 @@ class Remote(LazyMixin, Iterable): pass # END exception handling # END for each line - finalize_process(proc) return output diff --git a/git/repo/base.py b/git/repo/base.py index dcf98152..2a63492b 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -40,13 +40,17 @@ from gitdb.util import ( hex_to_bin ) -from fun import ( +from .fun import ( rev_parse, is_git_dir, find_git_dir, read_gitfile, touch, ) +from git.compat import ( + text_type, + defenc +) import os import sys @@ -176,11 +180,11 @@ class Repo(object): # Description property def _get_description(self): filename = join(self.git_dir, 'description') - return file(filename).read().rstrip() + return open(filename, 'rb').read().rstrip().decode(defenc) def _set_description(self, descr): filename = join(self.git_dir, 'description') - file(filename, 'w').write(descr + '\n') + open(filename, 'wb').write((descr + '\n').encode(defenc)) description = property(_get_description, _set_description, doc="the project's description") @@ -389,7 +393,7 @@ class Repo(object): if rev is None: return self.head.commit else: - return self.rev_parse(unicode(rev) + "^0") + return self.rev_parse(text_type(rev) + "^0") def iter_trees(self, *args, **kwargs): """:return: Iterator yielding Tree objects @@ -412,7 +416,7 @@ class Repo(object): if rev is None: return self.head.commit.tree else: - return self.rev_parse(unicode(rev) + "^{tree}") + return self.rev_parse(text_type(rev) + "^{tree}") def iter_commits(self, rev=None, paths='', **kwargs): """A list of Commit objects representing the history of a given ref/commit @@ -463,8 +467,8 @@ class Repo(object): if os.path.exists(alternates_path): try: - f = open(alternates_path) - alts = f.read() + f = open(alternates_path, 'rb') + alts = f.read().decode(defenc) finally: f.close() return alts.strip().splitlines() @@ -488,8 +492,8 @@ class Repo(object): os.remove(alternates_path) else: try: - f = open(alternates_path, 'w') - f.write("\n".join(alts)) + f = open(alternates_path, 'wb') + f.write("\n".join(alts).encode(defenc)) finally: f.close() # END file handling @@ -547,6 +551,7 @@ class Repo(object): prefix = "?? " untracked_files = list() for line in proc.stdout: + line = line.decode(defenc) if not line.startswith(prefix): continue filename = line[len(prefix):].rstrip('\n') @@ -728,7 +733,10 @@ class Repo(object): # sure repo = cls(os.path.abspath(path), odbt=odbt) if repo.remotes: - repo.remotes[0].config_writer.set_value('url', repo.remotes[0].url.replace("\\\\", "\\").replace("\\", "/")) + writer = repo.remotes[0].config_writer + writer.set_value('url', repo.remotes[0].url.replace("\\\\", "\\").replace("\\", "/")) + # PY3: be sure cleanup is performed and lock is released + writer.release() # END handle remote repo return repo @@ -760,7 +768,7 @@ class Repo(object): def archive(self, ostream, treeish=None, prefix=None, **kwargs): """Archive the tree at the given revision. - :parm ostream: file compatible stream object to which the archive will be written + :parm ostream: file compatible stream object to which the archive will be written as bytes :parm treeish: is the treeish name/id, defaults to active branch :parm prefix: is the optional prefix to prepend to each filename in the archive :parm kwargs: diff --git a/git/repo/fun.py b/git/repo/fun.py index b8905517..233666c9 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -1,5 +1,7 @@ """Package with general repository related functions""" import os +from string import digits + from gitdb.exc import BadObject from git.refs import SymbolicReference from git.objects import Object @@ -11,14 +13,15 @@ from gitdb.util import ( hex_to_bin, bin_to_hex ) -from string import digits +from git.compat import xrange + __all__ = ('rev_parse', 'is_git_dir', 'touch', 'read_gitfile', 'find_git_dir', 'name_to_object', 'short_to_long', 'deref_tag', 'to_commit') def touch(filename): - fp = open(filename, "a") + fp = open(filename, "ab") fp.close() @@ -147,7 +150,7 @@ def to_commit(obj): def rev_parse(repo, rev): """ :return: Object at the given revision, either Commit, Tag, Tree or Blob - :param rev: git-rev-parse compatible revision specification, please see + :param rev: git-rev-parse compatible revision specification as string, please see http://www.kernel.org/pub/software/scm/git/docs/git-rev-parse.html for details :note: Currently there is no access to the rev-log, rev-specs may only contain diff --git a/git/test/fixtures/git_config_global b/git/test/fixtures/git_config_global index 1a55397f..56fbd3b3 100644 --- a/git/test/fixtures/git_config_global +++ b/git/test/fixtures/git_config_global @@ -1,3 +1,4 @@ +# just a comment [alias] st = status ci = commit diff --git a/git/test/lib/asserts.py b/git/test/lib/asserts.py index 0f2fd99a..60a888b3 100644 --- a/git/test/lib/asserts.py +++ b/git/test/lib/asserts.py @@ -7,13 +7,6 @@ import re import stat -__all__ = ['assert_instance_of', 'assert_not_instance_of', - 'assert_none', 'assert_not_none', - 'assert_match', 'assert_not_match', 'assert_mode_644', - 'assert_mode_755', - 'assert_equal', 'assert_not_equal', 'assert_raises', 'patch', 'raises', - 'assert_true', 'assert_false'] - from nose.tools import ( assert_equal, assert_not_equal, @@ -23,9 +16,14 @@ from nose.tools import ( assert_false ) -from mock import ( - patch -) +from mock import patch + +__all__ = ['assert_instance_of', 'assert_not_instance_of', + 'assert_none', 'assert_not_none', + 'assert_match', 'assert_not_match', 'assert_mode_644', + 'assert_mode_755', + 'assert_equal', 'assert_not_equal', 'assert_raises', 'patch', 'raises', + 'assert_true', 'assert_false'] def assert_instance_of(expected, actual, msg=None): diff --git a/git/test/lib/helper.py b/git/test/lib/helper.py index 9c935ce0..bd679512 100644 --- a/git/test/lib/helper.py +++ b/git/test/lib/helper.py @@ -6,12 +6,14 @@ from __future__ import print_function import os import sys -from git import Repo, Remote, GitCommandError, Git from unittest import TestCase import time import tempfile import shutil -import cStringIO +import io + +from git import Repo, Remote, GitCommandError, Git +from git.compat import string_types GIT_REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) @@ -46,8 +48,8 @@ class StringProcessAdapter(object): Its tailored to work with the test system only""" def __init__(self, input_string): - self.stdout = cStringIO.StringIO(input_string) - self.stderr = cStringIO.StringIO() + self.stdout = io.BytesIO(input_string) + self.stderr = io.BytesIO() def wait(self): return 0 @@ -89,7 +91,7 @@ def with_rw_repo(working_tree_ref, bare=False): To make working with relative paths easier, the cwd will be set to the working dir of the repository. """ - assert isinstance(working_tree_ref, basestring), "Decorator requires ref name for working tree checkout" + assert isinstance(working_tree_ref, string_types), "Decorator requires ref name for working tree checkout" def argument_passer(func): def repo_creator(self): @@ -152,7 +154,7 @@ def with_rw_and_rw_remote_repo(working_tree_ref): See working dir info in with_rw_repo :note: We attempt to launch our own invocation of git-daemon, which will be shutdown at the end of the test. """ - assert isinstance(working_tree_ref, basestring), "Decorator requires ref name for working tree checkout" + assert isinstance(working_tree_ref, string_types), "Decorator requires ref name for working tree checkout" def argument_passer(func): def remote_repo_creator(self): @@ -177,6 +179,7 @@ def with_rw_and_rw_remote_repo(working_tree_ref): pass crw.set(section, "receivepack", True) # release lock + crw.release() del(crw) # initialize the remote - first do it as local remote and pull, then @@ -191,7 +194,7 @@ def with_rw_and_rw_remote_repo(working_tree_ref): temp_dir = os.path.dirname(_mktemp()) # On windows, this will fail ... we deal with failures anyway and default to telling the user to do it try: - gd = Git().daemon(temp_dir, as_process=True) + gd = Git().daemon(temp_dir, enable='receive-pack', as_process=True) # yes, I know ... fortunately, this is always going to work if sleep time is just large enough time.sleep(0.5) except Exception: @@ -213,7 +216,8 @@ def with_rw_and_rw_remote_repo(working_tree_ref): msg += 'Otherwise, run: git-daemon "%s"' % temp_dir raise AssertionError(msg) else: - msg = 'Please start a git-daemon to run this test, execute: git-daemon "%s"' % temp_dir + msg = 'Please start a git-daemon to run this test, execute: git daemon --enable=receive-pack "%s"' + msg %= temp_dir raise AssertionError(msg) # END make assertion # END catch ls remote error @@ -225,7 +229,8 @@ def with_rw_and_rw_remote_repo(working_tree_ref): return func(self, rw_repo, rw_remote_repo) finally: # gd.proc.kill() ... no idea why that doesn't work - os.kill(gd.proc.pid, 15) + if gd is not None: + os.kill(gd.proc.pid, 15) os.chdir(prev_cwd) rw_repo.git.clear_cache() @@ -233,7 +238,8 @@ def with_rw_and_rw_remote_repo(working_tree_ref): shutil.rmtree(repo_dir, onerror=_rmtree_onerror) shutil.rmtree(remote_repo_dir, onerror=_rmtree_onerror) - gd.proc.wait() + if gd is not None: + gd.proc.wait() # END cleanup # END bare repo creator remote_repo_creator.__name__ = func.__name__ diff --git a/git/test/performance/test_commit.py b/git/test/performance/test_commit.py index a890c833..7d3e87c4 100644 --- a/git/test/performance/test_commit.py +++ b/git/test/performance/test_commit.py @@ -4,13 +4,15 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php from __future__ import print_function +from io import BytesIO +from time import time +import sys + from .lib import TestBigRepoRW from git import Commit from gitdb import IStream +from git.compat import xrange from git.test.test_commit import assert_commit_serialization -from cStringIO import StringIO -from time import time -import sys class TestPerformance(TestBigRepoRW): @@ -90,7 +92,7 @@ class TestPerformance(TestBigRepoRW): hc.committer, hc.committed_date, hc.committer_tz_offset, str(i), parents=hc.parents, encoding=hc.encoding) - stream = StringIO() + stream = BytesIO() cm._serialize(stream) slen = stream.tell() stream.seek(0) diff --git a/git/test/performance/test_streams.py b/git/test/performance/test_streams.py index ff664c10..aecb7728 100644 --- a/git/test/performance/test_streams.py +++ b/git/test/performance/test_streams.py @@ -80,7 +80,7 @@ class TestObjDBPerformance(TestBigRepoR): elapsed_readchunks = time() - st stream.seek(0) - assert ''.join(chunks) == stream.getvalue() + assert b''.join(chunks) == stream.getvalue() cs_kib = cs / 1000 print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" diff --git a/git/test/test_base.py b/git/test/test_base.py index a14d4680..301384ef 100644 --- a/git/test/test_base.py +++ b/git/test/test_base.py @@ -1,12 +1,13 @@ +#-*-coding:utf-8-*- # test_base.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php - -import git.objects.base as base import os +import tempfile +import git.objects.base as base from git.test.lib import ( TestBase, assert_raises, @@ -68,10 +69,13 @@ class TestBase(TestBase): data = data_stream.read() assert data - tmpfile = os.tmpfile() + tmpfilename = tempfile.mktemp(suffix='test-stream') + tmpfile = open(tmpfilename, 'wb+') assert item == item.stream_data(tmpfile) tmpfile.seek(0) assert tmpfile.read() == data + tmpfile.close() + os.remove(tmpfilename) # END stream to file directly # END for each object type to create @@ -85,7 +89,7 @@ class TestBase(TestBase): assert base.Object in get_object_type_by_name(tname).mro() # END for each known type - assert_raises(ValueError, get_object_type_by_name, "doesntexist") + assert_raises(ValueError, get_object_type_by_name, b"doesntexist") def test_object_resolution(self): # objects must be resolved to shas so they compare equal @@ -106,3 +110,13 @@ class TestBase(TestBase): assert not rw_repo.config_reader("repository").getboolean("core", "bare") assert rw_remote_repo.config_reader("repository").getboolean("core", "bare") assert os.path.isdir(os.path.join(rw_repo.working_tree_dir, 'lib')) + + @with_rw_repo('0.1.6') + def test_add_unicode(self, rw_repo): + filename = u"שלום.txt" + + file_path = os.path.join(rw_repo.working_dir, filename) + open(file_path, "wb").write(b'something') + + rw_repo.git.add(rw_repo.working_dir) + rw_repo.index.commit('message') diff --git a/git/test/test_commit.py b/git/test/test_commit.py index bfad6fd6..1f0f8c56 100644 --- a/git/test/test_commit.py +++ b/git/test/test_commit.py @@ -19,9 +19,12 @@ from git import ( Actor, ) from gitdb import IStream -from gitdb.util import hex_to_bin +from git.compat import ( + string_types, + text_type +) -from cStringIO import StringIO +from io import BytesIO import time import sys import re @@ -40,14 +43,14 @@ def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False) # assert that we deserialize commits correctly, hence we get the same # sha on serialization - stream = StringIO() + stream = BytesIO() cm._serialize(stream) ns += 1 streamlen = stream.tell() stream.seek(0) istream = rwrepo.odb.store(IStream(Commit.type, streamlen, stream)) - assert istream.hexsha == cm.hexsha + assert istream.hexsha == cm.hexsha.encode('ascii') nc = Commit(rwrepo, Commit.NULL_BIN_SHA, cm.tree, cm.author, cm.authored_date, cm.author_tz_offset, @@ -55,7 +58,7 @@ def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False) cm.message, cm.parents, cm.encoding) assert nc.parents == cm.parents - stream = StringIO() + stream = BytesIO() nc._serialize(stream) ns += 1 streamlen = stream.tell() @@ -125,11 +128,11 @@ class TestCommit(TestBase): def test_unicode_actor(self): # assure we can parse unicode actors correctly - name = "Üäöß ÄußÉ".decode("utf-8") + name = u"Üäöß ÄußÉ" assert len(name) == 9 special = Actor._from_string(u"%s <something@this.com>" % name) assert special.name == name - assert isinstance(special.name, unicode) + assert isinstance(special.name, text_type) def test_traversal(self): start = self.rorepo.commit("a4d06724202afccd2b5c54f81bcf2bf26dea7fff") @@ -142,13 +145,13 @@ class TestCommit(TestBase): # basic branch first, depth first dfirst = start.traverse(branch_first=False) bfirst = start.traverse(branch_first=True) - assert dfirst.next() == p0 - assert dfirst.next() == p00 + assert next(dfirst) == p0 + assert next(dfirst) == p00 - assert bfirst.next() == p0 - assert bfirst.next() == p1 - assert bfirst.next() == p00 - assert bfirst.next() == p10 + assert next(bfirst) == p0 + assert next(bfirst) == p1 + assert next(bfirst) == p00 + assert next(bfirst) == p10 # at some point, both iterations should stop assert list(bfirst)[-1] == first @@ -157,19 +160,19 @@ class TestCommit(TestBase): assert len(l[0]) == 2 # ignore self - assert start.traverse(ignore_self=False).next() == start + assert next(start.traverse(ignore_self=False)) == start # depth assert len(list(start.traverse(ignore_self=False, depth=0))) == 1 # prune - assert start.traverse(branch_first=1, prune=lambda i, d: i == p0).next() == p1 + assert next(start.traverse(branch_first=1, prune=lambda i, d: i == p0)) == p1 # predicate - assert start.traverse(branch_first=1, predicate=lambda i, d: i == p1).next() == p1 + assert next(start.traverse(branch_first=1, predicate=lambda i, d: i == p1)) == p1 # traversal should stop when the beginning is reached - self.failUnlessRaises(StopIteration, first.traverse().next) + self.failUnlessRaises(StopIteration, next, first.traverse()) # parents of the first commit should be empty ( as the only parent has a null # sha ) @@ -206,7 +209,7 @@ class TestCommit(TestBase): first_parent=True, bisect_all=True) - commits = Commit._iter_from_process_or_stream(self.rorepo, StringProcessAdapter(revs)) + commits = Commit._iter_from_process_or_stream(self.rorepo, StringProcessAdapter(revs.encode('ascii'))) expected_ids = ( '7156cece3c49544abb6bf7a0c218eb36646fad6d', '1f66cfbbce58b4b552b041707a12d437cc5f400a', @@ -220,8 +223,10 @@ class TestCommit(TestBase): assert self.rorepo.tag('refs/tags/0.1.5').commit.count() == 143 def test_list(self): + # This doesn't work anymore, as we will either attempt getattr with bytes, or compare 20 byte string + # with actual 20 byte bytes. This usage makes no sense anyway assert isinstance(Commit.list_items(self.rorepo, '0.1.5', max_count=5)[ - hex_to_bin('5117c9c8a4d3af19a9958677e45cda9269de1541')], Commit) + '5117c9c8a4d3af19a9958677e45cda9269de1541'], Commit) def test_str(self): commit = Commit(self.rorepo, Commit.NULL_BIN_SHA) @@ -243,14 +248,14 @@ class TestCommit(TestBase): c = self.rorepo.commit('0.1.5') for skip in (0, 1): piter = c.iter_parents(skip=skip) - first_parent = piter.next() + first_parent = next(piter) assert first_parent != c assert first_parent == c.parents[0] # END for each - def test_base(self): + def test_name_rev(self): name_rev = self.rorepo.head.commit.name_rev - assert isinstance(name_rev, basestring) + assert isinstance(name_rev, string_types) @with_rw_repo('HEAD', bare=True) def test_serialization(self, rwrepo): @@ -263,16 +268,16 @@ class TestCommit(TestBase): # create a commit with unicode in the message, and the author's name # Verify its serialization and deserialization cmt = self.rorepo.commit('0.1.6') - assert isinstance(cmt.message, unicode) # it automatically decodes it as such - assert isinstance(cmt.author.name, unicode) # same here + assert isinstance(cmt.message, text_type) # it automatically decodes it as such + assert isinstance(cmt.author.name, text_type) # same here - cmt.message = "üäêèß".decode("utf-8") + cmt.message = u"üäêèß" assert len(cmt.message) == 5 - cmt.author.name = "äüß".decode("utf-8") + cmt.author.name = u"äüß" assert len(cmt.author.name) == 3 - cstream = StringIO() + cstream = BytesIO() cmt._serialize(cstream) cstream.seek(0) assert len(cstream.getvalue()) @@ -288,7 +293,7 @@ class TestCommit(TestBase): def test_gpgsig(self): cmt = self.rorepo.commit() - cmt._deserialize(open(fixture_path('commit_with_gpgsig'))) + cmt._deserialize(open(fixture_path('commit_with_gpgsig'), 'rb')) fixture_sig = """-----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.11 (GNU/Linux) @@ -312,9 +317,9 @@ JzJMZDRLQLFvnzqZuCjE cmt.gpgsig = "<test\ndummy\nsig>" assert cmt.gpgsig != fixture_sig - cstream = StringIO() + cstream = BytesIO() cmt._serialize(cstream) - assert re.search(r"^gpgsig <test\n dummy\n sig>$", cstream.getvalue(), re.MULTILINE) + assert re.search(r"^gpgsig <test\n dummy\n sig>$", cstream.getvalue().decode('ascii'), re.MULTILINE) cstream.seek(0) cmt.gpgsig = None @@ -322,6 +327,6 @@ JzJMZDRLQLFvnzqZuCjE assert cmt.gpgsig == "<test\ndummy\nsig>" cmt.gpgsig = None - cstream = StringIO() + cstream = BytesIO() cmt._serialize(cstream) - assert not re.search(r"^gpgsig ", cstream.getvalue(), re.MULTILINE) + assert not re.search(r"^gpgsig ", cstream.getvalue().decode('ascii'), re.MULTILINE) diff --git a/git/test/test_config.py b/git/test/test_config.py index d1c8e72f..546a2fe1 100644 --- a/git/test/test_config.py +++ b/git/test/test_config.py @@ -11,16 +11,19 @@ from git.test.lib import ( from git import ( GitConfigParser ) -import StringIO +from git.compat import ( + string_types, +) +import io from copy import copy -from ConfigParser import NoSectionError +from git.config import cp class TestBase(TestCase): def _to_memcache(self, file_path): - fp = open(file_path, "r") - sio = StringIO.StringIO(fp.read()) + fp = open(file_path, "rb") + sio = io.BytesIO(fp.read()) sio.name = file_path return sio @@ -38,7 +41,7 @@ class TestBase(TestCase): w_config.write() # enforce writing # we stripped lines when reading, so the results differ - assert file_obj.getvalue() != file_obj_orig.getvalue() + assert file_obj.getvalue() and file_obj.getvalue() != file_obj_orig.getvalue() # creating an additional config writer must fail due to exclusive access self.failUnlessRaises(IOError, GitConfigParser, file_obj, read_only=False) @@ -85,7 +88,7 @@ class TestBase(TestCase): num_options += 1 val = r_config.get(section, option) val_typed = r_config.get_value(section, option) - assert isinstance(val_typed, (bool, long, float, basestring)) + assert isinstance(val_typed, (bool, int, float, ) + string_types) assert val assert "\n" not in option assert "\n" not in val @@ -104,4 +107,4 @@ class TestBase(TestCase): assert r_config.get_value("doesnt", "exist", default) == default # it raises if there is no default though - self.failUnlessRaises(NoSectionError, r_config.get_value, "doesnt", "exist") + self.failUnlessRaises(cp.NoSectionError, r_config.get_value, "doesnt", "exist") diff --git a/git/test/test_fun.py b/git/test/test_fun.py index bf178aaa..40d040b9 100644 --- a/git/test/test_fun.py +++ b/git/test/test_fun.py @@ -24,13 +24,13 @@ from stat import ( ) from git.index import IndexFile -from cStringIO import StringIO +from io import BytesIO class TestFun(TestBase): def _assert_index_entries(self, entries, trees): - index = IndexFile.from_tree(self.rorepo, *[self.rorepo.tree(bin_to_hex(t)) for t in trees]) + index = IndexFile.from_tree(self.rorepo, *[self.rorepo.tree(bin_to_hex(t).decode('ascii')) for t in trees]) assert entries assert len(index.entries) == len(entries) for entry in entries: @@ -72,7 +72,7 @@ class TestFun(TestBase): def mktree(self, odb, entries): """create a tree from the given tree entries and safe it to the database""" - sio = StringIO() + sio = BytesIO() tree_to_stream(entries, sio.write) sio.seek(0) istream = odb.store(IStream(str_tree_type, len(sio.getvalue()), sio)) @@ -91,9 +91,9 @@ class TestFun(TestBase): assert has_conflict == (len([e for e in entries if e.stage != 0]) > 0) mktree = self.mktree - shaa = "\1" * 20 - shab = "\2" * 20 - shac = "\3" * 20 + shaa = b"\1" * 20 + shab = b"\2" * 20 + shac = b"\3" * 20 odb = rwrepo.odb @@ -256,6 +256,6 @@ class TestFun(TestBase): assert entries # END for each commit - def test_tree_entries_from_data(self): + def test_tree_entries_from_data_with_failing_name_decode(self): r = tree_entries_from_data(b'100644 \x9f\0aaa') - assert r == [('aaa', 33188, '\x9f')], r + assert r == [(b'aaa', 33188, b'\x9f')], r diff --git a/git/test/test_git.py b/git/test/test_git.py index 553f8d1b..502e6091 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -1,3 +1,4 @@ +#-*-coding:utf-8-*- # test_git.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # @@ -16,6 +17,8 @@ from git.test.lib import (TestBase, from git import (Git, GitCommandError) +from git.compat import PY3 + class TestGit(TestBase): @@ -32,12 +35,20 @@ class TestGit(TestBase): assert_equal(git.call_args, ((['git', 'version'],), {})) def test_call_unpack_args_unicode(self): - args = Git._Git__unpack_args(u'Unicode' + unichr(40960)) - assert_equal(args, ['Unicode\xea\x80\x80']) + args = Git._Git__unpack_args(u'Unicode€™') + if PY3: + mangled_value = 'Unicode\u20ac\u2122' + else: + mangled_value = 'Unicode\xe2\x82\xac\xe2\x84\xa2' + assert_equal(args, [mangled_value]) def test_call_unpack_args(self): - args = Git._Git__unpack_args(['git', 'log', '--', u'Unicode' + unichr(40960)]) - assert_equal(args, ['git', 'log', '--', 'Unicode\xea\x80\x80']) + args = Git._Git__unpack_args(['git', 'log', '--', u'Unicode€™']) + if PY3: + mangled_value = 'Unicode\u20ac\u2122' + else: + mangled_value = 'Unicode\xe2\x82\xac\xe2\x84\xa2' + assert_equal(args, ['git', 'log', '--', mangled_value]) @raises(GitCommandError) def test_it_raises_errors(self): @@ -75,13 +86,13 @@ class TestGit(TestBase): import subprocess as sp hexsha = "b2339455342180c7cc1e9bba3e9f181f7baa5167" g = self.git.cat_file(batch_check=True, istream=sp.PIPE, as_process=True) - g.stdin.write("b2339455342180c7cc1e9bba3e9f181f7baa5167\n") + g.stdin.write(b"b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() obj_info = g.stdout.readline() # read header + data g = self.git.cat_file(batch=True, istream=sp.PIPE, as_process=True) - g.stdin.write("b2339455342180c7cc1e9bba3e9f181f7baa5167\n") + g.stdin.write(b"b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() obj_info_two = g.stdout.readline() assert obj_info == obj_info_two @@ -92,7 +103,7 @@ class TestGit(TestBase): g.stdout.read(1) # now we should be able to read a new object - g.stdin.write("b2339455342180c7cc1e9bba3e9f181f7baa5167\n") + g.stdin.write(b"b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() assert g.stdout.readline() == obj_info diff --git a/git/test/test_index.py b/git/test/test_index.py index 15fff8d4..f7504b32 100644 --- a/git/test/test_index.py +++ b/git/test/test_index.py @@ -20,6 +20,7 @@ from git import ( GitCommandError, CheckoutError, ) +from git.compat import string_types from gitdb.util import hex_to_bin import os import sys @@ -30,7 +31,7 @@ from stat import ( ST_MODE ) -from StringIO import StringIO +from io import BytesIO from gitdb.base import IStream from git.objects import Blob from git.index.typ import ( @@ -47,7 +48,7 @@ class TestIndex(TestBase): def _assert_fprogress(self, entries): assert len(entries) == len(self._fprogress_map) - for path, call_count in self._fprogress_map.iteritems(): + for path, call_count in self._fprogress_map.items(): assert call_count == 2 # END for each item in progress map self._reset_progress() @@ -85,7 +86,7 @@ class TestIndex(TestBase): assert index.version > 0 # test entry - entry = index.entries.itervalues().next() + entry = next(iter(index.entries.values())) for attr in ("path", "ctime", "mtime", "dev", "inode", "mode", "uid", "gid", "size", "binsha", "hexsha", "stage"): getattr(entry, attr) @@ -99,7 +100,7 @@ class TestIndex(TestBase): # test stage index_merge = IndexFile(self.rorepo, fixture_path("index_merge")) assert len(index_merge.entries) == 106 - assert len(list(e for e in index_merge.entries.itervalues() if e.stage != 0)) + assert len(list(e for e in index_merge.entries.values() if e.stage != 0)) # write the data - it must match the original tmpfile = tempfile.mktemp() @@ -166,7 +167,7 @@ class TestIndex(TestBase): assert unmerged_blob_map # pick the first blob at the first stage we find and use it as resolved version - three_way_index.resolve_blobs(l[0][1] for l in unmerged_blob_map.itervalues()) + three_way_index.resolve_blobs(l[0][1] for l in unmerged_blob_map.values()) tree = three_way_index.write_tree() assert isinstance(tree, Tree) num_blobs = 0 @@ -200,7 +201,7 @@ class TestIndex(TestBase): # Add a change with a NULL sha that should conflict with next_commit. We # pretend there was a change, but we do not even bother adding a proper # sha for it ( which makes things faster of course ) - manifest_fake_entry = BaseIndexEntry((manifest_entry[0], "\0" * 20, 0, manifest_entry[3])) + manifest_fake_entry = BaseIndexEntry((manifest_entry[0], b"\0" * 20, 0, manifest_entry[3])) # try write flag self._assert_entries(rw_repo.index.add([manifest_fake_entry], write=False)) # add actually resolves the null-hex-sha for us as a feature, but we can @@ -235,7 +236,7 @@ class TestIndex(TestBase): # now make a proper three way merge with unmerged entries unmerged_tree = IndexFile.from_tree(rw_repo, parent_commit, tree, next_commit) unmerged_blobs = unmerged_tree.unmerged_blobs() - assert len(unmerged_blobs) == 1 and unmerged_blobs.keys()[0] == manifest_key[0] + assert len(unmerged_blobs) == 1 and list(unmerged_blobs.keys())[0] == manifest_key[0] @with_rw_repo('0.1.6') def test_index_file_diffing(self, rw_repo): @@ -294,7 +295,7 @@ class TestIndex(TestBase): assert index.diff(None) # reset the working copy as well to current head,to pull 'back' as well - new_data = "will be reverted" + new_data = b"will be reverted" file_path = os.path.join(rw_repo.working_tree_dir, "CHANGES") fp = open(file_path, "wb") fp.write(new_data) @@ -311,7 +312,7 @@ class TestIndex(TestBase): # test full checkout test_file = os.path.join(rw_repo.working_tree_dir, "CHANGES") - open(test_file, 'ab').write("some data") + open(test_file, 'ab').write(b"some data") rval = index.checkout(None, force=True, fprogress=self._fprogress) assert 'CHANGES' in list(rval) self._assert_fprogress([None]) @@ -335,7 +336,7 @@ class TestIndex(TestBase): self.failUnlessRaises(CheckoutError, index.checkout, paths=["doesnt/exist"]) # checkout file with modifications - append_data = "hello" + append_data = b"hello" fp = open(test_file, "ab") fp.write(append_data) fp.close() @@ -343,15 +344,15 @@ class TestIndex(TestBase): index.checkout(test_file) except CheckoutError as e: assert len(e.failed_files) == 1 and e.failed_files[0] == os.path.basename(test_file) - assert (len(e.failed_files) == len(e.failed_reasons)) and isinstance(e.failed_reasons[0], basestring) + assert (len(e.failed_files) == len(e.failed_reasons)) and isinstance(e.failed_reasons[0], string_types) assert len(e.valid_files) == 0 - assert open(test_file).read().endswith(append_data) + assert open(test_file, 'rb').read().endswith(append_data) else: raise AssertionError("Exception CheckoutError not thrown") # if we force it it should work index.checkout(test_file, force=True) - assert not open(test_file).read().endswith(append_data) + assert not open(test_file, 'rb').read().endswith(append_data) # checkout directory shutil.rmtree(os.path.join(rw_repo.working_tree_dir, "lib")) @@ -378,14 +379,16 @@ class TestIndex(TestBase): uname = "Some Developer" umail = "sd@company.com" - rw_repo.config_writer().set_value("user", "name", uname) - rw_repo.config_writer().set_value("user", "email", umail) + writer = rw_repo.config_writer() + writer.set_value("user", "name", uname) + writer.set_value("user", "email", umail) + writer.release() # remove all of the files, provide a wild mix of paths, BaseIndexEntries, # IndexEntries def mixed_iterator(): count = 0 - for entry in index.entries.itervalues(): + for entry in index.entries.values(): type_id = count % 4 if type_id == 0: # path yield entry.path @@ -499,7 +502,7 @@ class TestIndex(TestBase): # mode 0 not allowed null_hex_sha = Diff.NULL_HEX_SHA - null_bin_sha = "\0" * 20 + null_bin_sha = b"\0" * 20 self.failUnlessRaises(ValueError, index.reset( new_commit).add, [BaseIndexEntry((0, null_bin_sha, 0, "doesntmatter"))]) @@ -525,7 +528,7 @@ class TestIndex(TestBase): assert S_ISLNK(index.entries[index.entry_key("my_real_symlink", 0)].mode) # we expect only the target to be written - assert index.repo.odb.stream(entries[0].binsha).read() == target + assert index.repo.odb.stream(entries[0].binsha).read().decode('ascii') == target # END real symlink test # add fake symlink and assure it checks-our as symlink @@ -617,7 +620,7 @@ class TestIndex(TestBase): for fid in range(3): fname = 'newfile%i' % fid - open(fname, 'wb').write("abcd") + open(fname, 'wb').write(b"abcd") yield Blob(rw_repo, Blob.NULL_BIN_SHA, 0o100644, fname) # END for each new file # END path producer @@ -697,9 +700,9 @@ class TestIndex(TestBase): # instead of throwing the Exception we are expecting. This is # a quick hack to make this test fail when expected. rw_bare_repo._working_tree_dir = None - contents = 'This is a StringIO file' + contents = b'This is a BytesIO file' filesize = len(contents) - fileobj = StringIO(contents) + fileobj = BytesIO(contents) filename = 'my-imaginary-file' istream = rw_bare_repo.odb.store( IStream(Blob.type, filesize, fileobj)) @@ -715,5 +718,5 @@ class TestIndex(TestBase): try: rw_bare_repo.index.add([path]) except Exception as e: - asserted = "does not have a working tree" in e.message + asserted = "does not have a working tree" in str(e) assert asserted, "Adding using a filename is not correctly asserted." diff --git a/git/test/test_reflog.py b/git/test/test_reflog.py index 4efb8025..3571e083 100644 --- a/git/test/test_reflog.py +++ b/git/test/test_reflog.py @@ -8,6 +8,7 @@ from git.refs import ( RefLog ) from git.util import Actor +from gitdb.util import hex_to_bin import tempfile import shutil @@ -51,7 +52,7 @@ class TestRefLog(TestBase): assert len(reflog) # iter_entries works with path and with stream - assert len(list(RefLog.iter_entries(open(rlp_master)))) + assert len(list(RefLog.iter_entries(open(rlp_master, 'rb')))) assert len(list(RefLog.iter_entries(rlp_master))) # raise on invalid revlog @@ -65,7 +66,7 @@ class TestRefLog(TestBase): self.failUnlessRaises(ValueError, RefLog().write) # test serialize and deserialize - results must match exactly - binsha = chr(255) * 20 + binsha = hex_to_bin(('f' * 40).encode('ascii')) msg = "my reflog message" cr = self.rorepo.config_reader() for rlp in (rlp_head, rlp_master): diff --git a/git/test/test_refs.py b/git/test/test_refs.py index af33765a..14b91cfe 100644 --- a/git/test/test_refs.py +++ b/git/test/test_refs.py @@ -105,9 +105,11 @@ class TestRefs(TestBase): tv = "testopt" writer.set_value(tv, 1) assert writer.get_value(tv) == 1 - del(writer) + writer.release() assert head.config_reader().get_value(tv) == 1 - head.config_writer().remove_option(tv) + writer = head.config_writer() + writer.remove_option(tv) + writer.release() # after the clone, we might still have a tracking branch setup head.set_tracking_branch(None) diff --git a/git/test/test_remote.py b/git/test/test_remote.py index a8d5179a..75dc19c5 100644 --- a/git/test/test_remote.py +++ b/git/test/test_remote.py @@ -23,6 +23,7 @@ from git import ( GitCommandError ) from git.util import IterableList +from git.compat import string_types import tempfile import shutil import os @@ -97,7 +98,7 @@ class TestRemote(TestBase): # self._print_fetchhead(remote.repo) assert len(results) > 0 and isinstance(results[0], FetchInfo) for info in results: - assert isinstance(info.note, basestring) + assert isinstance(info.note, string_types) if isinstance(info.ref, Reference): assert info.flags != 0 # END reference type flags handling @@ -113,7 +114,7 @@ class TestRemote(TestBase): assert len(results) > 0 and isinstance(results[0], PushInfo) for info in results: assert info.flags - assert isinstance(info.summary, basestring) + assert isinstance(info.summary, string_types) if info.old_commit is not None: assert isinstance(info.old_commit, Commit) if info.flags & info.ERROR: diff --git a/git/test/test_repo.py b/git/test/test_repo.py index f6b46a6e..f216039e 100644 --- a/git/test/test_repo.py +++ b/git/test/test_repo.py @@ -30,12 +30,16 @@ from git import ( from git.util import join_path_native from git.exc import BadObject from gitdb.util import bin_to_hex +from git.compat import ( + string_types, + defenc +) import os import sys import tempfile import shutil -from cStringIO import StringIO +from io import BytesIO class TestRepo(TestBase): @@ -258,13 +262,16 @@ class TestRepo(TestBase): assert self.rorepo.tag('refs/tags/0.1.5').commit def test_archive(self): - tmpfile = os.tmpfile() - self.rorepo.archive(tmpfile, '0.1.5') - assert tmpfile.tell() + tmpfile = tempfile.mktemp(suffix='archive-test') + stream = open(tmpfile, 'wb') + self.rorepo.archive(stream, '0.1.5') + assert stream.tell() + stream.close() + os.remove(tmpfile) @patch.object(Git, '_call_process') def test_should_display_blame_information(self, git): - git.return_value = fixture('blame') + git.return_value = fixture('blame').decode(defenc) b = self.rorepo.blame('master', 'lib/git.py') assert_equal(13, len(b)) assert_equal(2, len(b[0])) @@ -286,7 +293,7 @@ class TestRepo(TestBase): # test the 'lines per commit' entries tlist = b[0][1] assert_true(tlist) - assert_true(isinstance(tlist[0], basestring)) + assert_true(isinstance(tlist[0], string_types)) assert_true(len(tlist) < sum(len(t) for t in tlist)) # test for single-char bug def test_blame_real(self): @@ -335,6 +342,7 @@ class TestRepo(TestBase): try: writer = self.rorepo.config_writer(config_level) assert not writer.read_only + writer.release() except IOError: # its okay not to get a writer for some configuration files if we # have no permissions @@ -349,7 +357,8 @@ class TestRepo(TestBase): tag = self.rorepo.create_tag("new_tag", "HEAD~2") self.rorepo.delete_tag(tag) - self.rorepo.config_writer() + writer = self.rorepo.config_writer() + writer.release() remote = self.rorepo.create_remote("new_remote", "git@server:repo.git") self.rorepo.delete_remote(remote) @@ -361,27 +370,27 @@ class TestRepo(TestBase): def test_git_cmd(self): # test CatFileContentStream, just to be very sure we have no fencepost errors # last \n is the terminating newline that it expects - l1 = "0123456789\n" - l2 = "abcdefghijklmnopqrstxy\n" - l3 = "z\n" - d = "%s%s%s\n" % (l1, l2, l3) + l1 = b"0123456789\n" + l2 = b"abcdefghijklmnopqrstxy\n" + l3 = b"z\n" + d = l1 + l2 + l3 + b"\n" l1p = l1[:5] # full size # size is without terminating newline def mkfull(): - return Git.CatFileContentStream(len(d) - 1, StringIO(d)) + return Git.CatFileContentStream(len(d) - 1, BytesIO(d)) ts = 5 def mktiny(): - return Git.CatFileContentStream(ts, StringIO(d)) + return Git.CatFileContentStream(ts, BytesIO(d)) # readlines no limit s = mkfull() lines = s.readlines() - assert len(lines) == 3 and lines[-1].endswith('\n') + assert len(lines) == 3 and lines[-1].endswith(b'\n') assert s._stream.tell() == len(d) # must have scrubbed to the end # realines line limit @@ -565,7 +574,7 @@ class TestRepo(TestBase): # try partial parsing max_items = 40 for i, binsha in enumerate(self.rorepo.odb.sha_iter()): - assert rev_parse(bin_to_hex(binsha)[:8 - (i % 2)]).binsha == binsha + assert rev_parse(bin_to_hex(binsha)[:8 - (i % 2)].decode('ascii')).binsha == binsha if i > max_items: # this is rather slow currently, as rev_parse returns an object # which requires accessing packs, it has some additional overhead @@ -644,6 +653,6 @@ class TestRepo(TestBase): assert os.path.abspath(git_file_repo.git_dir) == real_path_abs # Test using an absolute gitdir path in the .git file. - open(git_file_path, 'wb').write('gitdir: %s\n' % real_path_abs) + open(git_file_path, 'wb').write(('gitdir: %s\n' % real_path_abs).encode('ascii')) git_file_repo = Repo(rwrepo.working_tree_dir) assert os.path.abspath(git_file_repo.git_dir) == real_path_abs diff --git a/git/test/test_stats.py b/git/test/test_stats.py index c4535b75..884ab1ab 100644 --- a/git/test/test_stats.py +++ b/git/test/test_stats.py @@ -10,12 +10,13 @@ from git.test.lib import ( assert_equal ) from git import Stats +from git.compat import defenc class TestStats(TestBase): def test_list_from_string(self): - output = fixture('diff_numstat') + output = fixture('diff_numstat').decode(defenc) stats = Stats._list_from_string(self.rorepo, output) assert_equal(2, stats.total['files']) diff --git a/git/test/test_submodule.py b/git/test/test_submodule.py index ec3459e4..99996ce3 100644 --- a/git/test/test_submodule.py +++ b/git/test/test_submodule.py @@ -9,6 +9,7 @@ from git.exc import InvalidGitRepositoryError from git.objects.submodule.base import Submodule from git.objects.submodule.root import RootModule, RootUpdateProgress from git.util import to_native_path_linux, join_path_native +from git.compat import string_types import shutil import git import sys @@ -76,10 +77,10 @@ class TestSubmodule(TestBase): self.failUnlessRaises(InvalidGitRepositoryError, getattr, sm, 'branch') # branch_path works, as its just a string - assert isinstance(sm.branch_path, basestring) + assert isinstance(sm.branch_path, string_types) # some commits earlier we still have a submodule, but its at a different commit - smold = Submodule.iter_items(rwrepo, self.k_subm_changed).next() + smold = next(Submodule.iter_items(rwrepo, self.k_subm_changed)) assert smold.binsha != sm.binsha assert smold != sm # the name changed @@ -98,7 +99,7 @@ class TestSubmodule(TestBase): # for faster checkout, set the url to the local path new_smclone_path = to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, sm.path)) writer.set_value('url', new_smclone_path) - del(writer) + writer.release() assert sm.config_reader().get_value('url') == new_smclone_path assert sm.url == new_smclone_path # END handle bare repo @@ -195,7 +196,9 @@ class TestSubmodule(TestBase): # adjust the path of the submodules module to point to the local destination new_csmclone_path = to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, sm.path, csm.path)) - csm.config_writer().set_value('url', new_csmclone_path) + writer = csm.config_writer() + writer.set_value('url', new_csmclone_path) + writer.release() assert csm.url == new_csmclone_path # dry-run does nothing @@ -256,8 +259,12 @@ class TestSubmodule(TestBase): # NOTE: As we did a few updates in the meanwhile, the indices were reset # Hence we create some changes csm.set_parent_commit(csm.repo.head.commit) - sm.config_writer().set_value("somekey", "somevalue") - csm.config_writer().set_value("okey", "ovalue") + writer = sm.config_writer() + writer.set_value("somekey", "somevalue") + writer.release() + writer = csm.config_writer() + writer.set_value("okey", "ovalue") + writer.release() self.failUnlessRaises(InvalidGitRepositoryError, sm.remove) # if we remove the dirty index, it would work sm.module().index.reset() @@ -405,7 +412,8 @@ class TestSubmodule(TestBase): assert len(rm.list_items(rm.module())) == 1 rm.config_reader() - rm.config_writer() + w = rm.config_writer() + w.release() # deep traversal gitdb / async rsmsp = [sm.path for sm in rm.traverse()] @@ -430,8 +438,9 @@ class TestSubmodule(TestBase): assert not sm.module_exists() # was never updated after rwrepo's clone # assure we clone from a local source - sm.config_writer().set_value( - 'url', to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, sm.path))) + writer = sm.config_writer() + writer.set_value('url', to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, sm.path))) + writer.release() # dry-run does nothing sm.update(recursive=False, dry_run=True, progress=prog) @@ -439,7 +448,9 @@ class TestSubmodule(TestBase): sm.update(recursive=False) assert sm.module_exists() - sm.config_writer().set_value('path', fp) # change path to something with prefix AFTER url change + writer = sm.config_writer() + writer.set_value('path', fp) # change path to something with prefix AFTER url change + writer.release() # update fails as list_items in such a situations cannot work, as it cannot # find the entry at the changed path @@ -503,7 +514,9 @@ class TestSubmodule(TestBase): # repository at the different url nsm.set_parent_commit(csmremoved) nsmurl = to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, rsmsp[0])) - nsm.config_writer().set_value('url', nsmurl) + writer = nsm.config_writer() + writer.set_value('url', nsmurl) + writer.release() csmpathchange = rwrepo.index.commit("changed url") nsm.set_parent_commit(csmpathchange) @@ -531,7 +544,9 @@ class TestSubmodule(TestBase): nsmm = nsm.module() prev_commit = nsmm.head.commit for branch in ("some_virtual_branch", cur_branch.name): - nsm.config_writer().set_value(Submodule.k_head_option, git.Head.to_full_path(branch)) + writer = nsm.config_writer() + writer.set_value(Submodule.k_head_option, git.Head.to_full_path(branch)) + writer.release() csmbranchchange = rwrepo.index.commit("changed branch to %s" % branch) nsm.set_parent_commit(csmbranchchange) # END for each branch to change @@ -559,7 +574,9 @@ class TestSubmodule(TestBase): assert nsm.exists() and nsm.module_exists() and len(nsm.children()) >= 1 # assure we pull locally only nsmc = nsm.children()[0] - nsmc.config_writer().set_value('url', async_url) + writer = nsmc.config_writer() + writer.set_value('url', async_url) + writer.release() rm.update(recursive=True, progress=prog, dry_run=True) # just to run the code rm.update(recursive=True, progress=prog) diff --git a/git/test/test_tree.py b/git/test/test_tree.py index d2e3606b..7a16b777 100644 --- a/git/test/test_tree.py +++ b/git/test/test_tree.py @@ -11,7 +11,7 @@ from git import ( Blob ) -from cStringIO import StringIO +from io import BytesIO class TestTree(TestBase): @@ -30,7 +30,7 @@ class TestTree(TestBase): orig_data = tree.data_stream.read() orig_cache = tree._cache - stream = StringIO() + stream = BytesIO() tree._serialize(stream) assert stream.getvalue() == orig_data @@ -82,7 +82,7 @@ class TestTree(TestBase): mod.set_done() # multiple times are okay # serialize, its different now - stream = StringIO() + stream = BytesIO() testtree._serialize(stream) stream.seek(0) assert stream.getvalue() != orig_data @@ -138,6 +138,7 @@ class TestTree(TestBase): # END check for slash # slashes in paths are supported as well + # NOTE: on py3, / doesn't work with strings anymore ... assert root[item.path] == item == root / item.path # END for each item assert found_slash diff --git a/git/test/test_util.py b/git/test/test_util.py index 888eb4ee..c6ca6920 100644 --- a/git/test/test_util.py +++ b/git/test/test_util.py @@ -24,6 +24,7 @@ from git.objects.util import ( parse_date, ) from git.cmd import dashify +from git.compat import string_types import time @@ -104,7 +105,7 @@ class TestUtils(TestBase): # now that we are here, test our conversion functions as well utctz = altz_to_utctz_str(offset) - assert isinstance(utctz, basestring) + assert isinstance(utctz, string_types) assert utctz_to_altz(verify_utctz(utctz)) == offset # END assert rval utility diff --git a/git/util.py b/git/util.py index fecd9fa2..4de736d3 100644 --- a/git/util.py +++ b/git/util.py @@ -15,7 +15,8 @@ import getpass # NOTE: Some of the unused imports might be used/imported by others. # Handle once test-cases are back up and running. -from exc import GitCommandError +from .exc import GitCommandError +from .compat import MAXSIZE # Most of these are unused here, but are for use by git-python modules so these # don't see gitdb all the time. Flake of course doesn't like it. @@ -445,7 +446,7 @@ class IndexFileSHA1Writer(object): def __init__(self, f): self.f = f - self.sha1 = make_sha("") + self.sha1 = make_sha(b"") def write(self, data): self.sha1.update(data) @@ -489,10 +490,7 @@ class LockFile(object): def _has_lock(self): """:return: True if we have a lock and if the lockfile still exists :raise AssertionError: if our lock-file does not exist""" - if not self._owns_lock: - return False - - return True + return self._owns_lock def _obtain_lock_or_raise(self): """Create a lock file as flag for other instances, mark our instance as lock-holder @@ -530,7 +528,7 @@ class LockFile(object): # on bloody windows, the file needs write permissions to be removable. # Why ... if os.name == 'nt': - os.chmod(lfp, int("0777", 8)) + os.chmod(lfp, 0o777) # END handle win32 os.remove(lfp) except OSError: @@ -548,7 +546,7 @@ class BlockingLockFile(LockFile): can never be obtained.""" __slots__ = ("_check_interval", "_max_block_time") - def __init__(self, file_path, check_interval_s=0.3, max_block_time_s=sys.maxint): + def __init__(self, file_path, check_interval_s=0.3, max_block_time_s=MAXSIZE): """Configure the instance :parm check_interval_s: |