diff options
54 files changed, 721 insertions, 452 deletions
@@ -83,10 +83,6 @@ In short, I want to make a new release of 0.3 with all contributions and fixes i The goals I have set for myself, in order, are as follows, all on branch 0.3. -* bring the test suite back online to work with the most commonly used git version -* merge all open pull requests, may there be a test-case or not, back. If something breaks, fix it if possible or let the contributor know -* conform git-python's structure and toolchain to the one used in my [other OSS projects](https://github.com/Byron/bcore) -* evaluate all open issues and close them if possible * evaluate python 3.3 compatibility and establish it if possible While that is happening, I will try hard to foster community around the project. This means being more responsive on the mailing list and in issues, as well as setting up clear guide lines about the [contribution](http://rfc.zeromq.org/spec:22) and maintenance workflow. @@ -1 +1 @@ -0.3.3 +0.3.4 diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 924743bd..b7479e4f 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,10 +2,18 @@ Changelog ========= +0.3.4 - Python 3 Support +======================== +* Internally, hexadecimal SHA1 are treated as ascii encoded strings. Binary SHA1 are treated as bytes. +* Id attribute of Commit objects is now `hexsha`, instead of `binsha`. The latter makes no sense in python 3 and I see no application of it anyway besides its artificial usage in test cases. +* **IMPORTANT**: If you were using the config_writer(), you implicitly relied on __del__ to work as expected to flush changes. To be sure changes are flushed under PY3, you will have to call the new `release()` method to trigger a flush. For some reason, __del__ is not called necessarily anymore when a symbol goes out of scope. +* The `Tree` now has a `.join('name')` method which is equivalent to `tree / 'name'` + 0.3.3 ===== * When fetching, pulling or pushing, and an error occours, it will not be reported on stdout anymore. However, if there is a fatal error, it will still result in a GitCommandError to be thrown. This goes hand in hand with improved fetch result parsing. * Code Cleanup (in preparation for python 3 support) + * Applied autopep8 and cleaned up code * Using python logging module instead of print statments to signal certain kinds of errors @@ -7,18 +7,24 @@ import os import sys import logging -from util import ( - LazyMixin, - stream_copy -) -from exc import GitCommandError - from subprocess import ( call, Popen, PIPE ) + +from .util import ( + LazyMixin, + stream_copy +) +from .exc import GitCommandError +from git.compat import ( + string_types, + defenc, + PY3 +) + execute_kwargs = ('istream', 'with_keep_cwd', 'with_extended_output', 'with_exceptions', 'as_process', 'output_stream') @@ -114,7 +120,7 @@ class Git(LazyMixin): :raise GitCommandError: if the return status is not 0""" status = self.proc.wait() if status != 0: - raise GitCommandError(self.args, status, self.proc.stderr.read()) + raise GitCommandError(self.args, status, self.proc.stderr.read().decode(defenc)) # END status handling return status # END auto interrupt @@ -314,6 +320,7 @@ class Git(LazyMixin): always be created with a pipe due to issues with subprocess. This merely is a workaround as data will be copied from the output pipe to the given output stream directly. + Judging from the implementation, you shouldn't use this flag ! :param subprocess_kwargs: Keyword arguments to be passed to subprocess.Popen. Please note that @@ -365,15 +372,15 @@ class Git(LazyMixin): # Wait for the process to return status = 0 - stdout_value = '' - stderr_value = '' + stdout_value = b'' + stderr_value = b'' try: if output_stream is None: stdout_value, stderr_value = proc.communicate() # strip trailing "\n" - if stdout_value.endswith("\n"): + if stdout_value.endswith(b"\n"): stdout_value = stdout_value[:-1] - if stderr_value.endswith("\n"): + if stderr_value.endswith(b"\n"): stderr_value = stderr_value[:-1] status = proc.returncode else: @@ -381,7 +388,7 @@ class Git(LazyMixin): stdout_value = output_stream stderr_value = proc.stderr.read() # strip trailing "\n" - if stderr_value.endswith("\n"): + if stderr_value.endswith(b"\n"): stderr_value = stderr_value[:-1] status = proc.wait() # END stdout handling @@ -392,9 +399,10 @@ class Git(LazyMixin): if self.GIT_PYTHON_TRACE == 'full': cmdstr = " ".join(command) if stderr_value: - log.info("%s -> %d; stdout: '%s'; stderr: '%s'", cmdstr, status, stdout_value, stderr_value) + log.info("%s -> %d; stdout: '%s'; stderr: '%s'", + cmdstr, status, stdout_value.decode(defenc), stderr_value.decode(defenc)) elif stdout_value: - log.info("%s -> %d; stdout: '%s'", cmdstr, status, stdout_value) + log.info("%s -> %d; stdout: '%s'", cmdstr, status, stdout_value.decode(defenc)) else: log.info("%s -> %d", cmdstr, status) # END handle debug printing @@ -405,9 +413,12 @@ class Git(LazyMixin): else: raise GitCommandError(command, status, stderr_value) + if isinstance(stdout_value, bytes): # could also be output_stream + stdout_value = stdout_value.decode(defenc) + # Allow access to the command's status code if with_extended_output: - return (status, stdout_value, stderr_value) + return (status, stdout_value, stderr_value.decode(defenc)) else: return stdout_value @@ -433,16 +444,18 @@ class Git(LazyMixin): @classmethod def __unpack_args(cls, arg_list): if not isinstance(arg_list, (list, tuple)): - if isinstance(arg_list, unicode): - return [arg_list.encode('utf-8')] + # This is just required for unicode conversion, as subprocess can't handle it + # However, in any other case, passing strings (usually utf-8 encoded) is totally fine + if not PY3 and isinstance(arg_list, unicode): + return [arg_list.encode(defenc)] return [str(arg_list)] outlist = list() for arg in arg_list: if isinstance(arg_list, (list, tuple)): outlist.extend(cls.__unpack_args(arg)) - elif isinstance(arg_list, unicode): - outlist.append(arg_list.encode('utf-8')) + elif not PY3 and isinstance(arg_list, unicode): + outlist.append(arg_list.encode(defenc)) # END recursion else: outlist.append(str(arg)) @@ -498,8 +511,8 @@ class Git(LazyMixin): # Prepare the argument list opt_args = self.transform_kwargs(**kwargs) - ext_args = self.__unpack_args([a for a in args if a is not None]) + args = opt_args + ext_args def make_call(): @@ -567,14 +580,20 @@ class Git(LazyMixin): raise ValueError("Failed to parse header: %r" % header_line) return (tokens[0], tokens[1], int(tokens[2])) - def __prepare_ref(self, ref): - # required for command to separate refs on stdin - refstr = str(ref) # could be ref-object - if refstr.endswith("\n"): - return refstr - return refstr + "\n" + def _prepare_ref(self, ref): + # required for command to separate refs on stdin, as bytes + refstr = ref + if isinstance(ref, bytes): + # Assume 40 bytes hexsha - bin-to-ascii for some reason returns bytes, not text + refstr = ref.decode('ascii') + elif not isinstance(ref, string_types): + refstr = str(ref) # could be ref-object + + if not refstr.endswith("\n"): + refstr += "\n" + return refstr.encode(defenc) - def __get_persistent_cmd(self, attr_name, cmd_name, *args, **kwargs): + def _get_persistent_cmd(self, attr_name, cmd_name, *args, **kwargs): cur_val = getattr(self, attr_name) if cur_val is not None: return cur_val @@ -587,7 +606,7 @@ class Git(LazyMixin): return cmd def __get_object_header(self, cmd, ref): - cmd.stdin.write(self.__prepare_ref(ref)) + cmd.stdin.write(self._prepare_ref(ref)) cmd.stdin.flush() return self._parse_object_header(cmd.stdout.readline()) @@ -599,7 +618,7 @@ class Git(LazyMixin): once and reuses the command in subsequent calls. :return: (hexsha, type_string, size_as_int)""" - cmd = self.__get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) + cmd = self._get_persistent_cmd("cat_file_header", "cat_file", batch_check=True) return self.__get_object_header(cmd, ref) def get_object_data(self, ref): @@ -616,7 +635,7 @@ class Git(LazyMixin): :return: (hexsha, type_string, size_as_int, stream) :note: This method is not threadsafe, you need one independent Command instance per thread to be safe !""" - cmd = self.__get_persistent_cmd("cat_file_all", "cat_file", batch=True) + cmd = self._get_persistent_cmd("cat_file_all", "cat_file", batch=True) hexsha, typename, size = self.__get_object_header(cmd, ref) return (hexsha, typename, size, self.CatFileContentStream(size, cmd.stdout)) diff --git a/git/compat.py b/git/compat.py new file mode 100644 index 00000000..5c330e5b --- /dev/null +++ b/git/compat.py @@ -0,0 +1,66 @@ +#-*-coding:utf-8-*- +# config.py +# Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors +# +# This module is part of GitPython and is released under +# the BSD License: http://www.opensource.org/licenses/bsd-license.php +"""utilities to help provide compatibility with python 3""" +# flake8: noqa + +import sys + +from gitdb.utils.compat import ( + PY3, + xrange, + MAXSIZE, + izip, +) + +from gitdb.utils.encoding import ( + string_types, + text_type, + force_bytes, + force_text +) + +defenc = sys.getdefaultencoding() +if PY3: + import io + FileType = io.IOBase + def byte_ord(b): + return b + def bchr(n): + return bytes([n]) + def mviter(d): + return d.values() +else: + FileType = file + # usually, this is just ascii, which might not enough for our encoding needs + # Unless it's set specifically, we override it to be utf-8 + if defenc == 'ascii': + defenc = 'utf-8' + byte_ord = ord + bchr = chr + def mviter(d): + return d.itervalues() + + +def with_metaclass(meta, *bases): + """copied from https://github.com/Byron/bcore/blob/master/src/python/butility/future.py#L15""" + class metaclass(meta): + __call__ = type.__call__ + __init__ = type.__init__ + + def __new__(cls, name, nbases, d): + if nbases is None: + return type.__new__(cls, name, (), d) + # There may be clients who rely on this attribute to be set to a reasonable value, which is why + # we set the __metaclass__ attribute explicitly + if not PY3 and '___metaclass__' not in d: + d['__metaclass__'] = meta + # end + return meta(name, bases, d) + # end + # end metaclass + return metaclass(meta.__name__ + 'Helper', None, {}) + # end handle py2 diff --git a/git/config.py b/git/config.py index 6a85760c..eefab299 100644 --- a/git/config.py +++ b/git/config.py @@ -7,12 +7,23 @@ configuration files""" import re -import ConfigParser as cp +try: + import ConfigParser as cp +except ImportError: + # PY3 + import configparser as cp import inspect import logging +import abc from git.odict import OrderedDict from git.util import LockFile +from git.compat import ( + string_types, + FileType, + defenc, + with_metaclass +) __all__ = ('GitConfigParser', 'SectionConstraint') @@ -20,7 +31,7 @@ __all__ = ('GitConfigParser', 'SectionConstraint') log = logging.getLogger('git.config') -class MetaParserBuilder(type): +class MetaParserBuilder(abc.ABCMeta): """Utlity class wrapping base-class methods into decorators that assure read-only properties""" def __new__(metacls, name, bases, clsdict): @@ -31,7 +42,7 @@ class MetaParserBuilder(type): if kmm in clsdict: mutating_methods = clsdict[kmm] for base in bases: - methods = (t for t in inspect.getmembers(base, inspect.ismethod) if not t[0].startswith("_")) + methods = (t for t in inspect.getmembers(base, inspect.isroutine) if not t[0].startswith("_")) for name, method in methods: if name in clsdict: continue @@ -88,6 +99,12 @@ class SectionConstraint(object): self._config = config self._section_name = section + def __del__(self): + # Yes, for some reason, we have to call it explicitly for it to work in PY3 ! + # Apparently __del__ doesn't get call anymore if refcount becomes 0 + # Ridiculous ... . + self._config.release() + def __getattr__(self, attr): if attr in self._valid_attrs_: return lambda *args, **kwargs: self._call_config(attr, *args, **kwargs) @@ -103,8 +120,12 @@ class SectionConstraint(object): """return: Configparser instance we constrain""" return self._config + def release(self): + """Equivalent to GitConfigParser.release(), which is called on our underlying parser instance""" + return self._config.release() + -class GitConfigParser(cp.RawConfigParser, object): +class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, object)): """Implements specifics required to read git style configuration files. @@ -120,7 +141,6 @@ class GitConfigParser(cp.RawConfigParser, object): :note: The config is case-sensitive even when queried, hence section and option names must match perfectly.""" - __metaclass__ = MetaParserBuilder #{ Configuration # The lock type determines the type of lock to use in new configuration readers. @@ -142,7 +162,6 @@ class GitConfigParser(cp.RawConfigParser, object): # list of RawConfigParser methods able to change the instance _mutating_methods_ = ("add_section", "remove_section", "remove_option", "set") - __slots__ = ("_sections", "_defaults", "_file_or_files", "_read_only", "_is_initialized", '_lock') def __init__(self, file_or_files, read_only=True): """Initialize a configuration reader to read the given file_or_files and to @@ -154,11 +173,11 @@ class GitConfigParser(cp.RawConfigParser, object): :param read_only: If True, the ConfigParser may only read the data , but not change it. If False, only a single file path or file object may be given.""" - super(GitConfigParser, self).__init__() - # initialize base with ordered dictionaries to be sure we write the same - # file back - self._sections = OrderedDict() - self._defaults = OrderedDict() + cp.RawConfigParser.__init__(self, dict_type=OrderedDict) + + # Used in python 3, needs to stay in sync with sections for underlying implementation to work + if not hasattr(self, '_proxies'): + self._proxies = self._dict() self._file_or_files = file_or_files self._read_only = read_only @@ -171,7 +190,7 @@ class GitConfigParser(cp.RawConfigParser, object): "Write-ConfigParsers can operate on a single file only, multiple files have been passed") # END single file check - if not isinstance(file_or_files, basestring): + if not isinstance(file_or_files, string_types): file_or_files = file_or_files.name # END get filename from handle/stream # initialize lock base - we want to write @@ -182,9 +201,16 @@ class GitConfigParser(cp.RawConfigParser, object): def __del__(self): """Write pending changes if required and release locks""" + # NOTE: only consistent in PY2 + self.release() + + def release(self): + """Flush changes and release the configuration write lock. This instance must not be used anymore afterwards. + In Python 3, it's required to explicitly release locks and flush changes, as __del__ is not called + deterministically anymore.""" # checking for the lock here makes sure we do not raise during write() # in case an invalid parser was created who could not get a lock - if self.read_only or not self._lock._has_lock(): + if self.read_only or (self._lock and not self._lock._has_lock()): return try: @@ -192,6 +218,11 @@ class GitConfigParser(cp.RawConfigParser, object): self.write() except IOError: log.error("Exception during destruction of GitConfigParser", exc_info=True) + except ReferenceError: + # This happens in PY3 ... and usually means that some state cannot be written + # as the sections dict cannot be iterated + # Usually when shutting down the interpreter, don'y know how to fix this + pass finally: self._lock._release_lock() @@ -214,7 +245,8 @@ class GitConfigParser(cp.RawConfigParser, object): lineno = 0 e = None # None, or an exception while True: - line = fp.readline() + # we assume to read binary ! + line = fp.readline().decode(defenc) if not line: break lineno = lineno + 1 @@ -234,9 +266,9 @@ class GitConfigParser(cp.RawConfigParser, object): elif sectname == cp.DEFAULTSECT: cursect = self._defaults else: - # THE ONLY LINE WE CHANGED ! - cursect = OrderedDict((('__name__', sectname),)) + cursect = self._dict((('__name__', sectname),)) self._sections[sectname] = cursect + self._proxies[sectname] = None # So sections can't start with a continuation line optname = None # no section header in the file? @@ -287,7 +319,7 @@ class GitConfigParser(cp.RawConfigParser, object): # assume a path if it is not a file-object if not hasattr(file_object, "seek"): try: - fp = open(file_object) + fp = open(file_object, 'rb') close_fp = True except IOError: continue @@ -306,16 +338,17 @@ class GitConfigParser(cp.RawConfigParser, object): """Write an .ini-format representation of the configuration state in git compatible format""" def write_section(name, section_dict): - fp.write("[%s]\n" % name) + fp.write(("[%s]\n" % name).encode(defenc)) for (key, value) in section_dict.items(): if key != "__name__": - fp.write("\t%s = %s\n" % (key, str(value).replace('\n', '\n\t'))) + fp.write(("\t%s = %s\n" % (key, str(value).replace('\n', '\n\t'))).encode(defenc)) # END if key is not __name__ # END section writing if self._defaults: write_section(cp.DEFAULTSECT, self._defaults) - map(lambda t: write_section(t[0], t[1]), self._sections.items()) + for name, value in self._sections.items(): + write_section(name, value) @needs_values def write(self): @@ -329,12 +362,12 @@ class GitConfigParser(cp.RawConfigParser, object): close_fp = False # we have a physical file on disk, so get a lock - if isinstance(fp, (basestring, file)): + if isinstance(fp, string_types + (FileType, )): self._lock._obtain_lock() # END get lock for physical files if not hasattr(fp, "seek"): - fp = open(self._file_or_files, "w") + fp = open(self._file_or_files, "wb") close_fp = True else: fp.seek(0) @@ -363,8 +396,7 @@ class GitConfigParser(cp.RawConfigParser, object): @set_dirty_and_flush_changes def add_section(self, section): """Assures added options will stay in order""" - super(GitConfigParser, self).add_section(section) - self._sections[section] = OrderedDict() + return super(GitConfigParser, self).add_section(section) @property def read_only(self): @@ -387,7 +419,7 @@ class GitConfigParser(cp.RawConfigParser, object): return default raise - types = (long, float) + types = (int, float) for numtype in types: try: val = numtype(valuestr) @@ -408,7 +440,7 @@ class GitConfigParser(cp.RawConfigParser, object): if vl == 'true': return True - if not isinstance(valuestr, basestring): + if not isinstance(valuestr, string_types): raise TypeError("Invalid value type: only int, long, float and str are allowed", valuestr) return valuestr @@ -1,14 +1,8 @@ """Module with our own gitdb implementation - it uses the git command""" -from exc import ( - GitCommandError, - BadObject -) - from gitdb.base import ( OInfo, OStream ) - from gitdb.util import ( bin_to_hex, hex_to_bin @@ -16,6 +10,11 @@ from gitdb.util import ( from gitdb.db import GitDB from gitdb.db import LooseObjectDB +from .exc import ( + GitCommandError, + BadObject +) + __all__ = ('GitCmdObjectDB', 'GitDB') diff --git a/git/diff.py b/git/diff.py index 5325ad6b..3c4e8529 100644 --- a/git/diff.py +++ b/git/diff.py @@ -3,13 +3,15 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php - import re -from objects.blob import Blob -from objects.util import mode_str_to_int from gitdb.util import hex_to_bin +from .objects.blob import Blob +from .objects.util import mode_str_to_int + +from git.compat import defenc + __all__ = ('Diffable', 'DiffIndex', 'Diff') @@ -195,7 +197,7 @@ class Diff(object): """, re.VERBOSE | re.MULTILINE) # can be used for comparisons NULL_HEX_SHA = "0" * 40 - NULL_BIN_SHA = "\0" * 20 + NULL_BIN_SHA = b"\0" * 20 __slots__ = ("a_blob", "b_blob", "a_mode", "b_mode", "new_file", "deleted_file", "rename_from", "rename_to", "diff") @@ -294,7 +296,7 @@ class Diff(object): :param stream: result of 'git diff' as a stream (supporting file protocol) :return: git.DiffIndex """ # for now, we have to bake the stream - text = stream.read() + text = stream.read().decode(defenc) index = DiffIndex() diff_header = cls.re_header.match @@ -323,6 +325,7 @@ class Diff(object): # :100644 100644 687099101... 37c5e30c8... M .gitignore index = DiffIndex() for line in stream: + line = line.decode(defenc) if not line.startswith(":"): continue # END its not a valid diff line @@ -7,6 +7,8 @@ from gitdb.exc import * # NOQA +from git.compat import defenc + class InvalidGitRepositoryError(Exception): @@ -29,10 +31,12 @@ class GitCommandError(Exception): self.command = command def __str__(self): - ret = "'%s' returned exit status %i: %s" % \ - (' '.join(str(i) for i in self.command), self.status, self.stderr) - if self.stdout is not None: - ret += "\nstdout: %s" % self.stdout + ret = "'%s' returned with exit code %i" % \ + (' '.join(str(i) for i in self.command), self.status) + if self.stderr: + ret += "\nstderr: '%s'" % self.stderr.decode(defenc) + if self.stdout: + ret += "\nstdout: '%s'" % self.stdout.decode(defenc) return ret diff --git a/git/index/base.py b/git/index/base.py index fdcfcd12..cc883469 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -8,16 +8,16 @@ import os import sys import subprocess import glob -from cStringIO import StringIO +from io import BytesIO from stat import S_ISLNK -from typ import ( +from .typ import ( BaseIndexEntry, IndexEntry, ) -from util import ( +from .util import ( TemporaryFileSwap, post_clear_cache, default_index, @@ -25,7 +25,6 @@ from util import ( ) import git.diff as diff - from git.exc import ( GitCommandError, CheckoutError @@ -40,6 +39,14 @@ from git.objects import ( ) from git.objects.util import Serializable +from git.compat import ( + izip, + xrange, + string_types, + force_bytes, + defenc, + mviter +) from git.util import ( LazyMixin, @@ -49,7 +56,7 @@ from git.util import ( to_native_path_linux, ) -from fun import ( +from .fun import ( entry_key, write_cache, read_cache, @@ -62,7 +69,6 @@ from fun import ( from gitdb.base import IStream from gitdb.db import MemoryDB from gitdb.util import to_bin_sha -from itertools import izip __all__ = ('IndexFile', 'CheckoutError') @@ -101,7 +107,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): repository's index on demand.""" self.repo = repo self.version = self._VERSION - self._extension_data = '' + self._extension_data = b'' self._file_path = file_path or self._index_path() def _set_cache_(self, attr): @@ -161,9 +167,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): def _entries_sorted(self): """:return: list of entries, in a sorted fashion, first by path, then by stage""" - entries_sorted = self.entries.values() - entries_sorted.sort(key=lambda e: (e.path, e.stage)) # use path/stage as sort key - return entries_sorted + return sorted(self.entries.values(), key=lambda e: (e.path, e.stage)) def _serialize(self, stream, ignore_tree_extension_data=False): entries = self._entries_sorted() @@ -395,7 +399,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): fprogress(filepath, False, item) rval = None try: - proc.stdin.write("%s\n" % filepath) + proc.stdin.write(("%s\n" % filepath).encode(defenc)) except IOError: # pipe broke, usually because some error happend raise fmakeexc() @@ -414,7 +418,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): Function(t) returning True if tuple(stage, Blob) should be yielded by the iterator. A default filter, the BlobFilter, allows you to yield blobs only if they match a given list of paths. """ - for entry in self.entries.itervalues(): + for entry in mviter(self.entries): blob = entry.to_blob(self.repo) blob.size = entry.size output = (entry.stage, blob) @@ -439,7 +443,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): for stage, blob in self.iter_blobs(is_unmerged_blob): path_map.setdefault(blob.path, list()).append((stage, blob)) # END for each unmerged blob - for l in path_map.itervalues(): + for l in mviter(path_map): l.sort() return path_map @@ -542,7 +546,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): entries = list() for item in items: - if isinstance(item, basestring): + if isinstance(item, string_types): paths.append(self._to_relative_path(item)) elif isinstance(item, (Blob, Submodule)): entries.append(BaseIndexEntry.from_blob(item)) @@ -559,7 +563,8 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): st = os.lstat(filepath) # handles non-symlinks as well stream = None if S_ISLNK(st.st_mode): - stream = StringIO(os.readlink(filepath)) + # in PY3, readlink is string, but we need bytes. In PY2, it's just OS encoded bytes, we assume UTF-8 + stream = BytesIO(force_bytes(os.readlink(filepath), encoding='utf-8')) else: stream = open(filepath, 'rb') # END handle stream @@ -753,7 +758,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): for item in items: if isinstance(item, (BaseIndexEntry, (Blob, Submodule))): paths.append(self._to_relative_path(item.path)) - elif isinstance(item, basestring): + elif isinstance(item, string_types): paths.append(self._to_relative_path(item)) else: raise TypeError("Invalid item type: %r" % item) @@ -855,7 +860,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): # parse result - first 0:n/2 lines are 'checking ', the remaining ones # are the 'renaming' ones which we parse - for ln in xrange(len(mvlines) / 2, len(mvlines)): + for ln in xrange(int(len(mvlines) / 2), len(mvlines)): tokens = mvlines[ln].split(' to ') assert len(tokens) == 2, "Too many tokens in %s" % mvlines[ln] @@ -953,6 +958,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): if not stderr: return # line contents: + stderr = stderr.decode(defenc) # git-checkout-index: this already exists failed_files = list() failed_reasons = list() @@ -1001,11 +1007,11 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): proc = self.repo.git.checkout_index(*args, **kwargs) proc.wait() fprogress(None, True, None) - rval_iter = (e.path for e in self.entries.itervalues()) + rval_iter = (e.path for e in mviter(self.entries)) handle_stderr(proc, rval_iter) return rval_iter else: - if isinstance(paths, basestring): + if isinstance(paths, string_types): paths = [paths] # make sure we have our entries loaded before we start checkout_index @@ -1031,7 +1037,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): dir = co_path if not dir.endswith('/'): dir += '/' - for entry in self.entries.itervalues(): + for entry in mviter(self.entries): if entry.path.startswith(dir): p = entry.path self._write_path_to_stdin(proc, p, p, make_exc, @@ -1141,7 +1147,7 @@ class IndexFile(LazyMixin, diff.Diffable, Serializable): # index against anything but None is a reverse diff with the respective # item. Handle existing -R flags properly. Transform strings to the object # so that we can call diff on it - if isinstance(other, basestring): + if isinstance(other, string_types): other = self.repo.rev_parse(other) # END object conversion diff --git a/git/index/fun.py b/git/index/fun.py index eec90519..f0dee961 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -12,7 +12,7 @@ from stat import ( S_IFGITLINK = S_IFLNK | S_IFDIR # a submodule -from cStringIO import StringIO +from io import BytesIO from git.util import IndexFileSHA1Writer from git.exc import UnmergedEntriesError @@ -22,7 +22,7 @@ from git.objects.fun import ( traverse_trees_recursive ) -from typ import ( +from .typ import ( BaseIndexEntry, IndexEntry, CE_NAMEMASK, @@ -30,13 +30,14 @@ from typ import ( ) CE_NAMEMASK_INV = ~CE_NAMEMASK -from util import ( +from .util import ( pack, unpack ) from gitdb.base import IStream from gitdb.typ import str_tree_type +from git.compat import defenc __all__ = ('write_cache', 'read_cache', 'write_tree_from_cache', 'entry_key', 'stat_mode_to_index_mode', 'S_IFGITLINK') @@ -49,7 +50,7 @@ def stat_mode_to_index_mode(mode): return S_IFLNK if S_ISDIR(mode) or S_IFMT(mode) == S_IFGITLINK: # submodules return S_IFGITLINK - return S_IFREG | 0644 | (mode & 0100) # blobs with or without executable bit + return S_IFREG | 0o644 | (mode & 0o100) # blobs with or without executable bit def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1Writer): @@ -72,7 +73,7 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1 # header version = 2 - write("DIRC") + write(b"DIRC") write(pack(">LL", version, len(entries))) # body @@ -86,9 +87,9 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1 flags = plen | (entry[2] & CE_NAMEMASK_INV) # clear possible previous values write(pack(">LLLLLL20sH", entry[6], entry[7], entry[0], entry[8], entry[9], entry[10], entry[1], flags)) - write(path) + write(path.encode(defenc)) real_size = ((tell() - beginoffset + 8) & ~7) - write("\0" * ((beginoffset + real_size) - tell())) + write(b"\0" * ((beginoffset + real_size) - tell())) # END for each entry # write previously cached extensions data @@ -102,7 +103,7 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1 def read_header(stream): """Return tuple(version_long, num_entries) from the given stream""" type_id = stream.read(4) - if type_id != "DIRC": + if type_id != b"DIRC": raise AssertionError("Invalid index file header: %r" % type_id) version, num_entries = unpack(">LL", stream.read(4 * 2)) @@ -142,7 +143,7 @@ def read_cache(stream): (dev, ino, mode, uid, gid, size, sha, flags) = \ unpack(">LLLLLL20sH", read(20 + 4 * 6 + 2)) path_size = flags & CE_NAMEMASK - path = read(path_size) + path = read(path_size).decode(defenc) real_size = ((tell() - beginoffset + 8) & ~7) read((beginoffset + real_size) - tell()) @@ -218,7 +219,7 @@ def write_tree_from_cache(entries, odb, sl, si=0): # END for each entry # finally create the tree - sio = StringIO() + sio = BytesIO() tree_to_stream(tree_items, sio.write) sio.seek(0) diff --git a/git/index/typ.py b/git/index/typ.py index 222252c5..0998ecb0 100644 --- a/git/index/typ.py +++ b/git/index/typ.py @@ -1,15 +1,14 @@ """Module with additional types used by the index""" -from util import ( +from binascii import b2a_hex + +from .util import ( pack, unpack ) +from git.objects import Blob -from binascii import ( - b2a_hex, -) -from git.objects import Blob __all__ = ('BlobFilter', 'BaseIndexEntry', 'IndexEntry') #{ Invariants @@ -76,7 +75,7 @@ class BaseIndexEntry(tuple): @property def hexsha(self): """hex version of our sha""" - return b2a_hex(self[1]) + return b2a_hex(self[1]).decode('ascii') @property def stage(self): diff --git a/git/objects/__init__.py b/git/objects/__init__.py index 70fc52cb..ee642876 100644 --- a/git/objects/__init__.py +++ b/git/objects/__init__.py @@ -7,9 +7,10 @@ import inspect from .base import * # Fix import dependency - add IndexObject to the util module, so that it can be # imported by the submodule.base -from .submodule import util -util.IndexObject = IndexObject -util.Object = Object +from .submodule import util as smutil +smutil.IndexObject = IndexObject +smutil.Object = Object +del(smutil) from .submodule.base import * from .submodule.root import * diff --git a/git/objects/base.py b/git/objects/base.py index 20147e57..3f595d9d 100644 --- a/git/objects/base.py +++ b/git/objects/base.py @@ -3,8 +3,8 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php +from .util import get_object_type_by_name from git.util import LazyMixin, join_path_native, stream_copy -from util import get_object_type_by_name from gitdb.util import ( bin_to_hex, basename @@ -21,7 +21,7 @@ class Object(LazyMixin): """Implements an Object which may be Blobs, Trees, Commits and Tags""" NULL_HEX_SHA = '0' * 40 - NULL_BIN_SHA = '\0' * 20 + NULL_BIN_SHA = b'\0' * 20 TYPES = (dbtyp.str_blob_type, dbtyp.str_tree_type, dbtyp.str_commit_type, dbtyp.str_tag_type) __slots__ = ("repo", "binsha", "size") @@ -60,7 +60,7 @@ class Object(LazyMixin): :param sha1: 20 byte binary sha1""" if sha1 == cls.NULL_BIN_SHA: # the NULL binsha is always the root commit - return get_object_type_by_name('commit')(repo, sha1) + return get_object_type_by_name(b'commit')(repo, sha1) # END handle special case oinfo = repo.odb.info(sha1) inst = get_object_type_by_name(oinfo.type)(repo, oinfo.binsha) @@ -94,7 +94,7 @@ class Object(LazyMixin): def __str__(self): """:return: string of our SHA1 as understood by all git commands""" - return bin_to_hex(self.binsha) + return self.hexsha def __repr__(self): """:return: string with pythonic representation of our object""" @@ -103,7 +103,8 @@ class Object(LazyMixin): @property def hexsha(self): """:return: 40 byte hex version of our 20 byte binary sha""" - return bin_to_hex(self.binsha) + # b2a_hex produces bytes + return bin_to_hex(self.binsha).decode('ascii') @property def data_stream(self): diff --git a/git/objects/blob.py b/git/objects/blob.py index b05e5b84..322f6992 100644 --- a/git/objects/blob.py +++ b/git/objects/blob.py @@ -3,9 +3,8 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php - from mimetypes import guess_type -import base +from . import base __all__ = ('Blob', ) diff --git a/git/objects/commit.py b/git/objects/commit.py index 9c733695..8f93d1b9 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -4,6 +4,8 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php +from gitdb import IStream +from gitdb.util import hex_to_bin from git.util import ( Actor, Iterable, @@ -11,26 +13,24 @@ from git.util import ( finalize_process ) from git.diff import Diffable -from tree import Tree -from gitdb import IStream -from cStringIO import StringIO -import base -from gitdb.util import ( - hex_to_bin -) -from util import ( +from .tree import Tree +from . import base +from .util import ( Traversable, Serializable, parse_date, altz_to_utctz_str, parse_actor_and_date ) +from git.compat import text_type + from time import ( time, altzone ) import os +from io import BytesIO import logging log = logging.getLogger('git.objects.commit') @@ -62,7 +62,7 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): "author", "authored_date", "author_tz_offset", "committer", "committed_date", "committer_tz_offset", "message", "parents", "encoding", "gpgsig") - _id_attribute_ = "binsha" + _id_attribute_ = "hexsha" def __init__(self, repo, binsha, tree=None, author=None, authored_date=None, author_tz_offset=None, committer=None, committed_date=None, committer_tz_offset=None, @@ -133,7 +133,7 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): if attr in Commit.__slots__: # read the data in a chunk, its faster - then provide a file wrapper binsha, typename, self.size, stream = self.repo.odb.stream(self.binsha) - self._deserialize(StringIO(stream.read())) + self._deserialize(BytesIO(stream.read())) else: super(Commit, self)._set_cache_(attr) # END handle attrs @@ -345,7 +345,7 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): committer, committer_time, committer_offset, message, parent_commits, conf_encoding) - stream = StringIO() + stream = BytesIO() new_commit._serialize(stream) streamlen = stream.tell() stream.seek(0) @@ -373,43 +373,36 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): def _serialize(self, stream): write = stream.write - write("tree %s\n" % self.tree) + write(("tree %s\n" % self.tree).encode('ascii')) for p in self.parents: - write("parent %s\n" % p) + write(("parent %s\n" % p).encode('ascii')) a = self.author aname = a.name - if isinstance(aname, unicode): - aname = aname.encode(self.encoding) - # END handle unicode in name - c = self.committer fmt = "%s %s <%s> %s %s\n" - write(fmt % ("author", aname, a.email, - self.authored_date, - altz_to_utctz_str(self.author_tz_offset))) + write((fmt % ("author", aname, a.email, + self.authored_date, + altz_to_utctz_str(self.author_tz_offset))).encode(self.encoding)) # encode committer aname = c.name - if isinstance(aname, unicode): - aname = aname.encode(self.encoding) - # END handle unicode in name - write(fmt % ("committer", aname, c.email, - self.committed_date, - altz_to_utctz_str(self.committer_tz_offset))) + write((fmt % ("committer", aname, c.email, + self.committed_date, + altz_to_utctz_str(self.committer_tz_offset))).encode(self.encoding)) if self.encoding != self.default_encoding: - write("encoding %s\n" % self.encoding) + write(("encoding %s\n" % self.encoding).encode('ascii')) if self.gpgsig: - write("gpgsig") + write(b"gpgsig") for sigline in self.gpgsig.rstrip("\n").split("\n"): - write(" " + sigline + "\n") + write((" " + sigline + "\n").encode('ascii')) - write("\n") + write(b"\n") # write plain bytes, be sure its encoded according to our encoding - if isinstance(self.message, unicode): + if isinstance(self.message, text_type): write(self.message.encode(self.encoding)) else: write(self.message) @@ -426,23 +419,25 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): next_line = None while True: parent_line = readline() - if not parent_line.startswith('parent'): + if not parent_line.startswith(b'parent'): next_line = parent_line break # END abort reading parents - self.parents.append(type(self)(self.repo, hex_to_bin(parent_line.split()[-1]))) + self.parents.append(type(self)(self.repo, hex_to_bin(parent_line.split()[-1].decode('ascii')))) # END for each parent line self.parents = tuple(self.parents) - self.author, self.authored_date, self.author_tz_offset = parse_actor_and_date(next_line) - self.committer, self.committed_date, self.committer_tz_offset = parse_actor_and_date(readline()) + # we don't know actual author encoding before we have parsed it, so keep the lines around + author_line = next_line + committer_line = readline() # we might run into one or more mergetag blocks, skip those for now next_line = readline() - while next_line.startswith('mergetag '): + while next_line.startswith(b'mergetag '): next_line = readline() while next_line.startswith(' '): next_line = readline() + # end skip mergetags # now we can have the encoding line, or an empty line followed by the optional # message. @@ -451,39 +446,40 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): # read headers enc = next_line buf = enc.strip() - while buf != "": - if buf[0:10] == "encoding ": - self.encoding = buf[buf.find(' ') + 1:] - elif buf[0:7] == "gpgsig ": - sig = buf[buf.find(' ') + 1:] + "\n" + while buf: + if buf[0:10] == b"encoding ": + self.encoding = buf[buf.find(' ') + 1:].decode('ascii') + elif buf[0:7] == b"gpgsig ": + sig = buf[buf.find(b' ') + 1:] + b"\n" is_next_header = False while True: sigbuf = readline() - if sigbuf == "": + if not sigbuf: break - if sigbuf[0:1] != " ": + if sigbuf[0:1] != b" ": buf = sigbuf.strip() is_next_header = True break sig += sigbuf[1:] - self.gpgsig = sig.rstrip("\n") + # end read all signature + self.gpgsig = sig.rstrip(b"\n").decode('ascii') if is_next_header: continue buf = readline().strip() - # decode the authors name + try: - self.author.name = self.author.name.decode(self.encoding) + self.author, self.authored_date, self.author_tz_offset = \ + parse_actor_and_date(author_line.decode(self.encoding)) except UnicodeDecodeError: - log.error("Failed to decode author name '%s' using encoding %s", self.author.name, self.encoding, + log.error("Failed to decode author line '%s' using encoding %s", author_line, self.encoding, exc_info=True) - # END handle author's encoding - # decode committer name try: - self.committer.name = self.committer.name.decode(self.encoding) + self.committer, self.committed_date, self.committer_tz_offset = \ + parse_actor_and_date(committer_line.decode(self.encoding)) except UnicodeDecodeError: - log.error("Failed to decode committer name '%s' using encoding %s", self.committer.name, self.encoding, + log.error("Failed to decode committer line '%s' using encoding %s", committer_line, self.encoding, exc_info=True) # END handle author's encoding @@ -495,6 +491,7 @@ class Commit(base.Object, Iterable, Diffable, Traversable, Serializable): except UnicodeDecodeError: log.error("Failed to decode message '%s' using encoding %s", self.message, self.encoding, exc_info=True) # END exception handling + return self #} END serializable implementation diff --git a/git/objects/fun.py b/git/objects/fun.py index 416a52e6..c04f80b5 100644 --- a/git/objects/fun.py +++ b/git/objects/fun.py @@ -1,5 +1,12 @@ """Module with functions which are supposed to be as fast as possible""" from stat import S_ISDIR +from git.compat import ( + byte_ord, + defenc, + xrange, + text_type, + bchr +) __all__ = ('tree_to_stream', 'tree_entries_from_data', 'traverse_trees_recursive', 'traverse_tree_recursive') @@ -13,13 +20,13 @@ def tree_to_stream(entries, write): bit_mask = 7 # 3 bits set for binsha, mode, name in entries: - mode_str = '' + mode_str = b'' for i in xrange(6): - mode_str = chr(((mode >> (i * 3)) & bit_mask) + ord_zero) + mode_str + mode_str = bchr(((mode >> (i * 3)) & bit_mask) + ord_zero) + mode_str # END for each 8 octal value # git slices away the first octal if its zero - if mode_str[0] == '0': + if byte_ord(mode_str[0]) == ord_zero: mode_str = mode_str[1:] # END save a byte @@ -28,17 +35,18 @@ def tree_to_stream(entries, write): # hence we must convert to an utf8 string for it to work properly. # According to my tests, this is exactly what git does, that is it just # takes the input literally, which appears to be utf8 on linux. - if isinstance(name, unicode): - name = name.encode("utf8") - write("%s %s\0%s" % (mode_str, name, binsha)) + if isinstance(name, text_type): + name = name.encode(defenc) + write(b''.join((mode_str, b' ', name, b'\0', binsha))) # END for each item def tree_entries_from_data(data): """Reads the binary representation of a tree and returns tuples of Tree items - :param data: data block with tree data + :param data: data block with tree data (as bytes) :return: list(tuple(binsha, mode, tree_relative_path), ...)""" ord_zero = ord('0') + space_ord = ord(' ') len_data = len(data) i = 0 out = list() @@ -48,10 +56,10 @@ def tree_entries_from_data(data): # read mode # Some git versions truncate the leading 0, some don't # The type will be extracted from the mode later - while data[i] != ' ': + while byte_ord(data[i]) != space_ord: # move existing mode integer up one level being 3 bits # and add the actual ordinal value of the character - mode = (mode << 3) + (ord(data[i]) - ord_zero) + mode = (mode << 3) + (byte_ord(data[i]) - ord_zero) i += 1 # END while reading mode @@ -61,7 +69,7 @@ def tree_entries_from_data(data): # parse name, it is NULL separated ns = i - while data[i] != '\0': + while byte_ord(data[i]) != 0: i += 1 # END while not reached NULL @@ -69,12 +77,9 @@ def tree_entries_from_data(data): # Only use the respective unicode object if the byte stream was encoded name = data[ns:i] try: - name_enc = name.decode("utf-8") + name = name.decode(defenc) except UnicodeDecodeError: pass - else: - if len(name) > len(name_enc): - name = name_enc # END handle encoding # byte is NULL, get next 20 diff --git a/git/objects/submodule/base.py b/git/objects/submodule/base.py index d6f8982b..0ec6f656 100644 --- a/git/objects/submodule/base.py +++ b/git/objects/submodule/base.py @@ -1,5 +1,5 @@ -import util -from util import ( +from . import util +from .util import ( mkhead, sm_name, sm_section, @@ -8,7 +8,7 @@ from util import ( find_first_remote_branch ) from git.objects.util import Traversable -from StringIO import StringIO # need a dict to set bloody .name field +from io import BytesIO # need a dict to set bloody .name field from git.util import ( Iterable, join_path_native, @@ -17,11 +17,15 @@ from git.util import ( rmtree ) -from git.config import SectionConstraint +from git.config import ( + SectionConstraint, + cp +) from git.exc import ( InvalidGitRepositoryError, NoSuchPathError ) +from git.compat import string_types import stat import git @@ -93,7 +97,7 @@ class Submodule(util.IndexObject, Iterable, Traversable): if url is not None: self._url = url if branch_path is not None: - assert isinstance(branch_path, basestring) + assert isinstance(branch_path, string_types) self._branch_path = branch_path if name is not None: self._name = name @@ -186,8 +190,8 @@ class Submodule(util.IndexObject, Iterable, Traversable): @classmethod def _sio_modules(cls, parent_commit): - """:return: Configuration file as StringIO - we only access it through the respective blob's data""" - sio = StringIO(parent_commit.tree[cls.k_modules_file].data_stream.read()) + """:return: Configuration file as BytesIO - we only access it through the respective blob's data""" + sio = BytesIO(parent_commit.tree[cls.k_modules_file].data_stream.read()) sio.name = cls.k_modules_file return sio @@ -301,6 +305,7 @@ class Submodule(util.IndexObject, Iterable, Traversable): writer.set_value(cls.k_head_option, br.path) sm._branch_path = br.path # END handle path + writer.release() del(writer) # we deliberatly assume that our head matches our index ! @@ -418,7 +423,9 @@ class Submodule(util.IndexObject, Iterable, Traversable): # the default implementation will be offended and not update the repository # Maybe this is a good way to assure it doesn't get into our way, but # we want to stay backwards compatible too ... . Its so redundant ! - self.repo.config_writer().set_value(sm_section(self.name), 'url', self.url) + writer = self.repo.config_writer() + writer.set_value(sm_section(self.name), 'url', self.url) + writer.release() # END handle dry_run # END handle initalization @@ -575,6 +582,7 @@ class Submodule(util.IndexObject, Iterable, Traversable): writer = self.config_writer(index=index) # auto-write writer.set_value('path', module_path) self.path = module_path + writer.release() del(writer) # END handle configuration flag except Exception: @@ -699,8 +707,12 @@ class Submodule(util.IndexObject, Iterable, Traversable): # now git config - need the config intact, otherwise we can't query # inforamtion anymore - self.repo.config_writer().remove_section(sm_section(self.name)) - self.config_writer().remove_section() + writer = self.repo.config_writer() + writer.remove_section(sm_section(self.name)) + writer.release() + writer = self.config_writer() + writer.remove_section() + writer.release() # END delete configuration # void our data not to delay invalid access @@ -799,14 +811,18 @@ class Submodule(util.IndexObject, Iterable, Traversable): """ :return: True if the submodule exists, False otherwise. Please note that a submodule may exist (in the .gitmodules file) even though its module - doesn't exist""" + doesn't exist on disk""" # keep attributes for later, and restore them if we have no valid data # this way we do not actually alter the state of the object loc = locals() for attr in self._cache_attrs: - if hasattr(self, attr): - loc[attr] = getattr(self, attr) - # END if we have the attribute cache + try: + if hasattr(self, attr): + loc[attr] = getattr(self, attr) + # END if we have the attribute cache + except cp.NoSectionError: + # on PY3, this can happen apparently ... don't know why this doesn't happen on PY2 + pass # END for each attr self._clear_cache() diff --git a/git/objects/submodule/root.py b/git/objects/submodule/root.py index 708749c7..8c9afff1 100644 --- a/git/objects/submodule/root.py +++ b/git/objects/submodule/root.py @@ -1,5 +1,8 @@ -from base import Submodule, UpdateProgress -from util import ( +from .base import ( + Submodule, + UpdateProgress +) +from .util import ( find_first_remote_branch ) from git.exc import InvalidGitRepositoryError diff --git a/git/objects/submodule/util.py b/git/objects/submodule/util.py index 01bd03b3..5604dec7 100644 --- a/git/objects/submodule/util.py +++ b/git/objects/submodule/util.py @@ -1,7 +1,7 @@ import git from git.exc import InvalidGitRepositoryError from git.config import GitConfigParser -from StringIO import StringIO +from io import BytesIO import weakref __all__ = ('sm_section', 'sm_name', 'mkhead', 'unbare_repo', 'find_first_remote_branch', @@ -83,7 +83,7 @@ class SubmoduleConfigParser(GitConfigParser): """Flush changes in our configuration file to the index""" assert self._smref is not None # should always have a file here - assert not isinstance(self._file_or_files, StringIO) + assert not isinstance(self._file_or_files, BytesIO) sm = self._smref() if sm is not None: diff --git a/git/objects/tag.py b/git/objects/tag.py index 3c379579..c8684447 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -4,12 +4,13 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php """ Module containing all object based types. """ -import base -from gitdb.util import hex_to_bin -from util import ( +from . import base +from .util import ( get_object_type_by_name, parse_actor_and_date ) +from gitdb.util import hex_to_bin +from git.compat import defenc __all__ = ("TagObject", ) @@ -52,11 +53,12 @@ class TagObject(base.Object): """Cache all our attributes at once""" if attr in TagObject.__slots__: ostream = self.repo.odb.stream(self.binsha) - lines = ostream.read().splitlines() + lines = ostream.read().decode(defenc).splitlines() obj, hexsha = lines[0].split(" ") # object <hexsha> type_token, type_name = lines[1].split(" ") # type <type_name> - self.object = get_object_type_by_name(type_name)(self.repo, hex_to_bin(hexsha)) + self.object = \ + get_object_type_by_name(type_name.encode('ascii'))(self.repo, hex_to_bin(hexsha)) self.tag = lines[2][4:] # tag <tag name> diff --git a/git/objects/tree.py b/git/objects/tree.py index c77e6056..f9bee01e 100644 --- a/git/objects/tree.py +++ b/git/objects/tree.py @@ -3,22 +3,21 @@ # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -import util -from base import IndexObject from git.util import join_path -from blob import Blob -from submodule.base import Submodule import git.diff as diff +from gitdb.util import to_bin_sha -from fun import ( +from . import util +from .base import IndexObject +from .blob import Blob +from .submodule.base import Submodule +from git.compat import string_types + +from .fun import ( tree_entries_from_data, tree_to_stream ) -from gitdb.util import ( - to_bin_sha, -) - __all__ = ("TreeModifier", "Tree") @@ -160,7 +159,7 @@ class Tree(IndexObject, diff.Diffable, util.Traversable, util.Serializable): raise TypeError("Unknown mode %o found in tree data for path '%s'" % (mode, path)) # END for each item - def __div__(self, file): + def join(self, file): """Find the named object in this tree's contents :return: ``git.Blob`` or ``git.Tree`` or ``git.Submodule`` @@ -193,6 +192,14 @@ class Tree(IndexObject, diff.Diffable, util.Traversable, util.Serializable): raise KeyError(msg % file) # END handle long paths + def __div__(self, file): + """For PY2 only""" + return self.join(file) + + def __truediv__(self, file): + """For PY3 only""" + return self.join(file) + @property def trees(self): """:return: list(Tree, ...) list of trees directly below this tree""" @@ -234,9 +241,9 @@ class Tree(IndexObject, diff.Diffable, util.Traversable, util.Serializable): info = self._cache[item] return self._map_id_to_type[info[1] >> 12](self.repo, info[0], info[1], join_path(self.path, info[2])) - if isinstance(item, basestring): + if isinstance(item, string_types): # compatability - return self.__div__(item) + return self.join(item) # END index is basestring raise TypeError("Invalid index type: %r" % item) diff --git a/git/objects/util.py b/git/objects/util.py index fdf9622b..cefef862 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -46,17 +46,17 @@ def get_object_type_by_name(object_type_name): :param object_type_name: Member of TYPES :raise ValueError: In case object_type_name is unknown""" - if object_type_name == "commit": - import commit + if object_type_name == b"commit": + from . import commit return commit.Commit - elif object_type_name == "tag": - import tag + elif object_type_name == b"tag": + from . import tag return tag.TagObject - elif object_type_name == "blob": - import blob + elif object_type_name == b"blob": + from . import blob return blob.Blob - elif object_type_name == "tree": - import tree + elif object_type_name == b"tree": + from . import tree return tree.Tree else: raise ValueError("Cannot handle unknown object type: %s" % object_type_name) diff --git a/git/refs/head.py b/git/refs/head.py index 25c994a3..750d15b6 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -1,12 +1,10 @@ -from symbolic import SymbolicReference -from reference import Reference - from git.config import SectionConstraint - from git.util import join_path - from git.exc import GitCommandError +from .symbolic import SymbolicReference +from .reference import Reference + __all__ = ["HEAD", "Head"] @@ -150,6 +148,7 @@ class Head(Reference): writer.set_value(self.k_config_remote, remote_reference.remote_name) writer.set_value(self.k_config_remote_ref, Head.to_full_path(remote_reference.remote_head)) # END handle ref value + writer.release() return self diff --git a/git/refs/log.py b/git/refs/log.py index 07465e6f..7708dd73 100644 --- a/git/refs/log.py +++ b/git/refs/log.py @@ -17,6 +17,11 @@ from git.objects.util import ( Serializable, altz_to_utctz_str, ) +from git.compat import ( + xrange, + string_types, + defenc +) import time import re @@ -34,9 +39,8 @@ class RefLogEntry(tuple): def __repr__(self): """Representation of ourselves in git reflog format""" act = self.actor - name = act.name.encode('utf-8') time = self.time - return self._fmt % (self.oldhexsha, self.newhexsha, name, act.email, + return self._fmt % (self.oldhexsha, self.newhexsha, act.name, act.email, time[0], altz_to_utctz_str(time[1]), self.message) @property @@ -78,7 +82,7 @@ class RefLogEntry(tuple): @classmethod def from_line(cls, line): """:return: New RefLogEntry instance from the given revlog line. - :param line: line without trailing newline + :param line: line bytes without trailing newline :raise ValueError: If line could not be parsed""" fields = line.split('\t', 1) if len(fields) == 1: @@ -89,6 +93,7 @@ class RefLogEntry(tuple): raise ValueError("Line must have up to two TAB-separated fields." " Got %s" % repr(line)) # END handle first split + oldhexsha = info[:40] newhexsha = info[41:81] for hexsha in (oldhexsha, newhexsha): @@ -174,7 +179,7 @@ class RefLog(list, Serializable): :param stream: file-like object containing the revlog in its native format or basestring instance pointing to a file to read""" new_entry = RefLogEntry.from_line - if isinstance(stream, basestring): + if isinstance(stream, string_types): stream = file_contents_ro_filepath(stream) # END handle stream type while True: @@ -253,15 +258,18 @@ class RefLog(list, Serializable): # END handle sha type assure_directory_exists(filepath, is_file=True) committer = isinstance(config_reader, Actor) and config_reader or Actor.committer(config_reader) - entry = RefLogEntry( - (bin_to_hex(oldbinsha), bin_to_hex(newbinsha), committer, (int(time.time()), time.altzone), message)) + entry = RefLogEntry(( + bin_to_hex(oldbinsha).decode('ascii'), + bin_to_hex(newbinsha).decode('ascii'), + committer, (int(time.time()), time.altzone), message + )) lf = LockFile(filepath) lf._obtain_lock_or_raise() - fd = open(filepath, 'a') + fd = open(filepath, 'ab') try: - fd.write(repr(entry)) + fd.write(repr(entry).encode(defenc)) finally: fd.close() lf._release_lock() @@ -286,7 +294,7 @@ class RefLog(list, Serializable): # write all entries for e in self: - write(repr(e)) + write(repr(e).encode(defenc)) # END for each entry def _deserialize(self, stream): diff --git a/git/refs/reference.py b/git/refs/reference.py index b07ac0cd..8741ebb9 100644 --- a/git/refs/reference.py +++ b/git/refs/reference.py @@ -1,8 +1,9 @@ -from symbolic import SymbolicReference from git.util import ( LazyMixin, Iterable, ) +from .symbolic import SymbolicReference + __all__ = ["Reference"] diff --git a/git/refs/remote.py b/git/refs/remote.py index e3827ad9..b692e6df 100644 --- a/git/refs/remote.py +++ b/git/refs/remote.py @@ -1,7 +1,8 @@ -from head import Head from git.util import join_path from gitdb.util import join +from .head import Head + import os diff --git a/git/refs/symbolic.py b/git/refs/symbolic.py index e0f5531a..cbb129d4 100644 --- a/git/refs/symbolic.py +++ b/git/refs/symbolic.py @@ -1,4 +1,5 @@ import os + from git.objects import Object, Commit from git.util import ( join_path, @@ -18,8 +19,12 @@ from gitdb.util import ( hex_to_bin, LockedFD ) +from git.compat import ( + string_types, + defenc +) -from log import RefLog +from .log import RefLog __all__ = ["SymbolicReference"] @@ -77,10 +82,10 @@ class SymbolicReference(object): @classmethod def _iter_packed_refs(cls, repo): - """Returns an iterator yielding pairs of sha1/path pairs for the corresponding refs. + """Returns an iterator yielding pairs of sha1/path pairs (as bytes) for the corresponding refs. :note: The packed refs file will be kept open as long as we iterate""" try: - fp = open(cls._get_packed_refs_path(repo), 'rb') + fp = open(cls._get_packed_refs_path(repo), 'rt') for line in fp: line = line.strip() if not line: @@ -121,12 +126,12 @@ class SymbolicReference(object): @classmethod def _get_ref_info(cls, repo, ref_path): - """Return: (sha, target_ref_path) if available, the sha the file at + """Return: (str(sha), str(target_ref_path)) if available, the sha the file at rela_path points to, or None. target_ref_path is the reference we point to, or None""" tokens = None try: - fp = open(join(repo.git_dir, ref_path), 'r') + fp = open(join(repo.git_dir, ref_path), 'rt') value = fp.read().rstrip() fp.close() # Don't only split on spaces, but on whitespace, which allows to parse lines like @@ -139,7 +144,8 @@ class SymbolicReference(object): for sha, path in cls._iter_packed_refs(repo): if path != ref_path: continue - tokens = (sha, path) + # sha will be used + tokens = sha, path break # END for each packed ref # END handle packed refs @@ -273,7 +279,7 @@ class SymbolicReference(object): elif isinstance(ref, Object): obj = ref write_value = ref.hexsha - elif isinstance(ref, basestring): + elif isinstance(ref, string_types): try: obj = self.repo.rev_parse(ref + "^{}") # optionally deref tags write_value = obj.hexsha @@ -303,7 +309,7 @@ class SymbolicReference(object): lfd = LockedFD(fpath) fd = lfd.open(write=True, stream=True) - fd.write(write_value) + fd.write(write_value.encode('ascii')) lfd.commit() # Adjust the reflog @@ -424,6 +430,7 @@ class SymbolicReference(object): # in the line # If we deleted the last line and this one is a tag-reference object, # we drop it as well + line = line.decode(defenc) if (line.startswith('#') or full_ref_path not in line) and \ (not dropped_last_line or dropped_last_line and not line.startswith('^')): new_lines.append(line) @@ -441,7 +448,7 @@ class SymbolicReference(object): if made_change: # write-binary is required, otherwise windows will # open the file in text mode and change LF to CRLF ! - open(pack_file_path, 'wb').writelines(new_lines) + open(pack_file_path, 'wb').writelines(l.encode(defenc) for l in new_lines) # END write out file # END open exception handling # END handle deletion @@ -473,7 +480,7 @@ class SymbolicReference(object): target_data = target.path if not resolve: target_data = "ref: " + target_data - existing_data = open(abs_ref_path, 'rb').read().strip() + existing_data = open(abs_ref_path, 'rb').read().decode(defenc).strip() if existing_data != target_data: raise OSError("Reference at %r does already exist, pointing to %r, requested was %r" % (full_ref_path, existing_data, target_data)) diff --git a/git/refs/tag.py b/git/refs/tag.py index 6509c891..3334e53c 100644 --- a/git/refs/tag.py +++ b/git/refs/tag.py @@ -1,4 +1,4 @@ -from reference import Reference +from .reference import Reference __all__ = ["TagReference", "Tag"] diff --git a/git/remote.py b/git/remote.py index 44b7ffaa..484bc031 100644 --- a/git/remote.py +++ b/git/remote.py @@ -5,33 +5,35 @@ # the BSD License: http://www.opensource.org/licenses/bsd-license.php # Module implementing a remote object allowing easy access to git remotes +import re +import os -from exc import GitCommandError -from ConfigParser import NoOptionError -from config import SectionConstraint - -from git.util import ( - LazyMixin, - Iterable, - IterableList, - RemoteProgress +from .exc import GitCommandError +from .config import ( + SectionConstraint, + cp, ) - -from refs import ( +from .refs import ( Reference, RemoteReference, SymbolicReference, TagReference ) + +from git.util import ( + LazyMixin, + Iterable, + IterableList, + RemoteProgress +) from git.util import ( join_path, finalize_process ) from gitdb.util import join +from git.compat import defenc -import re -import os __all__ = ('RemoteProgress', 'PushInfo', 'FetchInfo', 'Remote') @@ -45,16 +47,16 @@ def digest_process_messages(fh, progress): :param fh: File handle to read from :return: list(line, ...) list of lines without linebreaks that did not contain progress information""" - line_so_far = '' + line_so_far = b'' dropped_lines = list() while True: - char = fh.read(1) + char = fh.read(1) # reads individual single byte strings if not char: break - if char in ('\r', '\n') and line_so_far: - dropped_lines.extend(progress._parse_progress_line(line_so_far)) - line_so_far = '' + if char in (b'\r', b'\n') and line_so_far: + dropped_lines.extend(progress._parse_progress_line(line_so_far.decode(defenc))) + line_so_far = b'' else: line_so_far += char # END process parsed line @@ -136,7 +138,7 @@ class PushInfo(object): @classmethod def _from_line(cls, remote, line): """Create a new PushInfo instance as parsed from line which is expected to be like - refs/heads/master:refs/heads/master 05d2687..1d0568e""" + refs/heads/master:refs/heads/master 05d2687..1d0568e as bytes""" control_character, from_to, summary = line.split('\t', 3) flags = 0 @@ -390,7 +392,7 @@ class Remote(LazyMixin, Iterable): # even though a slot of the same name exists try: return self._config_reader.get(attr) - except NoOptionError: + except cp.NoOptionError: return super(Remote, self).__getattr__(attr) # END handle exception @@ -520,6 +522,7 @@ class Remote(LazyMixin, Iterable): def _get_fetch_info_from_stderr(self, proc, progress): # skip first line as it is some remote info we are not interested in + # TODO: Use poll() to process stdout and stderr at same time output = IterableList('name') # lines which are no progress are fetch info lines @@ -542,8 +545,8 @@ class Remote(LazyMixin, Iterable): # END for each line # read head information - fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'r') - fetch_head_info = fp.readlines() + fp = open(join(self.repo.git_dir, 'FETCH_HEAD'), 'rb') + fetch_head_info = [l.decode(defenc) for l in fp.readlines()] fp.close() # NOTE: We assume to fetch at least enough progress lines to allow matching each fetch head line with it. @@ -560,10 +563,12 @@ class Remote(LazyMixin, Iterable): # we hope stdout can hold all the data, it should ... # read the lines manually as it will use carriage returns between the messages # to override the previous one. This is why we read the bytes manually + # TODO: poll() on file descriptors to know what to read next, process streams concurrently digest_process_messages(proc.stderr, progress) output = IterableList('name') for line in proc.stdout.readlines(): + line = line.decode(defenc) try: output.append(PushInfo._from_line(self, line)) except ValueError: @@ -571,7 +576,6 @@ class Remote(LazyMixin, Iterable): pass # END exception handling # END for each line - finalize_process(proc) return output diff --git a/git/repo/base.py b/git/repo/base.py index dcf98152..2a63492b 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -40,13 +40,17 @@ from gitdb.util import ( hex_to_bin ) -from fun import ( +from .fun import ( rev_parse, is_git_dir, find_git_dir, read_gitfile, touch, ) +from git.compat import ( + text_type, + defenc +) import os import sys @@ -176,11 +180,11 @@ class Repo(object): # Description property def _get_description(self): filename = join(self.git_dir, 'description') - return file(filename).read().rstrip() + return open(filename, 'rb').read().rstrip().decode(defenc) def _set_description(self, descr): filename = join(self.git_dir, 'description') - file(filename, 'w').write(descr + '\n') + open(filename, 'wb').write((descr + '\n').encode(defenc)) description = property(_get_description, _set_description, doc="the project's description") @@ -389,7 +393,7 @@ class Repo(object): if rev is None: return self.head.commit else: - return self.rev_parse(unicode(rev) + "^0") + return self.rev_parse(text_type(rev) + "^0") def iter_trees(self, *args, **kwargs): """:return: Iterator yielding Tree objects @@ -412,7 +416,7 @@ class Repo(object): if rev is None: return self.head.commit.tree else: - return self.rev_parse(unicode(rev) + "^{tree}") + return self.rev_parse(text_type(rev) + "^{tree}") def iter_commits(self, rev=None, paths='', **kwargs): """A list of Commit objects representing the history of a given ref/commit @@ -463,8 +467,8 @@ class Repo(object): if os.path.exists(alternates_path): try: - f = open(alternates_path) - alts = f.read() + f = open(alternates_path, 'rb') + alts = f.read().decode(defenc) finally: f.close() return alts.strip().splitlines() @@ -488,8 +492,8 @@ class Repo(object): os.remove(alternates_path) else: try: - f = open(alternates_path, 'w') - f.write("\n".join(alts)) + f = open(alternates_path, 'wb') + f.write("\n".join(alts).encode(defenc)) finally: f.close() # END file handling @@ -547,6 +551,7 @@ class Repo(object): prefix = "?? " untracked_files = list() for line in proc.stdout: + line = line.decode(defenc) if not line.startswith(prefix): continue filename = line[len(prefix):].rstrip('\n') @@ -728,7 +733,10 @@ class Repo(object): # sure repo = cls(os.path.abspath(path), odbt=odbt) if repo.remotes: - repo.remotes[0].config_writer.set_value('url', repo.remotes[0].url.replace("\\\\", "\\").replace("\\", "/")) + writer = repo.remotes[0].config_writer + writer.set_value('url', repo.remotes[0].url.replace("\\\\", "\\").replace("\\", "/")) + # PY3: be sure cleanup is performed and lock is released + writer.release() # END handle remote repo return repo @@ -760,7 +768,7 @@ class Repo(object): def archive(self, ostream, treeish=None, prefix=None, **kwargs): """Archive the tree at the given revision. - :parm ostream: file compatible stream object to which the archive will be written + :parm ostream: file compatible stream object to which the archive will be written as bytes :parm treeish: is the treeish name/id, defaults to active branch :parm prefix: is the optional prefix to prepend to each filename in the archive :parm kwargs: diff --git a/git/repo/fun.py b/git/repo/fun.py index b8905517..233666c9 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -1,5 +1,7 @@ """Package with general repository related functions""" import os +from string import digits + from gitdb.exc import BadObject from git.refs import SymbolicReference from git.objects import Object @@ -11,14 +13,15 @@ from gitdb.util import ( hex_to_bin, bin_to_hex ) -from string import digits +from git.compat import xrange + __all__ = ('rev_parse', 'is_git_dir', 'touch', 'read_gitfile', 'find_git_dir', 'name_to_object', 'short_to_long', 'deref_tag', 'to_commit') def touch(filename): - fp = open(filename, "a") + fp = open(filename, "ab") fp.close() @@ -147,7 +150,7 @@ def to_commit(obj): def rev_parse(repo, rev): """ :return: Object at the given revision, either Commit, Tag, Tree or Blob - :param rev: git-rev-parse compatible revision specification, please see + :param rev: git-rev-parse compatible revision specification as string, please see http://www.kernel.org/pub/software/scm/git/docs/git-rev-parse.html for details :note: Currently there is no access to the rev-log, rev-specs may only contain diff --git a/git/test/fixtures/git_config_global b/git/test/fixtures/git_config_global index 1a55397f..56fbd3b3 100644 --- a/git/test/fixtures/git_config_global +++ b/git/test/fixtures/git_config_global @@ -1,3 +1,4 @@ +# just a comment [alias] st = status ci = commit diff --git a/git/test/lib/asserts.py b/git/test/lib/asserts.py index 0f2fd99a..60a888b3 100644 --- a/git/test/lib/asserts.py +++ b/git/test/lib/asserts.py @@ -7,13 +7,6 @@ import re import stat -__all__ = ['assert_instance_of', 'assert_not_instance_of', - 'assert_none', 'assert_not_none', - 'assert_match', 'assert_not_match', 'assert_mode_644', - 'assert_mode_755', - 'assert_equal', 'assert_not_equal', 'assert_raises', 'patch', 'raises', - 'assert_true', 'assert_false'] - from nose.tools import ( assert_equal, assert_not_equal, @@ -23,9 +16,14 @@ from nose.tools import ( assert_false ) -from mock import ( - patch -) +from mock import patch + +__all__ = ['assert_instance_of', 'assert_not_instance_of', + 'assert_none', 'assert_not_none', + 'assert_match', 'assert_not_match', 'assert_mode_644', + 'assert_mode_755', + 'assert_equal', 'assert_not_equal', 'assert_raises', 'patch', 'raises', + 'assert_true', 'assert_false'] def assert_instance_of(expected, actual, msg=None): diff --git a/git/test/lib/helper.py b/git/test/lib/helper.py index 9c935ce0..bd679512 100644 --- a/git/test/lib/helper.py +++ b/git/test/lib/helper.py @@ -6,12 +6,14 @@ from __future__ import print_function import os import sys -from git import Repo, Remote, GitCommandError, Git from unittest import TestCase import time import tempfile import shutil -import cStringIO +import io + +from git import Repo, Remote, GitCommandError, Git +from git.compat import string_types GIT_REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) @@ -46,8 +48,8 @@ class StringProcessAdapter(object): Its tailored to work with the test system only""" def __init__(self, input_string): - self.stdout = cStringIO.StringIO(input_string) - self.stderr = cStringIO.StringIO() + self.stdout = io.BytesIO(input_string) + self.stderr = io.BytesIO() def wait(self): return 0 @@ -89,7 +91,7 @@ def with_rw_repo(working_tree_ref, bare=False): To make working with relative paths easier, the cwd will be set to the working dir of the repository. """ - assert isinstance(working_tree_ref, basestring), "Decorator requires ref name for working tree checkout" + assert isinstance(working_tree_ref, string_types), "Decorator requires ref name for working tree checkout" def argument_passer(func): def repo_creator(self): @@ -152,7 +154,7 @@ def with_rw_and_rw_remote_repo(working_tree_ref): See working dir info in with_rw_repo :note: We attempt to launch our own invocation of git-daemon, which will be shutdown at the end of the test. """ - assert isinstance(working_tree_ref, basestring), "Decorator requires ref name for working tree checkout" + assert isinstance(working_tree_ref, string_types), "Decorator requires ref name for working tree checkout" def argument_passer(func): def remote_repo_creator(self): @@ -177,6 +179,7 @@ def with_rw_and_rw_remote_repo(working_tree_ref): pass crw.set(section, "receivepack", True) # release lock + crw.release() del(crw) # initialize the remote - first do it as local remote and pull, then @@ -191,7 +194,7 @@ def with_rw_and_rw_remote_repo(working_tree_ref): temp_dir = os.path.dirname(_mktemp()) # On windows, this will fail ... we deal with failures anyway and default to telling the user to do it try: - gd = Git().daemon(temp_dir, as_process=True) + gd = Git().daemon(temp_dir, enable='receive-pack', as_process=True) # yes, I know ... fortunately, this is always going to work if sleep time is just large enough time.sleep(0.5) except Exception: @@ -213,7 +216,8 @@ def with_rw_and_rw_remote_repo(working_tree_ref): msg += 'Otherwise, run: git-daemon "%s"' % temp_dir raise AssertionError(msg) else: - msg = 'Please start a git-daemon to run this test, execute: git-daemon "%s"' % temp_dir + msg = 'Please start a git-daemon to run this test, execute: git daemon --enable=receive-pack "%s"' + msg %= temp_dir raise AssertionError(msg) # END make assertion # END catch ls remote error @@ -225,7 +229,8 @@ def with_rw_and_rw_remote_repo(working_tree_ref): return func(self, rw_repo, rw_remote_repo) finally: # gd.proc.kill() ... no idea why that doesn't work - os.kill(gd.proc.pid, 15) + if gd is not None: + os.kill(gd.proc.pid, 15) os.chdir(prev_cwd) rw_repo.git.clear_cache() @@ -233,7 +238,8 @@ def with_rw_and_rw_remote_repo(working_tree_ref): shutil.rmtree(repo_dir, onerror=_rmtree_onerror) shutil.rmtree(remote_repo_dir, onerror=_rmtree_onerror) - gd.proc.wait() + if gd is not None: + gd.proc.wait() # END cleanup # END bare repo creator remote_repo_creator.__name__ = func.__name__ diff --git a/git/test/performance/test_commit.py b/git/test/performance/test_commit.py index a890c833..7d3e87c4 100644 --- a/git/test/performance/test_commit.py +++ b/git/test/performance/test_commit.py @@ -4,13 +4,15 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php from __future__ import print_function +from io import BytesIO +from time import time +import sys + from .lib import TestBigRepoRW from git import Commit from gitdb import IStream +from git.compat import xrange from git.test.test_commit import assert_commit_serialization -from cStringIO import StringIO -from time import time -import sys class TestPerformance(TestBigRepoRW): @@ -90,7 +92,7 @@ class TestPerformance(TestBigRepoRW): hc.committer, hc.committed_date, hc.committer_tz_offset, str(i), parents=hc.parents, encoding=hc.encoding) - stream = StringIO() + stream = BytesIO() cm._serialize(stream) slen = stream.tell() stream.seek(0) diff --git a/git/test/performance/test_streams.py b/git/test/performance/test_streams.py index ff664c10..aecb7728 100644 --- a/git/test/performance/test_streams.py +++ b/git/test/performance/test_streams.py @@ -80,7 +80,7 @@ class TestObjDBPerformance(TestBigRepoR): elapsed_readchunks = time() - st stream.seek(0) - assert ''.join(chunks) == stream.getvalue() + assert b''.join(chunks) == stream.getvalue() cs_kib = cs / 1000 print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" diff --git a/git/test/test_base.py b/git/test/test_base.py index a14d4680..301384ef 100644 --- a/git/test/test_base.py +++ b/git/test/test_base.py @@ -1,12 +1,13 @@ +#-*-coding:utf-8-*- # test_base.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php - -import git.objects.base as base import os +import tempfile +import git.objects.base as base from git.test.lib import ( TestBase, assert_raises, @@ -68,10 +69,13 @@ class TestBase(TestBase): data = data_stream.read() assert data - tmpfile = os.tmpfile() + tmpfilename = tempfile.mktemp(suffix='test-stream') + tmpfile = open(tmpfilename, 'wb+') assert item == item.stream_data(tmpfile) tmpfile.seek(0) assert tmpfile.read() == data + tmpfile.close() + os.remove(tmpfilename) # END stream to file directly # END for each object type to create @@ -85,7 +89,7 @@ class TestBase(TestBase): assert base.Object in get_object_type_by_name(tname).mro() # END for each known type - assert_raises(ValueError, get_object_type_by_name, "doesntexist") + assert_raises(ValueError, get_object_type_by_name, b"doesntexist") def test_object_resolution(self): # objects must be resolved to shas so they compare equal @@ -106,3 +110,13 @@ class TestBase(TestBase): assert not rw_repo.config_reader("repository").getboolean("core", "bare") assert rw_remote_repo.config_reader("repository").getboolean("core", "bare") assert os.path.isdir(os.path.join(rw_repo.working_tree_dir, 'lib')) + + @with_rw_repo('0.1.6') + def test_add_unicode(self, rw_repo): + filename = u"שלום.txt" + + file_path = os.path.join(rw_repo.working_dir, filename) + open(file_path, "wb").write(b'something') + + rw_repo.git.add(rw_repo.working_dir) + rw_repo.index.commit('message') diff --git a/git/test/test_commit.py b/git/test/test_commit.py index bfad6fd6..1f0f8c56 100644 --- a/git/test/test_commit.py +++ b/git/test/test_commit.py @@ -19,9 +19,12 @@ from git import ( Actor, ) from gitdb import IStream -from gitdb.util import hex_to_bin +from git.compat import ( + string_types, + text_type +) -from cStringIO import StringIO +from io import BytesIO import time import sys import re @@ -40,14 +43,14 @@ def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False) # assert that we deserialize commits correctly, hence we get the same # sha on serialization - stream = StringIO() + stream = BytesIO() cm._serialize(stream) ns += 1 streamlen = stream.tell() stream.seek(0) istream = rwrepo.odb.store(IStream(Commit.type, streamlen, stream)) - assert istream.hexsha == cm.hexsha + assert istream.hexsha == cm.hexsha.encode('ascii') nc = Commit(rwrepo, Commit.NULL_BIN_SHA, cm.tree, cm.author, cm.authored_date, cm.author_tz_offset, @@ -55,7 +58,7 @@ def assert_commit_serialization(rwrepo, commit_id, print_performance_info=False) cm.message, cm.parents, cm.encoding) assert nc.parents == cm.parents - stream = StringIO() + stream = BytesIO() nc._serialize(stream) ns += 1 streamlen = stream.tell() @@ -125,11 +128,11 @@ class TestCommit(TestBase): def test_unicode_actor(self): # assure we can parse unicode actors correctly - name = "Üäöß ÄußÉ".decode("utf-8") + name = u"Üäöß ÄußÉ" assert len(name) == 9 special = Actor._from_string(u"%s <something@this.com>" % name) assert special.name == name - assert isinstance(special.name, unicode) + assert isinstance(special.name, text_type) def test_traversal(self): start = self.rorepo.commit("a4d06724202afccd2b5c54f81bcf2bf26dea7fff") @@ -142,13 +145,13 @@ class TestCommit(TestBase): # basic branch first, depth first dfirst = start.traverse(branch_first=False) bfirst = start.traverse(branch_first=True) - assert dfirst.next() == p0 - assert dfirst.next() == p00 + assert next(dfirst) == p0 + assert next(dfirst) == p00 - assert bfirst.next() == p0 - assert bfirst.next() == p1 - assert bfirst.next() == p00 - assert bfirst.next() == p10 + assert next(bfirst) == p0 + assert next(bfirst) == p1 + assert next(bfirst) == p00 + assert next(bfirst) == p10 # at some point, both iterations should stop assert list(bfirst)[-1] == first @@ -157,19 +160,19 @@ class TestCommit(TestBase): assert len(l[0]) == 2 # ignore self - assert start.traverse(ignore_self=False).next() == start + assert next(start.traverse(ignore_self=False)) == start # depth assert len(list(start.traverse(ignore_self=False, depth=0))) == 1 # prune - assert start.traverse(branch_first=1, prune=lambda i, d: i == p0).next() == p1 + assert next(start.traverse(branch_first=1, prune=lambda i, d: i == p0)) == p1 # predicate - assert start.traverse(branch_first=1, predicate=lambda i, d: i == p1).next() == p1 + assert next(start.traverse(branch_first=1, predicate=lambda i, d: i == p1)) == p1 # traversal should stop when the beginning is reached - self.failUnlessRaises(StopIteration, first.traverse().next) + self.failUnlessRaises(StopIteration, next, first.traverse()) # parents of the first commit should be empty ( as the only parent has a null # sha ) @@ -206,7 +209,7 @@ class TestCommit(TestBase): first_parent=True, bisect_all=True) - commits = Commit._iter_from_process_or_stream(self.rorepo, StringProcessAdapter(revs)) + commits = Commit._iter_from_process_or_stream(self.rorepo, StringProcessAdapter(revs.encode('ascii'))) expected_ids = ( '7156cece3c49544abb6bf7a0c218eb36646fad6d', '1f66cfbbce58b4b552b041707a12d437cc5f400a', @@ -220,8 +223,10 @@ class TestCommit(TestBase): assert self.rorepo.tag('refs/tags/0.1.5').commit.count() == 143 def test_list(self): + # This doesn't work anymore, as we will either attempt getattr with bytes, or compare 20 byte string + # with actual 20 byte bytes. This usage makes no sense anyway assert isinstance(Commit.list_items(self.rorepo, '0.1.5', max_count=5)[ - hex_to_bin('5117c9c8a4d3af19a9958677e45cda9269de1541')], Commit) + '5117c9c8a4d3af19a9958677e45cda9269de1541'], Commit) def test_str(self): commit = Commit(self.rorepo, Commit.NULL_BIN_SHA) @@ -243,14 +248,14 @@ class TestCommit(TestBase): c = self.rorepo.commit('0.1.5') for skip in (0, 1): piter = c.iter_parents(skip=skip) - first_parent = piter.next() + first_parent = next(piter) assert first_parent != c assert first_parent == c.parents[0] # END for each - def test_base(self): + def test_name_rev(self): name_rev = self.rorepo.head.commit.name_rev - assert isinstance(name_rev, basestring) + assert isinstance(name_rev, string_types) @with_rw_repo('HEAD', bare=True) def test_serialization(self, rwrepo): @@ -263,16 +268,16 @@ class TestCommit(TestBase): # create a commit with unicode in the message, and the author's name # Verify its serialization and deserialization cmt = self.rorepo.commit('0.1.6') - assert isinstance(cmt.message, unicode) # it automatically decodes it as such - assert isinstance(cmt.author.name, unicode) # same here + assert isinstance(cmt.message, text_type) # it automatically decodes it as such + assert isinstance(cmt.author.name, text_type) # same here - cmt.message = "üäêèß".decode("utf-8") + cmt.message = u"üäêèß" assert len(cmt.message) == 5 - cmt.author.name = "äüß".decode("utf-8") + cmt.author.name = u"äüß" assert len(cmt.author.name) == 3 - cstream = StringIO() + cstream = BytesIO() cmt._serialize(cstream) cstream.seek(0) assert len(cstream.getvalue()) @@ -288,7 +293,7 @@ class TestCommit(TestBase): def test_gpgsig(self): cmt = self.rorepo.commit() - cmt._deserialize(open(fixture_path('commit_with_gpgsig'))) + cmt._deserialize(open(fixture_path('commit_with_gpgsig'), 'rb')) fixture_sig = """-----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.11 (GNU/Linux) @@ -312,9 +317,9 @@ JzJMZDRLQLFvnzqZuCjE cmt.gpgsig = "<test\ndummy\nsig>" assert cmt.gpgsig != fixture_sig - cstream = StringIO() + cstream = BytesIO() cmt._serialize(cstream) - assert re.search(r"^gpgsig <test\n dummy\n sig>$", cstream.getvalue(), re.MULTILINE) + assert re.search(r"^gpgsig <test\n dummy\n sig>$", cstream.getvalue().decode('ascii'), re.MULTILINE) cstream.seek(0) cmt.gpgsig = None @@ -322,6 +327,6 @@ JzJMZDRLQLFvnzqZuCjE assert cmt.gpgsig == "<test\ndummy\nsig>" cmt.gpgsig = None - cstream = StringIO() + cstream = BytesIO() cmt._serialize(cstream) - assert not re.search(r"^gpgsig ", cstream.getvalue(), re.MULTILINE) + assert not re.search(r"^gpgsig ", cstream.getvalue().decode('ascii'), re.MULTILINE) diff --git a/git/test/test_config.py b/git/test/test_config.py index d1c8e72f..546a2fe1 100644 --- a/git/test/test_config.py +++ b/git/test/test_config.py @@ -11,16 +11,19 @@ from git.test.lib import ( from git import ( GitConfigParser ) -import StringIO +from git.compat import ( + string_types, +) +import io from copy import copy -from ConfigParser import NoSectionError +from git.config import cp class TestBase(TestCase): def _to_memcache(self, file_path): - fp = open(file_path, "r") - sio = StringIO.StringIO(fp.read()) + fp = open(file_path, "rb") + sio = io.BytesIO(fp.read()) sio.name = file_path return sio @@ -38,7 +41,7 @@ class TestBase(TestCase): w_config.write() # enforce writing # we stripped lines when reading, so the results differ - assert file_obj.getvalue() != file_obj_orig.getvalue() + assert file_obj.getvalue() and file_obj.getvalue() != file_obj_orig.getvalue() # creating an additional config writer must fail due to exclusive access self.failUnlessRaises(IOError, GitConfigParser, file_obj, read_only=False) @@ -85,7 +88,7 @@ class TestBase(TestCase): num_options += 1 val = r_config.get(section, option) val_typed = r_config.get_value(section, option) - assert isinstance(val_typed, (bool, long, float, basestring)) + assert isinstance(val_typed, (bool, int, float, ) + string_types) assert val assert "\n" not in option assert "\n" not in val @@ -104,4 +107,4 @@ class TestBase(TestCase): assert r_config.get_value("doesnt", "exist", default) == default # it raises if there is no default though - self.failUnlessRaises(NoSectionError, r_config.get_value, "doesnt", "exist") + self.failUnlessRaises(cp.NoSectionError, r_config.get_value, "doesnt", "exist") diff --git a/git/test/test_fun.py b/git/test/test_fun.py index bf178aaa..40d040b9 100644 --- a/git/test/test_fun.py +++ b/git/test/test_fun.py @@ -24,13 +24,13 @@ from stat import ( ) from git.index import IndexFile -from cStringIO import StringIO +from io import BytesIO class TestFun(TestBase): def _assert_index_entries(self, entries, trees): - index = IndexFile.from_tree(self.rorepo, *[self.rorepo.tree(bin_to_hex(t)) for t in trees]) + index = IndexFile.from_tree(self.rorepo, *[self.rorepo.tree(bin_to_hex(t).decode('ascii')) for t in trees]) assert entries assert len(index.entries) == len(entries) for entry in entries: @@ -72,7 +72,7 @@ class TestFun(TestBase): def mktree(self, odb, entries): """create a tree from the given tree entries and safe it to the database""" - sio = StringIO() + sio = BytesIO() tree_to_stream(entries, sio.write) sio.seek(0) istream = odb.store(IStream(str_tree_type, len(sio.getvalue()), sio)) @@ -91,9 +91,9 @@ class TestFun(TestBase): assert has_conflict == (len([e for e in entries if e.stage != 0]) > 0) mktree = self.mktree - shaa = "\1" * 20 - shab = "\2" * 20 - shac = "\3" * 20 + shaa = b"\1" * 20 + shab = b"\2" * 20 + shac = b"\3" * 20 odb = rwrepo.odb @@ -256,6 +256,6 @@ class TestFun(TestBase): assert entries # END for each commit - def test_tree_entries_from_data(self): + def test_tree_entries_from_data_with_failing_name_decode(self): r = tree_entries_from_data(b'100644 \x9f\0aaa') - assert r == [('aaa', 33188, '\x9f')], r + assert r == [(b'aaa', 33188, b'\x9f')], r diff --git a/git/test/test_git.py b/git/test/test_git.py index 553f8d1b..502e6091 100644 --- a/git/test/test_git.py +++ b/git/test/test_git.py @@ -1,3 +1,4 @@ +#-*-coding:utf-8-*- # test_git.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # @@ -16,6 +17,8 @@ from git.test.lib import (TestBase, from git import (Git, GitCommandError) +from git.compat import PY3 + class TestGit(TestBase): @@ -32,12 +35,20 @@ class TestGit(TestBase): assert_equal(git.call_args, ((['git', 'version'],), {})) def test_call_unpack_args_unicode(self): - args = Git._Git__unpack_args(u'Unicode' + unichr(40960)) - assert_equal(args, ['Unicode\xea\x80\x80']) + args = Git._Git__unpack_args(u'Unicode€™') + if PY3: + mangled_value = 'Unicode\u20ac\u2122' + else: + mangled_value = 'Unicode\xe2\x82\xac\xe2\x84\xa2' + assert_equal(args, [mangled_value]) def test_call_unpack_args(self): - args = Git._Git__unpack_args(['git', 'log', '--', u'Unicode' + unichr(40960)]) - assert_equal(args, ['git', 'log', '--', 'Unicode\xea\x80\x80']) + args = Git._Git__unpack_args(['git', 'log', '--', u'Unicode€™']) + if PY3: + mangled_value = 'Unicode\u20ac\u2122' + else: + mangled_value = 'Unicode\xe2\x82\xac\xe2\x84\xa2' + assert_equal(args, ['git', 'log', '--', mangled_value]) @raises(GitCommandError) def test_it_raises_errors(self): @@ -75,13 +86,13 @@ class TestGit(TestBase): import subprocess as sp hexsha = "b2339455342180c7cc1e9bba3e9f181f7baa5167" g = self.git.cat_file(batch_check=True, istream=sp.PIPE, as_process=True) - g.stdin.write("b2339455342180c7cc1e9bba3e9f181f7baa5167\n") + g.stdin.write(b"b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() obj_info = g.stdout.readline() # read header + data g = self.git.cat_file(batch=True, istream=sp.PIPE, as_process=True) - g.stdin.write("b2339455342180c7cc1e9bba3e9f181f7baa5167\n") + g.stdin.write(b"b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() obj_info_two = g.stdout.readline() assert obj_info == obj_info_two @@ -92,7 +103,7 @@ class TestGit(TestBase): g.stdout.read(1) # now we should be able to read a new object - g.stdin.write("b2339455342180c7cc1e9bba3e9f181f7baa5167\n") + g.stdin.write(b"b2339455342180c7cc1e9bba3e9f181f7baa5167\n") g.stdin.flush() assert g.stdout.readline() == obj_info diff --git a/git/test/test_index.py b/git/test/test_index.py index 15fff8d4..f7504b32 100644 --- a/git/test/test_index.py +++ b/git/test/test_index.py @@ -20,6 +20,7 @@ from git import ( GitCommandError, CheckoutError, ) +from git.compat import string_types from gitdb.util import hex_to_bin import os import sys @@ -30,7 +31,7 @@ from stat import ( ST_MODE ) -from StringIO import StringIO +from io import BytesIO from gitdb.base import IStream from git.objects import Blob from git.index.typ import ( @@ -47,7 +48,7 @@ class TestIndex(TestBase): def _assert_fprogress(self, entries): assert len(entries) == len(self._fprogress_map) - for path, call_count in self._fprogress_map.iteritems(): + for path, call_count in self._fprogress_map.items(): assert call_count == 2 # END for each item in progress map self._reset_progress() @@ -85,7 +86,7 @@ class TestIndex(TestBase): assert index.version > 0 # test entry - entry = index.entries.itervalues().next() + entry = next(iter(index.entries.values())) for attr in ("path", "ctime", "mtime", "dev", "inode", "mode", "uid", "gid", "size", "binsha", "hexsha", "stage"): getattr(entry, attr) @@ -99,7 +100,7 @@ class TestIndex(TestBase): # test stage index_merge = IndexFile(self.rorepo, fixture_path("index_merge")) assert len(index_merge.entries) == 106 - assert len(list(e for e in index_merge.entries.itervalues() if e.stage != 0)) + assert len(list(e for e in index_merge.entries.values() if e.stage != 0)) # write the data - it must match the original tmpfile = tempfile.mktemp() @@ -166,7 +167,7 @@ class TestIndex(TestBase): assert unmerged_blob_map # pick the first blob at the first stage we find and use it as resolved version - three_way_index.resolve_blobs(l[0][1] for l in unmerged_blob_map.itervalues()) + three_way_index.resolve_blobs(l[0][1] for l in unmerged_blob_map.values()) tree = three_way_index.write_tree() assert isinstance(tree, Tree) num_blobs = 0 @@ -200,7 +201,7 @@ class TestIndex(TestBase): # Add a change with a NULL sha that should conflict with next_commit. We # pretend there was a change, but we do not even bother adding a proper # sha for it ( which makes things faster of course ) - manifest_fake_entry = BaseIndexEntry((manifest_entry[0], "\0" * 20, 0, manifest_entry[3])) + manifest_fake_entry = BaseIndexEntry((manifest_entry[0], b"\0" * 20, 0, manifest_entry[3])) # try write flag self._assert_entries(rw_repo.index.add([manifest_fake_entry], write=False)) # add actually resolves the null-hex-sha for us as a feature, but we can @@ -235,7 +236,7 @@ class TestIndex(TestBase): # now make a proper three way merge with unmerged entries unmerged_tree = IndexFile.from_tree(rw_repo, parent_commit, tree, next_commit) unmerged_blobs = unmerged_tree.unmerged_blobs() - assert len(unmerged_blobs) == 1 and unmerged_blobs.keys()[0] == manifest_key[0] + assert len(unmerged_blobs) == 1 and list(unmerged_blobs.keys())[0] == manifest_key[0] @with_rw_repo('0.1.6') def test_index_file_diffing(self, rw_repo): @@ -294,7 +295,7 @@ class TestIndex(TestBase): assert index.diff(None) # reset the working copy as well to current head,to pull 'back' as well - new_data = "will be reverted" + new_data = b"will be reverted" file_path = os.path.join(rw_repo.working_tree_dir, "CHANGES") fp = open(file_path, "wb") fp.write(new_data) @@ -311,7 +312,7 @@ class TestIndex(TestBase): # test full checkout test_file = os.path.join(rw_repo.working_tree_dir, "CHANGES") - open(test_file, 'ab').write("some data") + open(test_file, 'ab').write(b"some data") rval = index.checkout(None, force=True, fprogress=self._fprogress) assert 'CHANGES' in list(rval) self._assert_fprogress([None]) @@ -335,7 +336,7 @@ class TestIndex(TestBase): self.failUnlessRaises(CheckoutError, index.checkout, paths=["doesnt/exist"]) # checkout file with modifications - append_data = "hello" + append_data = b"hello" fp = open(test_file, "ab") fp.write(append_data) fp.close() @@ -343,15 +344,15 @@ class TestIndex(TestBase): index.checkout(test_file) except CheckoutError as e: assert len(e.failed_files) == 1 and e.failed_files[0] == os.path.basename(test_file) - assert (len(e.failed_files) == len(e.failed_reasons)) and isinstance(e.failed_reasons[0], basestring) + assert (len(e.failed_files) == len(e.failed_reasons)) and isinstance(e.failed_reasons[0], string_types) assert len(e.valid_files) == 0 - assert open(test_file).read().endswith(append_data) + assert open(test_file, 'rb').read().endswith(append_data) else: raise AssertionError("Exception CheckoutError not thrown") # if we force it it should work index.checkout(test_file, force=True) - assert not open(test_file).read().endswith(append_data) + assert not open(test_file, 'rb').read().endswith(append_data) # checkout directory shutil.rmtree(os.path.join(rw_repo.working_tree_dir, "lib")) @@ -378,14 +379,16 @@ class TestIndex(TestBase): uname = "Some Developer" umail = "sd@company.com" - rw_repo.config_writer().set_value("user", "name", uname) - rw_repo.config_writer().set_value("user", "email", umail) + writer = rw_repo.config_writer() + writer.set_value("user", "name", uname) + writer.set_value("user", "email", umail) + writer.release() # remove all of the files, provide a wild mix of paths, BaseIndexEntries, # IndexEntries def mixed_iterator(): count = 0 - for entry in index.entries.itervalues(): + for entry in index.entries.values(): type_id = count % 4 if type_id == 0: # path yield entry.path @@ -499,7 +502,7 @@ class TestIndex(TestBase): # mode 0 not allowed null_hex_sha = Diff.NULL_HEX_SHA - null_bin_sha = "\0" * 20 + null_bin_sha = b"\0" * 20 self.failUnlessRaises(ValueError, index.reset( new_commit).add, [BaseIndexEntry((0, null_bin_sha, 0, "doesntmatter"))]) @@ -525,7 +528,7 @@ class TestIndex(TestBase): assert S_ISLNK(index.entries[index.entry_key("my_real_symlink", 0)].mode) # we expect only the target to be written - assert index.repo.odb.stream(entries[0].binsha).read() == target + assert index.repo.odb.stream(entries[0].binsha).read().decode('ascii') == target # END real symlink test # add fake symlink and assure it checks-our as symlink @@ -617,7 +620,7 @@ class TestIndex(TestBase): for fid in range(3): fname = 'newfile%i' % fid - open(fname, 'wb').write("abcd") + open(fname, 'wb').write(b"abcd") yield Blob(rw_repo, Blob.NULL_BIN_SHA, 0o100644, fname) # END for each new file # END path producer @@ -697,9 +700,9 @@ class TestIndex(TestBase): # instead of throwing the Exception we are expecting. This is # a quick hack to make this test fail when expected. rw_bare_repo._working_tree_dir = None - contents = 'This is a StringIO file' + contents = b'This is a BytesIO file' filesize = len(contents) - fileobj = StringIO(contents) + fileobj = BytesIO(contents) filename = 'my-imaginary-file' istream = rw_bare_repo.odb.store( IStream(Blob.type, filesize, fileobj)) @@ -715,5 +718,5 @@ class TestIndex(TestBase): try: rw_bare_repo.index.add([path]) except Exception as e: - asserted = "does not have a working tree" in e.message + asserted = "does not have a working tree" in str(e) assert asserted, "Adding using a filename is not correctly asserted." diff --git a/git/test/test_reflog.py b/git/test/test_reflog.py index 4efb8025..3571e083 100644 --- a/git/test/test_reflog.py +++ b/git/test/test_reflog.py @@ -8,6 +8,7 @@ from git.refs import ( RefLog ) from git.util import Actor +from gitdb.util import hex_to_bin import tempfile import shutil @@ -51,7 +52,7 @@ class TestRefLog(TestBase): assert len(reflog) # iter_entries works with path and with stream - assert len(list(RefLog.iter_entries(open(rlp_master)))) + assert len(list(RefLog.iter_entries(open(rlp_master, 'rb')))) assert len(list(RefLog.iter_entries(rlp_master))) # raise on invalid revlog @@ -65,7 +66,7 @@ class TestRefLog(TestBase): self.failUnlessRaises(ValueError, RefLog().write) # test serialize and deserialize - results must match exactly - binsha = chr(255) * 20 + binsha = hex_to_bin(('f' * 40).encode('ascii')) msg = "my reflog message" cr = self.rorepo.config_reader() for rlp in (rlp_head, rlp_master): diff --git a/git/test/test_refs.py b/git/test/test_refs.py index af33765a..14b91cfe 100644 --- a/git/test/test_refs.py +++ b/git/test/test_refs.py @@ -105,9 +105,11 @@ class TestRefs(TestBase): tv = "testopt" writer.set_value(tv, 1) assert writer.get_value(tv) == 1 - del(writer) + writer.release() assert head.config_reader().get_value(tv) == 1 - head.config_writer().remove_option(tv) + writer = head.config_writer() + writer.remove_option(tv) + writer.release() # after the clone, we might still have a tracking branch setup head.set_tracking_branch(None) diff --git a/git/test/test_remote.py b/git/test/test_remote.py index a8d5179a..75dc19c5 100644 --- a/git/test/test_remote.py +++ b/git/test/test_remote.py @@ -23,6 +23,7 @@ from git import ( GitCommandError ) from git.util import IterableList +from git.compat import string_types import tempfile import shutil import os @@ -97,7 +98,7 @@ class TestRemote(TestBase): # self._print_fetchhead(remote.repo) assert len(results) > 0 and isinstance(results[0], FetchInfo) for info in results: - assert isinstance(info.note, basestring) + assert isinstance(info.note, string_types) if isinstance(info.ref, Reference): assert info.flags != 0 # END reference type flags handling @@ -113,7 +114,7 @@ class TestRemote(TestBase): assert len(results) > 0 and isinstance(results[0], PushInfo) for info in results: assert info.flags - assert isinstance(info.summary, basestring) + assert isinstance(info.summary, string_types) if info.old_commit is not None: assert isinstance(info.old_commit, Commit) if info.flags & info.ERROR: diff --git a/git/test/test_repo.py b/git/test/test_repo.py index f6b46a6e..f216039e 100644 --- a/git/test/test_repo.py +++ b/git/test/test_repo.py @@ -30,12 +30,16 @@ from git import ( from git.util import join_path_native from git.exc import BadObject from gitdb.util import bin_to_hex +from git.compat import ( + string_types, + defenc +) import os import sys import tempfile import shutil -from cStringIO import StringIO +from io import BytesIO class TestRepo(TestBase): @@ -258,13 +262,16 @@ class TestRepo(TestBase): assert self.rorepo.tag('refs/tags/0.1.5').commit def test_archive(self): - tmpfile = os.tmpfile() - self.rorepo.archive(tmpfile, '0.1.5') - assert tmpfile.tell() + tmpfile = tempfile.mktemp(suffix='archive-test') + stream = open(tmpfile, 'wb') + self.rorepo.archive(stream, '0.1.5') + assert stream.tell() + stream.close() + os.remove(tmpfile) @patch.object(Git, '_call_process') def test_should_display_blame_information(self, git): - git.return_value = fixture('blame') + git.return_value = fixture('blame').decode(defenc) b = self.rorepo.blame('master', 'lib/git.py') assert_equal(13, len(b)) assert_equal(2, len(b[0])) @@ -286,7 +293,7 @@ class TestRepo(TestBase): # test the 'lines per commit' entries tlist = b[0][1] assert_true(tlist) - assert_true(isinstance(tlist[0], basestring)) + assert_true(isinstance(tlist[0], string_types)) assert_true(len(tlist) < sum(len(t) for t in tlist)) # test for single-char bug def test_blame_real(self): @@ -335,6 +342,7 @@ class TestRepo(TestBase): try: writer = self.rorepo.config_writer(config_level) assert not writer.read_only + writer.release() except IOError: # its okay not to get a writer for some configuration files if we # have no permissions @@ -349,7 +357,8 @@ class TestRepo(TestBase): tag = self.rorepo.create_tag("new_tag", "HEAD~2") self.rorepo.delete_tag(tag) - self.rorepo.config_writer() + writer = self.rorepo.config_writer() + writer.release() remote = self.rorepo.create_remote("new_remote", "git@server:repo.git") self.rorepo.delete_remote(remote) @@ -361,27 +370,27 @@ class TestRepo(TestBase): def test_git_cmd(self): # test CatFileContentStream, just to be very sure we have no fencepost errors # last \n is the terminating newline that it expects - l1 = "0123456789\n" - l2 = "abcdefghijklmnopqrstxy\n" - l3 = "z\n" - d = "%s%s%s\n" % (l1, l2, l3) + l1 = b"0123456789\n" + l2 = b"abcdefghijklmnopqrstxy\n" + l3 = b"z\n" + d = l1 + l2 + l3 + b"\n" l1p = l1[:5] # full size # size is without terminating newline def mkfull(): - return Git.CatFileContentStream(len(d) - 1, StringIO(d)) + return Git.CatFileContentStream(len(d) - 1, BytesIO(d)) ts = 5 def mktiny(): - return Git.CatFileContentStream(ts, StringIO(d)) + return Git.CatFileContentStream(ts, BytesIO(d)) # readlines no limit s = mkfull() lines = s.readlines() - assert len(lines) == 3 and lines[-1].endswith('\n') + assert len(lines) == 3 and lines[-1].endswith(b'\n') assert s._stream.tell() == len(d) # must have scrubbed to the end # realines line limit @@ -565,7 +574,7 @@ class TestRepo(TestBase): # try partial parsing max_items = 40 for i, binsha in enumerate(self.rorepo.odb.sha_iter()): - assert rev_parse(bin_to_hex(binsha)[:8 - (i % 2)]).binsha == binsha + assert rev_parse(bin_to_hex(binsha)[:8 - (i % 2)].decode('ascii')).binsha == binsha if i > max_items: # this is rather slow currently, as rev_parse returns an object # which requires accessing packs, it has some additional overhead @@ -644,6 +653,6 @@ class TestRepo(TestBase): assert os.path.abspath(git_file_repo.git_dir) == real_path_abs # Test using an absolute gitdir path in the .git file. - open(git_file_path, 'wb').write('gitdir: %s\n' % real_path_abs) + open(git_file_path, 'wb').write(('gitdir: %s\n' % real_path_abs).encode('ascii')) git_file_repo = Repo(rwrepo.working_tree_dir) assert os.path.abspath(git_file_repo.git_dir) == real_path_abs diff --git a/git/test/test_stats.py b/git/test/test_stats.py index c4535b75..884ab1ab 100644 --- a/git/test/test_stats.py +++ b/git/test/test_stats.py @@ -10,12 +10,13 @@ from git.test.lib import ( assert_equal ) from git import Stats +from git.compat import defenc class TestStats(TestBase): def test_list_from_string(self): - output = fixture('diff_numstat') + output = fixture('diff_numstat').decode(defenc) stats = Stats._list_from_string(self.rorepo, output) assert_equal(2, stats.total['files']) diff --git a/git/test/test_submodule.py b/git/test/test_submodule.py index ec3459e4..99996ce3 100644 --- a/git/test/test_submodule.py +++ b/git/test/test_submodule.py @@ -9,6 +9,7 @@ from git.exc import InvalidGitRepositoryError from git.objects.submodule.base import Submodule from git.objects.submodule.root import RootModule, RootUpdateProgress from git.util import to_native_path_linux, join_path_native +from git.compat import string_types import shutil import git import sys @@ -76,10 +77,10 @@ class TestSubmodule(TestBase): self.failUnlessRaises(InvalidGitRepositoryError, getattr, sm, 'branch') # branch_path works, as its just a string - assert isinstance(sm.branch_path, basestring) + assert isinstance(sm.branch_path, string_types) # some commits earlier we still have a submodule, but its at a different commit - smold = Submodule.iter_items(rwrepo, self.k_subm_changed).next() + smold = next(Submodule.iter_items(rwrepo, self.k_subm_changed)) assert smold.binsha != sm.binsha assert smold != sm # the name changed @@ -98,7 +99,7 @@ class TestSubmodule(TestBase): # for faster checkout, set the url to the local path new_smclone_path = to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, sm.path)) writer.set_value('url', new_smclone_path) - del(writer) + writer.release() assert sm.config_reader().get_value('url') == new_smclone_path assert sm.url == new_smclone_path # END handle bare repo @@ -195,7 +196,9 @@ class TestSubmodule(TestBase): # adjust the path of the submodules module to point to the local destination new_csmclone_path = to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, sm.path, csm.path)) - csm.config_writer().set_value('url', new_csmclone_path) + writer = csm.config_writer() + writer.set_value('url', new_csmclone_path) + writer.release() assert csm.url == new_csmclone_path # dry-run does nothing @@ -256,8 +259,12 @@ class TestSubmodule(TestBase): # NOTE: As we did a few updates in the meanwhile, the indices were reset # Hence we create some changes csm.set_parent_commit(csm.repo.head.commit) - sm.config_writer().set_value("somekey", "somevalue") - csm.config_writer().set_value("okey", "ovalue") + writer = sm.config_writer() + writer.set_value("somekey", "somevalue") + writer.release() + writer = csm.config_writer() + writer.set_value("okey", "ovalue") + writer.release() self.failUnlessRaises(InvalidGitRepositoryError, sm.remove) # if we remove the dirty index, it would work sm.module().index.reset() @@ -405,7 +412,8 @@ class TestSubmodule(TestBase): assert len(rm.list_items(rm.module())) == 1 rm.config_reader() - rm.config_writer() + w = rm.config_writer() + w.release() # deep traversal gitdb / async rsmsp = [sm.path for sm in rm.traverse()] @@ -430,8 +438,9 @@ class TestSubmodule(TestBase): assert not sm.module_exists() # was never updated after rwrepo's clone # assure we clone from a local source - sm.config_writer().set_value( - 'url', to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, sm.path))) + writer = sm.config_writer() + writer.set_value('url', to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, sm.path))) + writer.release() # dry-run does nothing sm.update(recursive=False, dry_run=True, progress=prog) @@ -439,7 +448,9 @@ class TestSubmodule(TestBase): sm.update(recursive=False) assert sm.module_exists() - sm.config_writer().set_value('path', fp) # change path to something with prefix AFTER url change + writer = sm.config_writer() + writer.set_value('path', fp) # change path to something with prefix AFTER url change + writer.release() # update fails as list_items in such a situations cannot work, as it cannot # find the entry at the changed path @@ -503,7 +514,9 @@ class TestSubmodule(TestBase): # repository at the different url nsm.set_parent_commit(csmremoved) nsmurl = to_native_path_linux(join_path_native(self.rorepo.working_tree_dir, rsmsp[0])) - nsm.config_writer().set_value('url', nsmurl) + writer = nsm.config_writer() + writer.set_value('url', nsmurl) + writer.release() csmpathchange = rwrepo.index.commit("changed url") nsm.set_parent_commit(csmpathchange) @@ -531,7 +544,9 @@ class TestSubmodule(TestBase): nsmm = nsm.module() prev_commit = nsmm.head.commit for branch in ("some_virtual_branch", cur_branch.name): - nsm.config_writer().set_value(Submodule.k_head_option, git.Head.to_full_path(branch)) + writer = nsm.config_writer() + writer.set_value(Submodule.k_head_option, git.Head.to_full_path(branch)) + writer.release() csmbranchchange = rwrepo.index.commit("changed branch to %s" % branch) nsm.set_parent_commit(csmbranchchange) # END for each branch to change @@ -559,7 +574,9 @@ class TestSubmodule(TestBase): assert nsm.exists() and nsm.module_exists() and len(nsm.children()) >= 1 # assure we pull locally only nsmc = nsm.children()[0] - nsmc.config_writer().set_value('url', async_url) + writer = nsmc.config_writer() + writer.set_value('url', async_url) + writer.release() rm.update(recursive=True, progress=prog, dry_run=True) # just to run the code rm.update(recursive=True, progress=prog) diff --git a/git/test/test_tree.py b/git/test/test_tree.py index d2e3606b..7a16b777 100644 --- a/git/test/test_tree.py +++ b/git/test/test_tree.py @@ -11,7 +11,7 @@ from git import ( Blob ) -from cStringIO import StringIO +from io import BytesIO class TestTree(TestBase): @@ -30,7 +30,7 @@ class TestTree(TestBase): orig_data = tree.data_stream.read() orig_cache = tree._cache - stream = StringIO() + stream = BytesIO() tree._serialize(stream) assert stream.getvalue() == orig_data @@ -82,7 +82,7 @@ class TestTree(TestBase): mod.set_done() # multiple times are okay # serialize, its different now - stream = StringIO() + stream = BytesIO() testtree._serialize(stream) stream.seek(0) assert stream.getvalue() != orig_data @@ -138,6 +138,7 @@ class TestTree(TestBase): # END check for slash # slashes in paths are supported as well + # NOTE: on py3, / doesn't work with strings anymore ... assert root[item.path] == item == root / item.path # END for each item assert found_slash diff --git a/git/test/test_util.py b/git/test/test_util.py index 888eb4ee..c6ca6920 100644 --- a/git/test/test_util.py +++ b/git/test/test_util.py @@ -24,6 +24,7 @@ from git.objects.util import ( parse_date, ) from git.cmd import dashify +from git.compat import string_types import time @@ -104,7 +105,7 @@ class TestUtils(TestBase): # now that we are here, test our conversion functions as well utctz = altz_to_utctz_str(offset) - assert isinstance(utctz, basestring) + assert isinstance(utctz, string_types) assert utctz_to_altz(verify_utctz(utctz)) == offset # END assert rval utility diff --git a/git/util.py b/git/util.py index fecd9fa2..4de736d3 100644 --- a/git/util.py +++ b/git/util.py @@ -15,7 +15,8 @@ import getpass # NOTE: Some of the unused imports might be used/imported by others. # Handle once test-cases are back up and running. -from exc import GitCommandError +from .exc import GitCommandError +from .compat import MAXSIZE # Most of these are unused here, but are for use by git-python modules so these # don't see gitdb all the time. Flake of course doesn't like it. @@ -445,7 +446,7 @@ class IndexFileSHA1Writer(object): def __init__(self, f): self.f = f - self.sha1 = make_sha("") + self.sha1 = make_sha(b"") def write(self, data): self.sha1.update(data) @@ -489,10 +490,7 @@ class LockFile(object): def _has_lock(self): """:return: True if we have a lock and if the lockfile still exists :raise AssertionError: if our lock-file does not exist""" - if not self._owns_lock: - return False - - return True + return self._owns_lock def _obtain_lock_or_raise(self): """Create a lock file as flag for other instances, mark our instance as lock-holder @@ -530,7 +528,7 @@ class LockFile(object): # on bloody windows, the file needs write permissions to be removable. # Why ... if os.name == 'nt': - os.chmod(lfp, int("0777", 8)) + os.chmod(lfp, 0o777) # END handle win32 os.remove(lfp) except OSError: @@ -548,7 +546,7 @@ class BlockingLockFile(LockFile): can never be obtained.""" __slots__ = ("_check_interval", "_max_block_time") - def __init__(self, file_path, check_interval_s=0.3, max_block_time_s=sys.maxint): + def __init__(self, file_path, check_interval_s=0.3, max_block_time_s=MAXSIZE): """Configure the instance :parm check_interval_s: @@ -112,8 +112,8 @@ GitPython is a python library used to interact with Git repositories""", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", - # "Programming Language :: Python :: 3", - # "Programming Language :: Python :: 3.3", - # "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", ] ) @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,flake8 +envlist = py26,py27,py33,py34,flake8 [testenv] commands = nosetests {posargs} |