From 033fdce822ded8f20d1860eac9b3d9cd3fe1f336 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sat, 29 Jun 2013 16:54:17 -0500 Subject: Add svn_util.py http://bugs.python.org/setuptools/file51/svn_versioning_2.patch by Jason Coombs --- setuptools/svn_utils.py | 110 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 setuptools/svn_utils.py (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py new file mode 100644 index 00000000..2a200fdf --- /dev/null +++ b/setuptools/svn_utils.py @@ -0,0 +1,110 @@ +import os +import re + +def get_entries_files(base, recurse=True): + for base,dirs,files in os.walk(os.curdir): + if '.svn' not in dirs: + dirs[:] = [] + continue # no sense walking uncontrolled subdirs + dirs.remove('.svn') + f = open(os.path.join(base,'.svn','entries')) + yield f.read() + f.close() + +class SVNEntries(object): + def __init__(self, data): + self.data = data + + @classmethod + def load(class_, base): + filename = os.path.join(base, '.svn', 'entries') + f = open(filename) + result = SVNEntries.read(f) + f.close() + return result + + @classmethod + def read(class_, file): + data = file.read() + is_xml = data.startswith('revision_line_number + and section[revision_line_number] + ] + return rev_numbers + + def get_undeleted_records(self): + undeleted = lambda s: s and s[0] and (len(s) < 6 or s[5] != 'delete') + result = [ + section[0] + for section in self.get_sections() + if undeleted(section) + ] + return result + +class SVNEntriesXML(SVNEntries): + def is_valid(self): + return True + + def get_url(self): + "Get repository URL" + urlre = re.compile('url="([^"]+)"') + return urlre.search(self.data).group(1) + + def parse_revision_numbers(self): + revre = re.compile('committed-rev="(\d+)"') + return [ + int(m.group(1)) + for m in revre.finditer(self.data) + ] + + def get_undeleted_records(self): + entries_pattern = re.compile(r'name="([^"]+)"(?![^>]+deleted="true")', re.I) + results = [ + unescape(match.group(1)) + for match in entries_pattern.finditer(self.data) + ] + return results + -- cgit v1.2.1 From 9abde94f08994212b960ff8677a1275ea311140e Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sun, 30 Jun 2013 00:13:56 -0500 Subject: Added a class for getting SVN information with a cmd. Using XML because the regular output is probably internationalized. Need to inspect setuptools/command/egg_info.py setuptools/command/sdist.py setuptools/package_index.py to see how used again to see if we will every be parsing a raw XML file or if always from .svn/enteries, if so looks like we can just replace the whole of the file parsing. --- setuptools/svn_utils.py | 116 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 16 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 2a200fdf..ad3b5f15 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -1,6 +1,10 @@ import os import re +#requires python >= 2.4 +from subprocess import Popen as _Popen, PIPE as _PIPE + + def get_entries_files(base, recurse=True): for base,dirs,files in os.walk(os.curdir): if '.svn' not in dirs: @@ -12,6 +16,17 @@ def get_entries_files(base, recurse=True): f.close() class SVNEntries(object): + known_svn_versions = { + '1.4.x': 8, + '1.5.x': 9, + '1.6.x': 10, + #11 didn't exist (maybe 1.7-dev) + #12 is the number in the file there is another + #number in .svn/wb.db that is at larger so + #looks like they won't be updating it any longer. + '1.7.x+': 12, + } + def __init__(self, data): self.data = data @@ -26,8 +41,27 @@ class SVNEntries(object): @classmethod def read(class_, file): data = file.read() - is_xml = data.startswith(']*path="(\.+)">', re.I) + entryre = re.compile(r'', re.M or re.I) + urlre = re.compile('(.*?)', re.I) + revre = re.compile(']*revision="(\d+)"', re.I) + namere = re.compile('(.*?)', re.I) + + def __get_cached_dir_data(self): + return self.dir_data + + def __get_cached_entries(self): + return self.entries + + def is_valid(self): + return bool(self.get_dir_data()) + + def get_dir_data(self): + #regard the shell argument, see: http://bugs.python.org/issue8557 + # and http://stackoverflow.com/questions/5658622/python-subprocess-popen-environment-path + proc = _Popen(['svn', 'info', '--xml', self.path], + stdout=_PIPE, shell=(sys.platform=='win32')) + data = unicode(proc.communicate()[0], encoding='utf-8') + self.dir_data = self.entryre.findall(data) + self.get_dir_data = self.__get_cached_dir_data + return self.dir_data + + def get_entries(self): + #regard the shell argument, see: http://bugs.python.org/issue8557 + # and http://stackoverflow.com/questions/5658622/python-subprocess-popen-environment-path + proc = _Popen(['svn', 'list', '--xml', self.path], + stdout=_PIPE, shell=(sys.platform=='win32')) + data = unicode(proc.communicate()[0], encoding='utf-8') + self.dir_data = self.entryre.findall(data) + self.get_dir_data = self.__get_cached_dir_data + return self.dir_data + + def get_url(self): + "Get repository URL" + return self.urlre.search(self.get_sections()[0]).group(1) + + def parse_revision_numbers(self): + #NOTE: if one has recently commited, the new revision doesn't get updated until svn update + if not self.is_valid(): + return list() + else: + return [ + int(m.group(1)) + for entry in self.get_enteries() + for m in self.revre.finditer(entry) + ] + + def get_undeleted_records(self): + #NOTE: Need to pars enteries? + if not self.is_valid(): + return list() + else: + return [ + m.group(1)) + for entry in self.get_enteries() + for m in self.namere.finditer(entry) + ] + -- cgit v1.2.1 From d907aab3c2c6ec26f0550152bbaaa513e4d05328 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sun, 30 Jun 2013 00:30:48 -0500 Subject: cleaned up the svn module, I think I can drop the old entry parsing all together --- setuptools/svn_utils.py | 86 +++++++++---------------------------------------- 1 file changed, 15 insertions(+), 71 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index ad3b5f15..3311d9cf 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -15,96 +15,40 @@ def get_entries_files(base, recurse=True): yield f.read() f.close() +#It would seem that svn info --xml and svn list --xml were fully supported by 1.3.x +#the special casing of the entry files seem to start at 1.4.x, so if we check +#for xml in entries and then fall back to the command line, this should catch everything. + class SVNEntries(object): - known_svn_versions = { - '1.4.x': 8, - '1.5.x': 9, - '1.6.x': 10, - #11 didn't exist (maybe 1.7-dev) - #12 is the number in the file there is another - #number in .svn/wb.db that is at larger so - #looks like they won't be updating it any longer. - '1.7.x+': 12, - } - - def __init__(self, data): + + def __init__(self, path, data): + self.path = path self.data = data @classmethod def load(class_, base): filename = os.path.join(base, '.svn', 'entries') f = open(filename) - result = SVNEntries.read(f) + result = SVNEntries.read(f, None) f.close() return result @classmethod - def read(class_, file): + def read(class_, file, path=None): data = file.read() if data.startswith('revision_line_number - and section[revision_line_number] - ] - return rev_numbers - - def get_undeleted_records(self): - undeleted = lambda s: s and s[0] and (len(s) < 6 or s[5] != 'delete') - result = [ - section[0] - for section in self.get_sections() - if undeleted(section) - ] - return result - class SVNEntriesXML(SVNEntries): def is_valid(self): return True @@ -171,7 +115,7 @@ class SVNEntriesCMD(SVNEntries): return self.urlre.search(self.get_sections()[0]).group(1) def parse_revision_numbers(self): - #NOTE: if one has recently commited, the new revision doesn't get updated until svn update + #NOTE: if one has recently committed, the new revision doesn't get updated until SVN update if not self.is_valid(): return list() else: @@ -182,7 +126,7 @@ class SVNEntriesCMD(SVNEntries): ] def get_undeleted_records(self): - #NOTE: Need to pars enteries? + #NOTE: Need to parse entities? if not self.is_valid(): return list() else: -- cgit v1.2.1 From a59adb15e013a6cc5701c8a3028bfeccbc2af612 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sun, 30 Jun 2013 02:04:47 -0500 Subject: added querying externals to the classes --- setuptools/svn_utils.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 3311d9cf..d36f64ea 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -49,6 +49,28 @@ class SVNEntries(object): all_revs = self.parse_revision_numbers() + [0] return max(all_revs) + def __get_cached_external_dirs(self): + return self.external_dirs + + def get_external_dirs(self): + #regard the shell argument, see: http://bugs.python.org/issue8557 + # and http://stackoverflow.com/questions/5658622/python-subprocess-popen-environment-path + proc = _Popen(['svn', 'propget', self.path], + stdout=_PIPE, shell=(sys.platform=='win32')) + data = unicode(proc.communicate()[0], encoding='utf-8').splitlines() + data = [line.split() for line in data] + + # http://svnbook.red-bean.com/en/1.6/svn.advanced.externals.html + #there appears to be three possible formats for externals since 1.5 + #but looks like we only need the local relative path names so it's just + #2 either the first column or the last (of 2 or 3) + index = -1 + if all("://" in line[-1] for line in data): + index = 0 + + self.external_dirs = [line[index] for line in data] + return self.dir_data + class SVNEntriesXML(SVNEntries): def is_valid(self): return True -- cgit v1.2.1 From d49263e8af94ea98e5b5a10f2fb9eb7c22e83946 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sun, 30 Jun 2013 13:13:33 -0500 Subject: Added SVNTextEntries for the moment as a fallback for no SVN/Rev8-10 Added Externals processing for all formats Will use dir-prop[-base] as a fallback otherwise CMD. --- setuptools/svn_utils.py | 135 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 121 insertions(+), 14 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index d36f64ea..dff1c4b0 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -19,14 +19,29 @@ def get_entries_files(base, recurse=True): #the special casing of the entry files seem to start at 1.4.x, so if we check #for xml in entries and then fall back to the command line, this should catch everything. +#TODO add the text entry back, and make its use dependent on the non existence of svn? + class SVNEntries(object): + svn_tool_version = '' def __init__(self, path, data): self.path = path self.data = data + if not self.svn_tool_version: + self.svn_tool_version = self.get_svn_tool_version() + + @staticmethod + def get_svn_tool_version(): + proc = _Popen(['svn', 'propget', self.path], + stdout=_PIPE, shell=(sys.platform=='win32')) + data = unicode(proc.communicate()[0], encoding='utf-8') + if data is not not: + return data.strip() + else: + return '' @classmethod - def load(class_, base): + def load_dir(class_, base): filename = os.path.join(base, '.svn', 'entries') f = open(filename) result = SVNEntries.read(f, None) @@ -41,9 +56,14 @@ class SVNEntries(object): #entries were originally xml so pre-1.4.x return SVNEntriesXML(data, path) else if path is None: - raise ValueError('Must have path to call svn') + result = SVNEntriesText(data, path) else: - return SVNEntriesCMD(data, path) + class_.svn_tool_version = class_.get_svn_tool_version() + result = SVNEntriesText(data, path) + if result.is_valid(): + return svnentriescmd(data, path) + else: + return result def parse_revision(self): all_revs = self.parse_revision_numbers() + [0] @@ -52,12 +72,29 @@ class SVNEntries(object): def __get_cached_external_dirs(self): return self.external_dirs - def get_external_dirs(self): - #regard the shell argument, see: http://bugs.python.org/issue8557 - # and http://stackoverflow.com/questions/5658622/python-subprocess-popen-environment-path - proc = _Popen(['svn', 'propget', self.path], - stdout=_PIPE, shell=(sys.platform=='win32')) - data = unicode(proc.communicate()[0], encoding='utf-8').splitlines() + def __get_externals_data(self, filename): + found = False + f = open(filename,'rt') + for line in iter(f.readline, ''): # can't use direct iter! + parts = line.split() + if len(parts)==2: + kind,length = parts + data = f.read(int(length)) + if kind=='K' and data=='svn:externals': + found = True + elif kind=='V' and found: + f.close() + break + else: + f.close() + return '' + + def get_external_dirs(self, filename): + data = self.__get_externals_data(filename) + + if not data: + return + data = [line.split() for line in data] # http://svnbook.red-bean.com/en/1.6/svn.advanced.externals.html @@ -65,11 +102,63 @@ class SVNEntries(object): #but looks like we only need the local relative path names so it's just #2 either the first column or the last (of 2 or 3) index = -1 - if all("://" in line[-1] for line in data): + if all("://" in line[-1] or not line[-1] for line in data): index = 0 - - self.external_dirs = [line[index] for line in data] - return self.dir_data + + self.external_dirs = [line[index] for line in data if line[index]] + self.get_external_dirs = self.__get_cached_external_dirs + return self.external_dirs + +class SVNEntriesText(SVNEntries): + known_svn_versions = { + '1.4.x': 8, + '1.5.x': 9, + '1.6.x': 10, + } + + def __get_cached_sections(self): + return self.sections + + def get_sections(self): + SECTION_DIVIDER = '\f\n' # or '\n\x0c\n'? + sections = self.data.split(SECTION_DIVIDER) + sections = list(map(str.splitlines, sections)) + try: + # remove the SVN version number from the first line + svn_version = int(sections[0].pop(0)) + if not svn_version in self.known_svn_versions.values(): + log.warn("Unknown subversion verson %d", svn_version) + except ValueError: + return + self.sections = sections + self.get_sections = self.__get_cached_sections + return self.sections + + def is_valid(self): + return bool(self.get_sections()) + + def get_url(self): + return self.get_sections()[0][4] + + def parse_revision_numbers(self): + revision_line_number = 9 + rev_numbers = [ + int(section[revision_line_number]) + for section in self.get_sections() + if len(section)>revision_line_number + and section[revision_line_number] + ] + return rev_numbers + + def get_undeleted_records(self): + undeleted = lambda s: s and s[0] and (len(s) < 6 or s[5] != 'delete') + result = [ + section[0] + for section in self.get_sections() + if undeleted(section) + ] + return result + class SVNEntriesXML(SVNEntries): def is_valid(self): @@ -85,6 +174,7 @@ class SVNEntriesXML(SVNEntries): return [ int(m.group(1)) for m in revre.finditer(self.data) + if m.group(1) ] def get_undeleted_records(self): @@ -92,6 +182,7 @@ class SVNEntriesXML(SVNEntries): results = [ unescape(match.group(1)) for match in entries_pattern.finditer(self.data) + if m.group(1) ] return results @@ -145,6 +236,7 @@ class SVNEntriesCMD(SVNEntries): int(m.group(1)) for entry in self.get_enteries() for m in self.revre.finditer(entry) + if m.group(1) ] def get_undeleted_records(self): @@ -155,6 +247,21 @@ class SVNEntriesCMD(SVNEntries): return [ m.group(1)) for entry in self.get_enteries() - for m in self.namere.finditer(entry) + for m in self.namere.finditer(entry) + if m.group(1) ] + def __get_externals_data(self, filename): + + #othewise will be called twice. + if filename.lower() != 'dir-props': + return '' + + #regard the shell argument, see: http://bugs.python.org/issue8557 + # and http://stackoverflow.com/questions/5658622/python-subprocess-popen-environment-path + proc = _Popen(['svn', 'propget', self.path], + stdout=_PIPE, shell=(sys.platform=='win32')) + try: + return unicode(proc.communicate()[0], encoding='utf-8').splitlines() + except ValueError: + return '' -- cgit v1.2.1 From 484a429fceee9708e9f5db8ce78c33c390a1a572 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sun, 30 Jun 2013 19:05:00 -0500 Subject: minor cleanups, added imports, --- setuptools/svn_utils.py | 118 ++++++++++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 50 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index dff1c4b0..34adc75e 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -1,25 +1,37 @@ import os import re +import sys +from distutils import log +from xml.sax.saxutils import unescape #requires python >= 2.4 from subprocess import Popen as _Popen, PIPE as _PIPE def get_entries_files(base, recurse=True): - for base,dirs,files in os.walk(os.curdir): + for base, dirs, _ in os.walk(os.curdir): if '.svn' not in dirs: dirs[:] = [] continue # no sense walking uncontrolled subdirs dirs.remove('.svn') - f = open(os.path.join(base,'.svn','entries')) + f = open(os.path.join(base, '.svn', 'entries')) yield f.read() f.close() -#It would seem that svn info --xml and svn list --xml were fully supported by 1.3.x -#the special casing of the entry files seem to start at 1.4.x, so if we check -#for xml in entries and then fall back to the command line, this should catch everything. +#It would seem that svn info --xml and svn list --xml were fully +#supported by 1.3.x the special casing of the entry files seem to start at +#1.4.x, so if we check for xml in entries and then fall back to the command +#line, this should catch everything. -#TODO add the text entry back, and make its use dependent on the non existence of svn? +#subprocess is called several times with shell=(sys.platform=='win32') +#see the follow for more information: +# http://bugs.python.org/issue8557 +# http://stackoverflow.com/questions/5658622/ +# python-subprocess-popen-environment-path + + +#TODO add the text entry back, and make its use dependent on the +# non existence of svn? class SVNEntries(object): svn_tool_version = '' @@ -30,12 +42,12 @@ class SVNEntries(object): if not self.svn_tool_version: self.svn_tool_version = self.get_svn_tool_version() - @staticmethod + @staticmethod def get_svn_tool_version(): - proc = _Popen(['svn', 'propget', self.path], + proc = _Popen(['svn', '--version', '--quiet'], stdout=_PIPE, shell=(sys.platform=='win32')) data = unicode(proc.communicate()[0], encoding='utf-8') - if data is not not: + if data: return data.strip() else: return '' @@ -55,13 +67,13 @@ class SVNEntries(object): if data.startswith(']+deleted="true")', re.I) + entries_pattern = re.compile(r'name="([^"]+)"(?![^>]+deleted="true")', + re.I) results = [ unescape(match.group(1)) for match in entries_pattern.finditer(self.data) - if m.group(1) + if match.group(1) ] return results @@ -190,9 +210,9 @@ class SVNEntriesXML(SVNEntries): class SVNEntriesCMD(SVNEntries): entrypathre = re.compile(r']*path="(\.+)">', re.I) entryre = re.compile(r'', re.M or re.I) - urlre = re.compile('(.*?)', re.I) - revre = re.compile(']*revision="(\d+)"', re.I) - namere = re.compile('(.*?)', re.I) + urlre = re.compile(r'(.*?)', re.I) + revre = re.compile(r']*revision="(\d+)"', re.I) + namere = re.compile(r'(.*?)', re.I) def __get_cached_dir_data(self): return self.dir_data @@ -204,9 +224,8 @@ class SVNEntriesCMD(SVNEntries): return bool(self.get_dir_data()) def get_dir_data(self): - #regard the shell argument, see: http://bugs.python.org/issue8557 - # and http://stackoverflow.com/questions/5658622/python-subprocess-popen-environment-path - proc = _Popen(['svn', 'info', '--xml', self.path], + #regarding the shell argument, see: http://bugs.python.org/issue8557 + proc = _Popen(['svn', 'info', '--xml', self.path], stdout=_PIPE, shell=(sys.platform=='win32')) data = unicode(proc.communicate()[0], encoding='utf-8') self.dir_data = self.entryre.findall(data) @@ -214,39 +233,39 @@ class SVNEntriesCMD(SVNEntries): return self.dir_data def get_entries(self): - #regard the shell argument, see: http://bugs.python.org/issue8557 - # and http://stackoverflow.com/questions/5658622/python-subprocess-popen-environment-path - proc = _Popen(['svn', 'list', '--xml', self.path], + #regarding the shell argument, see: http://bugs.python.org/issue8557 + proc = _Popen(['svn', 'list', '--xml', self.path], stdout=_PIPE, shell=(sys.platform=='win32')) data = unicode(proc.communicate()[0], encoding='utf-8') - self.dir_data = self.entryre.findall(data) + self.entries = self.entryre.findall(data) self.get_dir_data = self.__get_cached_dir_data - return self.dir_data + return self.entries def get_url(self): "Get repository URL" - return self.urlre.search(self.get_sections()[0]).group(1) + return self.urlre.search(self.get_entries()[0]).group(1) def parse_revision_numbers(self): - #NOTE: if one has recently committed, the new revision doesn't get updated until SVN update + #NOTE: if one has recently committed, + # the new revision doesn't get updated until SVN update if not self.is_valid(): return list() else: return [ - int(m.group(1)) - for entry in self.get_enteries() + int(m.group(1)) + for entry in self.get_enries() for m in self.revre.finditer(entry) if m.group(1) ] - + def get_undeleted_records(self): #NOTE: Need to parse entities? if not self.is_valid(): return list() else: return [ - m.group(1)) - for entry in self.get_enteries() + m.group(1) + for entry in self.get_entries() for m in self.namere.finditer(entry) if m.group(1) ] @@ -258,8 +277,7 @@ class SVNEntriesCMD(SVNEntries): return '' #regard the shell argument, see: http://bugs.python.org/issue8557 - # and http://stackoverflow.com/questions/5658622/python-subprocess-popen-environment-path - proc = _Popen(['svn', 'propget', self.path], + proc = _Popen(['svn', 'propget', self.path], stdout=_PIPE, shell=(sys.platform=='win32')) try: return unicode(proc.communicate()[0], encoding='utf-8').splitlines() -- cgit v1.2.1 From 18e5a03131563a31ccae33daa1dacd04fffbedd0 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sun, 30 Jun 2013 19:06:22 -0500 Subject: Oops didn't return a value --- setuptools/svn_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 34adc75e..52a1c07b 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -68,7 +68,7 @@ class SVNEntries(object): #entries were originally xml so pre-1.4.x return SVNEntriesXML(data, path) elif path is None: - result = SVNEntriesText(data, path) + return SVNEntriesText(data, path) else: class_.svn_tool_version = class_.get_svn_tool_version() result = SVNEntriesText(data, path) -- cgit v1.2.1 From bed501651b6e2811466ec344ce38249e6e5cfa14 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sun, 30 Jun 2013 21:11:35 -0500 Subject: Finished some 1.7 tests, and updated the zip file to include the .svn dirs Several fixes to get the code to pass the tests --- setuptools/svn_utils.py | 69 ++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 29 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 52a1c07b..7d30d322 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -45,7 +45,8 @@ class SVNEntries(object): @staticmethod def get_svn_tool_version(): proc = _Popen(['svn', '--version', '--quiet'], - stdout=_PIPE, shell=(sys.platform=='win32')) + stdout=_PIPE, stderr=_PIPE, + shell=(sys.platform=='win32')) data = unicode(proc.communicate()[0], encoding='utf-8') if data: return data.strip() @@ -56,7 +57,7 @@ class SVNEntries(object): def load_dir(class_, base): filename = os.path.join(base, '.svn', 'entries') f = open(filename) - result = SVNEntries.read(f, None) + result = SVNEntries.read(f, base) f.close() return result @@ -66,16 +67,15 @@ class SVNEntries(object): if data.startswith(']*path="(\.+)">', re.I) - entryre = re.compile(r'', re.M or re.I) - urlre = re.compile(r'(.*?)', re.I) + entryre = re.compile(r'', re.DOTALL or re.I) + urlre = re.compile(r'(.*?)', re.I) revre = re.compile(r']*revision="(\d+)"', re.I) namere = re.compile(r'(.*?)', re.I) @@ -238,12 +247,12 @@ class SVNEntriesCMD(SVNEntries): stdout=_PIPE, shell=(sys.platform=='win32')) data = unicode(proc.communicate()[0], encoding='utf-8') self.entries = self.entryre.findall(data) - self.get_dir_data = self.__get_cached_dir_data + self.get_entries = self.__get_cached_entries return self.entries def get_url(self): "Get repository URL" - return self.urlre.search(self.get_entries()[0]).group(1) + return self.urlre.search(self.get_dir_data()[0]).group(1) def parse_revision_numbers(self): #NOTE: if one has recently committed, @@ -253,7 +262,7 @@ class SVNEntriesCMD(SVNEntries): else: return [ int(m.group(1)) - for entry in self.get_enries() + for entry in self.get_entries() for m in self.revre.finditer(entry) if m.group(1) ] @@ -270,16 +279,18 @@ class SVNEntriesCMD(SVNEntries): if m.group(1) ] - def __get_externals_data(self, filename): + def _get_externals_data(self, filename): #othewise will be called twice. - if filename.lower() != 'dir-props': + if os.path.basename(filename).lower() != 'dir-props': return '' #regard the shell argument, see: http://bugs.python.org/issue8557 - proc = _Popen(['svn', 'propget', self.path], + proc = _Popen(['svn', 'propget', 'svn:externals', self.path], stdout=_PIPE, shell=(sys.platform=='win32')) try: - return unicode(proc.communicate()[0], encoding='utf-8').splitlines() + lines = unicode(proc.communicate()[0], encoding='utf-8') + lines = [line for line in lines.splitlines() if line] + return lines except ValueError: return '' -- cgit v1.2.1 From 866761320ed69d4a2085b05c66fd78f4f9c04fff Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sun, 30 Jun 2013 22:02:20 -0500 Subject: py3 fixes --- setuptools/svn_utils.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 7d30d322..86fed4d1 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -1,6 +1,7 @@ import os import re import sys +import codecs from distutils import log from xml.sax.saxutils import unescape @@ -8,15 +9,6 @@ from xml.sax.saxutils import unescape from subprocess import Popen as _Popen, PIPE as _PIPE -def get_entries_files(base, recurse=True): - for base, dirs, _ in os.walk(os.curdir): - if '.svn' not in dirs: - dirs[:] = [] - continue # no sense walking uncontrolled subdirs - dirs.remove('.svn') - f = open(os.path.join(base, '.svn', 'entries')) - yield f.read() - f.close() #It would seem that svn info --xml and svn list --xml were fully #supported by 1.3.x the special casing of the entry files seem to start at @@ -28,7 +20,19 @@ def get_entries_files(base, recurse=True): # http://bugs.python.org/issue8557 # http://stackoverflow.com/questions/5658622/ # python-subprocess-popen-environment-path +def _run_command(args, stdout=_PIPE, stderr=_PIPE): + #regarding the shell argument, see: http://bugs.python.org/issue8557 + proc = _Popen(args, stdout=stdout, stderr=stderr, + shell=(sys.platform=='win32')) + data = proc.communicate()[0] + #TODO: this is probably NOT always utf-8 + try: + data = unicode(data, encoding='utf-8') + except NameError: + data = str(data, encoding='utf-8') + + return proc.returncode, data #TODO add the text entry back, and make its use dependent on the # non existence of svn? @@ -44,10 +48,7 @@ class SVNEntries(object): @staticmethod def get_svn_tool_version(): - proc = _Popen(['svn', '--version', '--quiet'], - stdout=_PIPE, stderr=_PIPE, - shell=(sys.platform=='win32')) - data = unicode(proc.communicate()[0], encoding='utf-8') + _, data = _run_command(['svn', '--version', '--quiet']) if data: return data.strip() else: @@ -234,18 +235,14 @@ class SVNEntriesCMD(SVNEntries): def get_dir_data(self): #regarding the shell argument, see: http://bugs.python.org/issue8557 - proc = _Popen(['svn', 'info', '--xml', self.path], - stdout=_PIPE, shell=(sys.platform=='win32')) - data = unicode(proc.communicate()[0], encoding='utf-8') + _, data = _run_command(['svn', 'info', '--xml', self.path]) self.dir_data = self.entryre.findall(data) self.get_dir_data = self.__get_cached_dir_data return self.dir_data def get_entries(self): #regarding the shell argument, see: http://bugs.python.org/issue8557 - proc = _Popen(['svn', 'list', '--xml', self.path], - stdout=_PIPE, shell=(sys.platform=='win32')) - data = unicode(proc.communicate()[0], encoding='utf-8') + _, data = _run_command(['svn', 'list', '--xml', self.path]) self.entries = self.entryre.findall(data) self.get_entries = self.__get_cached_entries return self.entries @@ -285,11 +282,9 @@ class SVNEntriesCMD(SVNEntries): if os.path.basename(filename).lower() != 'dir-props': return '' - #regard the shell argument, see: http://bugs.python.org/issue8557 - proc = _Popen(['svn', 'propget', 'svn:externals', self.path], - stdout=_PIPE, shell=(sys.platform=='win32')) try: - lines = unicode(proc.communicate()[0], encoding='utf-8') + _, lines = _run_command(['svn', + 'propget', 'svn:externals', self.path]) lines = [line for line in lines.splitlines() if line] return lines except ValueError: -- cgit v1.2.1 From 72966e2b78d61d9716aa2eccdc72183ad3a4e22a Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 09:55:43 -0500 Subject: If using a command, we can query the working copy version directly. --- setuptools/svn_utils.py | 52 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 12 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 86fed4d1..c052f671 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -216,14 +216,40 @@ class SVNEntriesXML(SVNEntries): ] return results +import xml.dom.pulldom class SVNEntriesCMD(SVNEntries): + # + # + # + # + # ... + # ... + # + # + # ... + # + # + # ... + # + # + # ... + # entrypathre = re.compile(r']*path="(\.+)">', re.I) entryre = re.compile(r'', re.DOTALL or re.I) urlre = re.compile(r'(.*?)', re.I) revre = re.compile(r']*revision="(\d+)"', re.I) namere = re.compile(r'(.*?)', re.I) + #svnversion return values (previous implementations return max revision) + # 4123:4168 mixed revision working copy + # 4168M modified working copy + # 4123S switched working copy + # 4123:4168MS mixed revision, modified, switched working copy + svnverre = re.compile(r'(?:(\d+):)?(\d+)([a-z]*)\s*$', re.I) + def __get_cached_dir_data(self): return self.dir_data @@ -251,18 +277,20 @@ class SVNEntriesCMD(SVNEntries): "Get repository URL" return self.urlre.search(self.get_dir_data()[0]).group(1) - def parse_revision_numbers(self): - #NOTE: if one has recently committed, - # the new revision doesn't get updated until SVN update - if not self.is_valid(): - return list() - else: - return [ - int(m.group(1)) - for entry in self.get_entries() - for m in self.revre.finditer(entry) - if m.group(1) - ] + + def parse_revision(self): + _, data = _run_command(['svnversion', self.path]) + parsed = self.svnverre.match(data) + if parsed: + log.warn('Parsed!') + try: + #No max needed this command summarizes working copy since 1.0 + return int(parsed.group(2)) + except ValueError: + #This should only happen if the revision is WAY too big. + pass + log.warn(repr(data)) + return 0 def get_undeleted_records(self): #NOTE: Need to parse entities? -- cgit v1.2.1 From 3b871c4ee16fa03cfe87776267095465e213f8e0 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 10:01:05 -0500 Subject: If using a command, we can query the working copy version directly. --- setuptools/svn_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index c052f671..bbf10da7 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -277,6 +277,8 @@ class SVNEntriesCMD(SVNEntries): "Get repository URL" return self.urlre.search(self.get_dir_data()[0]).group(1) + def __get_cached_revision(self): + return self.revision def parse_revision(self): _, data = _run_command(['svnversion', self.path]) @@ -285,11 +287,12 @@ class SVNEntriesCMD(SVNEntries): log.warn('Parsed!') try: #No max needed this command summarizes working copy since 1.0 - return int(parsed.group(2)) + self.revision = int(parsed.group(2)) + self.parse_revision = self.__get_cached_revision + return self.revision except ValueError: #This should only happen if the revision is WAY too big. pass - log.warn(repr(data)) return 0 def get_undeleted_records(self): -- cgit v1.2.1 From 5fcf3d220cb37c29c121492803b210d3d4e22f8a Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 10:01:55 -0500 Subject: Remove some temporary logging --- setuptools/svn_utils.py | 1 - 1 file changed, 1 deletion(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index bbf10da7..e46e5ab6 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -284,7 +284,6 @@ class SVNEntriesCMD(SVNEntries): _, data = _run_command(['svnversion', self.path]) parsed = self.svnverre.match(data) if parsed: - log.warn('Parsed!') try: #No max needed this command summarizes working copy since 1.0 self.revision = int(parsed.group(2)) -- cgit v1.2.1 From ea662e20c92e57795176c85a20b3423f59efa6fb Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 11:06:35 -0500 Subject: get_url now uses pulldom --- setuptools/svn_utils.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index e46e5ab6..57cf6849 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -4,12 +4,12 @@ import sys import codecs from distutils import log from xml.sax.saxutils import unescape +import xml.dom.pulldom #requires python >= 2.4 from subprocess import Popen as _Popen, PIPE as _PIPE - #It would seem that svn info --xml and svn list --xml were fully #supported by 1.3.x the special casing of the entry files seem to start at #1.4.x, so if we check for xml in entries and then fall back to the command @@ -180,6 +180,7 @@ class SVNEntriesText(SVNEntries): return rev_numbers def get_undeleted_records(self): + #Note for self this skip and only returns the children undeleted = lambda s: s and s[0] and (len(s) < 6 or s[5] != 'delete') result = [ section[0] @@ -239,8 +240,6 @@ class SVNEntriesCMD(SVNEntries): # entrypathre = re.compile(r']*path="(\.+)">', re.I) entryre = re.compile(r'', re.DOTALL or re.I) - urlre = re.compile(r'(.*?)', re.I) - revre = re.compile(r']*revision="(\d+)"', re.I) namere = re.compile(r'(.*?)', re.I) #svnversion return values (previous implementations return max revision) @@ -257,13 +256,22 @@ class SVNEntriesCMD(SVNEntries): return self.entries def is_valid(self): - return bool(self.get_dir_data()) + return self.get_dir_data() is not None def get_dir_data(self): - #regarding the shell argument, see: http://bugs.python.org/issue8557 + #This returns the info entry for the directory ONLY _, data = _run_command(['svn', 'info', '--xml', self.path]) - self.dir_data = self.entryre.findall(data) - self.get_dir_data = self.__get_cached_dir_data + + doc = xml.dom.pulldom.parseString(data) + self.dir_data = None + for event, node in doc: + if event=='START_ELEMENT' and node.nodeName=='entry': + doc.expandNode(node) + self.dir_data = node + break + + if self.dir_data: + self.get_dir_data = self.__get_cached_dir_data return self.dir_data def get_entries(self): @@ -273,9 +281,14 @@ class SVNEntriesCMD(SVNEntries): self.get_entries = self.__get_cached_entries return self.entries + + def get_url(self): "Get repository URL" - return self.urlre.search(self.get_dir_data()[0]).group(1) + url_element = self.get_dir_data().getElementsByTagName("url")[0] + url_text = [t.nodeValue for t in url_element.childNodes + if t.nodeType == t.TEXT_NODE] + return "".join(url_text) def __get_cached_revision(self): return self.revision -- cgit v1.2.1 From bb21c63c493908916efafc907117522b30999b45 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 11:20:20 -0500 Subject: get_svn_method now direectly called svn_utils.parse_revision --- setuptools/svn_utils.py | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 57cf6849..e45f05bc 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -34,6 +34,25 @@ def _run_command(args, stdout=_PIPE, stderr=_PIPE): return proc.returncode, data +#svnversion return values (previous implementations return max revision) +# 4123:4168 mixed revision working copy +# 4168M modified working copy +# 4123S switched working copy +# 4123:4168MS mixed revision, modified, switched working copy +_SVN_VER_RE = re.compile(r'(?:(\d+):)?(\d+)([a-z]*)\s*$', re.I) + +def parse_revision(path): + _, data = _run_command(['svnversion', path]) + parsed = _SVN_VER_RE.match(data) + if parsed: + try: + #No max needed this command summarizes working copy since 1.0 + return int(parsed.group(2)) + except ValueError: + #This should only happen if the revision is WAY too big. + pass + return 0 + #TODO add the text entry back, and make its use dependent on the # non existence of svn? @@ -242,13 +261,6 @@ class SVNEntriesCMD(SVNEntries): entryre = re.compile(r'', re.DOTALL or re.I) namere = re.compile(r'(.*?)', re.I) - #svnversion return values (previous implementations return max revision) - # 4123:4168 mixed revision working copy - # 4168M modified working copy - # 4123S switched working copy - # 4123:4168MS mixed revision, modified, switched working copy - svnverre = re.compile(r'(?:(\d+):)?(\d+)([a-z]*)\s*$', re.I) - def __get_cached_dir_data(self): return self.dir_data @@ -294,18 +306,9 @@ class SVNEntriesCMD(SVNEntries): return self.revision def parse_revision(self): - _, data = _run_command(['svnversion', self.path]) - parsed = self.svnverre.match(data) - if parsed: - try: - #No max needed this command summarizes working copy since 1.0 - self.revision = int(parsed.group(2)) - self.parse_revision = self.__get_cached_revision - return self.revision - except ValueError: - #This should only happen if the revision is WAY too big. - pass - return 0 + self.revision = parse_revision(self.path) + self.parse_revision = self.__get_cached_revision + return self.revision def get_undeleted_records(self): #NOTE: Need to parse entities? -- cgit v1.2.1 From 9fad943dbfead926b3908edcef5c0dc4f4565f5e Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 11:27:45 -0500 Subject: get_svn_method now direectly called svn_utils.parse_revision --- setuptools/svn_utils.py | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index e45f05bc..93912b1a 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -185,9 +185,6 @@ class SVNEntriesText(SVNEntries): def is_valid(self): return bool(self.get_sections()) - def get_url(self): - return self.get_sections()[0][4] - def parse_revision_numbers(self): revision_line_number = 9 rev_numbers = [ @@ -213,11 +210,6 @@ class SVNEntriesXML(SVNEntries): def is_valid(self): return True - def get_url(self): - "Get repository URL" - urlre = re.compile(r'url="([^"]+)"') - return urlre.search(self.data).group(1) - def parse_revision_numbers(self): revre = re.compile(r'committed-rev="(\d+)"') return [ @@ -295,13 +287,6 @@ class SVNEntriesCMD(SVNEntries): - def get_url(self): - "Get repository URL" - url_element = self.get_dir_data().getElementsByTagName("url")[0] - url_text = [t.nodeValue for t in url_element.childNodes - if t.nodeType == t.TEXT_NODE] - return "".join(url_text) - def __get_cached_revision(self): return self.revision -- cgit v1.2.1 From a8b64bd22ce7e0bc684e34cf6a018892133c6d60 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 12:24:21 -0500 Subject: cannot use list since that requires repo access, initial recurse parsing --- setuptools/svn_utils.py | 62 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 20 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 93912b1a..5ada2e31 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -53,6 +53,36 @@ def parse_revision(path): pass return 0 +#svn list returns relative o +def parse_manifest(path): + #NOTE: Need to parse entities? + _, data = _run_command(['svn', 'info', '-R', '--xml', path]) + + doc = xml.dom.pulldom.parseString(data) + entries = list() + for event, node in doc: + if event=='START_ELEMENT' and node.nodeName=='entry': + doc.expandNode(node) + entries.append(node) + + if entries: + return [ + _get_entry_name(element) + for element in entries[1:] + if _get_entry_schedule(element).lower() != 'deleted' + ] + else: + return [] + +def _get_entry_name(entry): + return entry.getAttribute('path') + +def _get_entry_schedule(entry): + schedule = entry.getElementsByTagName('schedule')[0] + return "".join([t.nodeValue for t in schedule.childNodes + if t.nodeType == t.TEXT_NODE]) + + #TODO add the text entry back, and make its use dependent on the # non existence of svn? @@ -253,35 +283,28 @@ class SVNEntriesCMD(SVNEntries): entryre = re.compile(r'', re.DOTALL or re.I) namere = re.compile(r'(.*?)', re.I) - def __get_cached_dir_data(self): - return self.dir_data - def __get_cached_entries(self): return self.entries def is_valid(self): - return self.get_dir_data() is not None + return bool(self.get_entries()) def get_dir_data(self): #This returns the info entry for the directory ONLY - _, data = _run_command(['svn', 'info', '--xml', self.path]) + return self.get_entries()[0] + + def get_entries(self): + #regarding the shell argument, see: http://bugs.python.org/issue8557 + _, data = _run_command(['svn', 'info', + '--depth', 'immediates', '--xml', self.path]) doc = xml.dom.pulldom.parseString(data) - self.dir_data = None + self.entries = list() for event, node in doc: if event=='START_ELEMENT' and node.nodeName=='entry': doc.expandNode(node) - self.dir_data = node - break + self.entries.append(node) - if self.dir_data: - self.get_dir_data = self.__get_cached_dir_data - return self.dir_data - - def get_entries(self): - #regarding the shell argument, see: http://bugs.python.org/issue8557 - _, data = _run_command(['svn', 'list', '--xml', self.path]) - self.entries = self.entryre.findall(data) self.get_entries = self.__get_cached_entries return self.entries @@ -301,10 +324,9 @@ class SVNEntriesCMD(SVNEntries): return list() else: return [ - m.group(1) - for entry in self.get_entries() - for m in self.namere.finditer(entry) - if m.group(1) + _get_entry_name(element) + for element in self.get_entries()[1:] + if _get_entry_schedule(element).lower() != 'deleted' ] def _get_externals_data(self, filename): -- cgit v1.2.1 From f982fb0b277252574a9909db14c95ef4153d38bb Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 12:44:57 -0500 Subject: cannot use list since that requires repo access, initial recurse parsing --- setuptools/svn_utils.py | 88 ++++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 37 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 5ada2e31..f373379c 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -9,11 +9,10 @@ import xml.dom.pulldom #requires python >= 2.4 from subprocess import Popen as _Popen, PIPE as _PIPE - -#It would seem that svn info --xml and svn list --xml were fully -#supported by 1.3.x the special casing of the entry files seem to start at -#1.4.x, so if we check for xml in entries and then fall back to the command -#line, this should catch everything. +#NOTE: Use of the command line options +# require SVN 1.3 or newer (December 2005) +# and SVN 1.3 hsan't been supported by the +# developers since mid 2008. #subprocess is called several times with shell=(sys.platform=='win32') #see the follow for more information: @@ -42,7 +41,12 @@ def _run_command(args, stdout=_PIPE, stderr=_PIPE): _SVN_VER_RE = re.compile(r'(?:(\d+):)?(\d+)([a-z]*)\s*$', re.I) def parse_revision(path): - _, data = _run_command(['svnversion', path]) + code, data = _run_command(['svnversion', path]) + + if code: + log.warn("svnversion failed") + return [] + parsed = _SVN_VER_RE.match(data) if parsed: try: @@ -53,10 +57,14 @@ def parse_revision(path): pass return 0 -#svn list returns relative o -def parse_manifest(path): - #NOTE: Need to parse entities? - _, data = _run_command(['svn', 'info', '-R', '--xml', path]) + +def parse_dir_entries(path): + code, data = _run_command(['svn', 'info', + '--depth', 'immediates', '--xml', path]) + + if code: + log.warn("svn info failed") + return [] doc = xml.dom.pulldom.parseString(data) entries = list() @@ -82,6 +90,38 @@ def _get_entry_schedule(entry): return "".join([t.nodeValue for t in schedule.childNodes if t.nodeType == t.TEXT_NODE]) +#--xml wasn't supported until 1.5.x +#-R without --xml parses a bit funny +def parse_externals(path): + try: + _, lines = _run_command(['svn', + 'propget', 'svn:externals', path]) + + if code: + log.warn("svn propget failed") + return [] + + lines = [line for line in lines.splitlines() if line] + except ValueError: + lines = [] + + externals = [] + for line in lines: + line = line.split() + if not line: + continue + + #TODO: urlparse? + if "://" in line[-1] or ":\\\\" in line[-1]: + externals.append(line[0]) + else: + externals.append(line[-1]) + + return externals + + + + #TODO add the text entry back, and make its use dependent on the # non existence of svn? @@ -155,33 +195,7 @@ class SVNEntries(object): return '' def get_external_dirs(self, filename): - filename = os.path.join(self.path, '.svn', filename) - - data = self._get_externals_data(filename) - - if not data: - return - - # http://svnbook.red-bean.com/en/1.6/svn.advanced.externals.html - #there appears to be three possible formats for externals since 1.5 - #but looks like we only need the local relative path names so it's just - #2 either the first column or the last (of 2 or 3) Looks like - #mix and matching is allowed. - externals = list() - for line in data: - line = line.split() - if not line: - continue - - #TODO: urlparse? - if "://" in line[-1] or ":\\\\" in line[-1]: - externals.append(line[0]) - else: - externals.append(line[-1]) - - self.external_dirs = externals - self.get_external_dirs = self.__get_cached_external_dirs - return self.external_dirs + return parse_externals(self.path) class SVNEntriesText(SVNEntries): known_svn_versions = { -- cgit v1.2.1 From f4e33db5e479b87ac4217ec9b033a242aabe2ed7 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 12:49:15 -0500 Subject: got some global version done, SVN 1.3.x or later now required --- setuptools/svn_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index f373379c..5783489c 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -94,7 +94,7 @@ def _get_entry_schedule(entry): #-R without --xml parses a bit funny def parse_externals(path): try: - _, lines = _run_command(['svn', + code, lines = _run_command(['svn', 'propget', 'svn:externals', path]) if code: -- cgit v1.2.1 From eae24076512070377eac957c0b760bc0fc2e78cf Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 12:53:02 -0500 Subject: removed the objects --- setuptools/svn_utils.py | 242 ++---------------------------------------------- 1 file changed, 6 insertions(+), 236 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 5783489c..657393e2 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -119,240 +119,10 @@ def parse_externals(path): return externals +def get_svn_tool_version(): + _, data = _run_command(['svn', '--version', '--quiet']) + if data: + return data.strip() + else: + return '' - - - -#TODO add the text entry back, and make its use dependent on the -# non existence of svn? - -class SVNEntries(object): - svn_tool_version = '' - - def __init__(self, path, data): - self.path = path - self.data = data - if not self.svn_tool_version: - self.svn_tool_version = self.get_svn_tool_version() - - @staticmethod - def get_svn_tool_version(): - _, data = _run_command(['svn', '--version', '--quiet']) - if data: - return data.strip() - else: - return '' - - @classmethod - def load_dir(class_, base): - filename = os.path.join(base, '.svn', 'entries') - f = open(filename) - result = SVNEntries.read(f, base) - f.close() - return result - - @classmethod - def read(class_, file, path=None): - data = file.read() - - if data.startswith('revision_line_number - and section[revision_line_number] - ] - return rev_numbers - - def get_undeleted_records(self): - #Note for self this skip and only returns the children - undeleted = lambda s: s and s[0] and (len(s) < 6 or s[5] != 'delete') - result = [ - section[0] - for section in self.get_sections() - if undeleted(section) - ] - return result - - -class SVNEntriesXML(SVNEntries): - def is_valid(self): - return True - - def parse_revision_numbers(self): - revre = re.compile(r'committed-rev="(\d+)"') - return [ - int(m.group(1)) - for m in revre.finditer(self.data) - if m.group(1) - ] - - def get_undeleted_records(self): - entries_pattern = re.compile(r'name="([^"]+)"(?![^>]+deleted="true")', - re.I) - results = [ - unescape(match.group(1)) - for match in entries_pattern.finditer(self.data) - if match.group(1) - ] - return results - -import xml.dom.pulldom - -class SVNEntriesCMD(SVNEntries): - # - # - # - # - # ... - # ... - # - # - # ... - # - # - # ... - # - # - # ... - # - entrypathre = re.compile(r']*path="(\.+)">', re.I) - entryre = re.compile(r'', re.DOTALL or re.I) - namere = re.compile(r'(.*?)', re.I) - - def __get_cached_entries(self): - return self.entries - - def is_valid(self): - return bool(self.get_entries()) - - def get_dir_data(self): - #This returns the info entry for the directory ONLY - return self.get_entries()[0] - - def get_entries(self): - #regarding the shell argument, see: http://bugs.python.org/issue8557 - _, data = _run_command(['svn', 'info', - '--depth', 'immediates', '--xml', self.path]) - - doc = xml.dom.pulldom.parseString(data) - self.entries = list() - for event, node in doc: - if event=='START_ELEMENT' and node.nodeName=='entry': - doc.expandNode(node) - self.entries.append(node) - - self.get_entries = self.__get_cached_entries - return self.entries - - - - def __get_cached_revision(self): - return self.revision - - def parse_revision(self): - self.revision = parse_revision(self.path) - self.parse_revision = self.__get_cached_revision - return self.revision - - def get_undeleted_records(self): - #NOTE: Need to parse entities? - if not self.is_valid(): - return list() - else: - return [ - _get_entry_name(element) - for element in self.get_entries()[1:] - if _get_entry_schedule(element).lower() != 'deleted' - ] - - def _get_externals_data(self, filename): - - #othewise will be called twice. - if os.path.basename(filename).lower() != 'dir-props': - return '' - - try: - _, lines = _run_command(['svn', - 'propget', 'svn:externals', self.path]) - lines = [line for line in lines.splitlines() if line] - return lines - except ValueError: - return '' -- cgit v1.2.1 From fd74f8c9f7c50105c34c465933dce3f8a985625a Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 12:59:35 -0500 Subject: use urlparse for url detection --- setuptools/svn_utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 657393e2..0f54ba54 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -3,8 +3,8 @@ import re import sys import codecs from distutils import log -from xml.sax.saxutils import unescape import xml.dom.pulldom +import urlparse #requires python >= 2.4 from subprocess import Popen as _Popen, PIPE as _PIPE @@ -82,9 +82,11 @@ def parse_dir_entries(path): else: return [] + def _get_entry_name(entry): return entry.getAttribute('path') + def _get_entry_schedule(entry): schedule = entry.getElementsByTagName('schedule')[0] return "".join([t.nodeValue for t in schedule.childNodes @@ -111,14 +113,14 @@ def parse_externals(path): if not line: continue - #TODO: urlparse? - if "://" in line[-1] or ":\\\\" in line[-1]: + if urlparse.urlsplit(line[-1])[0]: externals.append(line[0]) else: externals.append(line[-1]) return externals + def get_svn_tool_version(): _, data = _run_command(['svn', '--version', '--quiet']) if data: -- cgit v1.2.1 From 9f09bb7acc04b06fd96ada0ce47c6422260160da Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Thu, 4 Jul 2013 13:10:35 -0500 Subject: consolidated externals and enteries because enteries need to file to interate over and both get called by the same callback. pep8 on svn_utils --- setuptools/svn_utils.py | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 0f54ba54..350f17c8 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -14,6 +14,15 @@ from subprocess import Popen as _Popen, PIPE as _PIPE # and SVN 1.3 hsan't been supported by the # developers since mid 2008. + +#svnversion return values (previous implementations return max revision) +# 4123:4168 mixed revision working copy +# 4168M modified working copy +# 4123S switched working copy +# 4123:4168MS mixed revision, modified, switched working copy +_SVN_VER_RE = re.compile(r'(?:(\d+):)?(\d+)([a-z]*)\s*$', re.I) + + #subprocess is called several times with shell=(sys.platform=='win32') #see the follow for more information: # http://bugs.python.org/issue8557 @@ -22,7 +31,7 @@ from subprocess import Popen as _Popen, PIPE as _PIPE def _run_command(args, stdout=_PIPE, stderr=_PIPE): #regarding the shell argument, see: http://bugs.python.org/issue8557 proc = _Popen(args, stdout=stdout, stderr=stderr, - shell=(sys.platform=='win32')) + shell=(sys.platform == 'win32')) data = proc.communicate()[0] #TODO: this is probably NOT always utf-8 @@ -33,12 +42,17 @@ def _run_command(args, stdout=_PIPE, stderr=_PIPE): return proc.returncode, data -#svnversion return values (previous implementations return max revision) -# 4123:4168 mixed revision working copy -# 4168M modified working copy -# 4123S switched working copy -# 4123:4168MS mixed revision, modified, switched working copy -_SVN_VER_RE = re.compile(r'(?:(\d+):)?(\d+)([a-z]*)\s*$', re.I) + +def _get_entry_name(entry): + return entry.getAttribute('path') + + +def _get_entry_schedule(entry): + schedule = entry.getElementsByTagName('schedule')[0] + return "".join([t.nodeValue + for t in schedule.childNodes + if t.nodeType == t.TEXT_NODE]) + def parse_revision(path): code, data = _run_command(['svnversion', path]) @@ -60,7 +74,7 @@ def parse_revision(path): def parse_dir_entries(path): code, data = _run_command(['svn', 'info', - '--depth', 'immediates', '--xml', path]) + '--depth', 'immediates', '--xml', path]) if code: log.warn("svn info failed") @@ -69,7 +83,7 @@ def parse_dir_entries(path): doc = xml.dom.pulldom.parseString(data) entries = list() for event, node in doc: - if event=='START_ELEMENT' and node.nodeName=='entry': + if event == 'START_ELEMENT' and node.nodeName == 'entry': doc.expandNode(node) entries.append(node) @@ -78,26 +92,17 @@ def parse_dir_entries(path): _get_entry_name(element) for element in entries[1:] if _get_entry_schedule(element).lower() != 'deleted' - ] + ] else: return [] -def _get_entry_name(entry): - return entry.getAttribute('path') - - -def _get_entry_schedule(entry): - schedule = entry.getElementsByTagName('schedule')[0] - return "".join([t.nodeValue for t in schedule.childNodes - if t.nodeType == t.TEXT_NODE]) - #--xml wasn't supported until 1.5.x #-R without --xml parses a bit funny def parse_externals(path): try: code, lines = _run_command(['svn', - 'propget', 'svn:externals', path]) + 'propget', 'svn:externals', path]) if code: log.warn("svn propget failed") @@ -127,4 +132,3 @@ def get_svn_tool_version(): return data.strip() else: return '' - -- cgit v1.2.1 From 201c2b84dde9b402a57adc7453e30bb526d32b37 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Fri, 5 Jul 2013 08:23:44 -0500 Subject: urlparse --> urllib.parse in py3 --- setuptools/svn_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 350f17c8..10c5861d 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -4,7 +4,11 @@ import sys import codecs from distutils import log import xml.dom.pulldom -import urlparse + +try: + import urlparse +except ImportError: + import urllib.parse as urlparse #requires python >= 2.4 from subprocess import Popen as _Popen, PIPE as _PIPE @@ -40,6 +44,7 @@ def _run_command(args, stdout=_PIPE, stderr=_PIPE): except NameError: data = str(data, encoding='utf-8') + #communciate calls wait() return proc.returncode, data -- cgit v1.2.1 From c2fbfb77c84c1d668313c0cce66cbdd0765bbb62 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Fri, 5 Jul 2013 10:37:42 -0500 Subject: fixed some issues with OSError have to emulate zipfile extract on py2.5 and earlier for tests --- setuptools/svn_utils.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 10c5861d..932ee75c 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -34,10 +34,14 @@ _SVN_VER_RE = re.compile(r'(?:(\d+):)?(\d+)([a-z]*)\s*$', re.I) # python-subprocess-popen-environment-path def _run_command(args, stdout=_PIPE, stderr=_PIPE): #regarding the shell argument, see: http://bugs.python.org/issue8557 - proc = _Popen(args, stdout=stdout, stderr=stderr, - shell=(sys.platform == 'win32')) + try: + proc = _Popen(args, stdout=stdout, stderr=stderr, + shell=(sys.platform == 'win32')) + + data = proc.communicate()[0] + except OSError: + return 1, '' - data = proc.communicate()[0] #TODO: this is probably NOT always utf-8 try: data = unicode(data, encoding='utf-8') @@ -60,11 +64,13 @@ def _get_entry_schedule(entry): def parse_revision(path): - code, data = _run_command(['svnversion', path]) + code, data = _run_command(['svnversion', '-c', path]) if code: log.warn("svnversion failed") - return [] + return 0 + else: + log.warn('Version: %s' % data.strip()) parsed = _SVN_VER_RE.match(data) if parsed: -- cgit v1.2.1 From f4b8903d723e005bcb89da2aa770572a29e9b915 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Fri, 5 Jul 2013 11:30:40 -0500 Subject: added some cmdline testing for svn_util and allowed negative numbers in the min number from svnversion. --- setuptools/svn_utils.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 932ee75c..19352821 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -24,7 +24,7 @@ from subprocess import Popen as _Popen, PIPE as _PIPE # 4168M modified working copy # 4123S switched working copy # 4123:4168MS mixed revision, modified, switched working copy -_SVN_VER_RE = re.compile(r'(?:(\d+):)?(\d+)([a-z]*)\s*$', re.I) +_SVN_VER_RE = re.compile(r'(?:([\-0-9]+):)?(\d+)([a-z]*)\s*$', re.I) #subprocess is called several times with shell=(sys.platform=='win32') @@ -91,6 +91,8 @@ def parse_dir_entries(path): log.warn("svn info failed") return [] + data = codecs.encode(data, 'UTF-8') + doc = xml.dom.pulldom.parseString(data) entries = list() for event, node in doc: @@ -143,3 +145,14 @@ def get_svn_tool_version(): return data.strip() else: return '' + +if __name__ == '__main__': + def entries_externals_finder(dirname): + for record in parse_dir_entries(dirname): + yield os.path.join(dirname, record) + + for name in parse_externals(dirname): + yield os.path.join(dirname, name) + + for name in entries_externals_finder(sys.argv[1]): + print(name) \ No newline at end of file -- cgit v1.2.1 From a1c51383a898d59abaa72aff3d0fcec82235019d Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sat, 6 Jul 2013 06:24:30 -0500 Subject: some notes on things that needs to be fixed after all --- setuptools/svn_utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index 19352821..a4c53f15 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -82,7 +82,7 @@ def parse_revision(path): pass return 0 - +#TODO: Need to do this with the -R because only root has .svn in 1.7.x def parse_dir_entries(path): code, data = _run_command(['svn', 'info', '--depth', 'immediates', '--xml', path]) @@ -110,7 +110,16 @@ def parse_dir_entries(path): return [] -#--xml wasn't supported until 1.5.x +#--xml wasn't supported until 1.5.x need to do -R +#TODO: -R looks like directories are seperated by blank lines +# with dir - prepened to first directory +# what about directories with spaces? +# put quotes around them +# what about the URL's? +# same +# convert to UTF-8 and use csv +# delimiter = space +# #-R without --xml parses a bit funny def parse_externals(path): try: -- cgit v1.2.1 From 574071348e7b04f83af905a46fd051f614c74f94 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sat, 20 Jul 2013 17:45:04 -0500 Subject: Additional Tests, Various fixes, and encoding dealings --- setuptools/svn_utils.py | 376 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 269 insertions(+), 107 deletions(-) (limited to 'setuptools/svn_utils.py') diff --git a/setuptools/svn_utils.py b/setuptools/svn_utils.py index a4c53f15..09aa5e25 100644 --- a/setuptools/svn_utils.py +++ b/setuptools/svn_utils.py @@ -1,31 +1,22 @@ import os import re import sys -import codecs from distutils import log import xml.dom.pulldom +import shlex +import locale +import unicodedata +from setuptools.compat import unicode, bytes try: import urlparse except ImportError: import urllib.parse as urlparse -#requires python >= 2.4 from subprocess import Popen as _Popen, PIPE as _PIPE -#NOTE: Use of the command line options -# require SVN 1.3 or newer (December 2005) -# and SVN 1.3 hsan't been supported by the -# developers since mid 2008. - - -#svnversion return values (previous implementations return max revision) -# 4123:4168 mixed revision working copy -# 4168M modified working copy -# 4123S switched working copy -# 4123:4168MS mixed revision, modified, switched working copy -_SVN_VER_RE = re.compile(r'(?:([\-0-9]+):)?(\d+)([a-z]*)\s*$', re.I) - +#NOTE: Use of the command line options require SVN 1.3 or newer (December 2005) +# and SVN 1.3 hasn't been supported by the developers since mid 2008. #subprocess is called several times with shell=(sys.platform=='win32') #see the follow for more information: @@ -33,27 +24,20 @@ _SVN_VER_RE = re.compile(r'(?:([\-0-9]+):)?(\d+)([a-z]*)\s*$', re.I) # http://stackoverflow.com/questions/5658622/ # python-subprocess-popen-environment-path def _run_command(args, stdout=_PIPE, stderr=_PIPE): - #regarding the shell argument, see: http://bugs.python.org/issue8557 - try: - proc = _Popen(args, stdout=stdout, stderr=stderr, - shell=(sys.platform == 'win32')) - - data = proc.communicate()[0] - except OSError: - return 1, '' - - #TODO: this is probably NOT always utf-8 - try: - data = unicode(data, encoding='utf-8') - except NameError: - data = str(data, encoding='utf-8') + #regarding the shell argument, see: http://bugs.python.org/issue8557 + try: + args = [fsdecode(x) for x in args] + proc = _Popen(args, stdout=stdout, stderr=stderr, + shell=(sys.platform == 'win32')) - #communciate calls wait() - return proc.returncode, data + data = proc.communicate()[0] + except OSError: + return 1, '' + data = consoledecode(data) -def _get_entry_name(entry): - return entry.getAttribute('path') + #communciate calls wait() + return proc.returncode, data def _get_entry_schedule(entry): @@ -63,105 +47,283 @@ def _get_entry_schedule(entry): if t.nodeType == t.TEXT_NODE]) -def parse_revision(path): - code, data = _run_command(['svnversion', '-c', path]) +def _get_target_property(target): + property_text = target.getElementsByTagName('property')[0] + return "".join([t.nodeValue + for t in property_text.childNodes + if t.nodeType == t.TEXT_NODE]) - if code: - log.warn("svnversion failed") - return 0 + +def _get_xml_data(decoded_str): + if sys.version_info < (3, 0): + #old versions want an encoded string + data = decoded_str.encode('utf-8') else: - log.warn('Version: %s' % data.strip()) + data = decoded_str + return data - parsed = _SVN_VER_RE.match(data) - if parsed: - try: - #No max needed this command summarizes working copy since 1.0 - return int(parsed.group(2)) - except ValueError: - #This should only happen if the revision is WAY too big. - pass - return 0 - -#TODO: Need to do this with the -R because only root has .svn in 1.7.x -def parse_dir_entries(path): - code, data = _run_command(['svn', 'info', - '--depth', 'immediates', '--xml', path]) - - if code: - log.warn("svn info failed") - return [] - data = codecs.encode(data, 'UTF-8') +def joinpath(prefix, suffix): + if not prefix or prefix == '.': + return suffix + return os.path.join(prefix, suffix) + + +def fsencode(path): + "Path must be unicode or in file system encoding already" + encoding = sys.getfilesystemencoding() + + if isinstance(path, unicode): + path = path.encode() + elif not isinstance(path, bytes): + raise TypeError('%s is not a string or byte type' + % type(path).__name__) + + #getfilessystemencoding doesn't have the mac-roman issue + if encoding == 'utf-8' and sys.platform == 'darwin': + path = path.decode('utf-8') + path = unicodedata.normalize('NFD', path) + path = path.encode('utf-8') - doc = xml.dom.pulldom.parseString(data) + return path + +def fsdecode(path): + "Path must be unicode or in file system encoding already" + encoding = sys.getfilesystemencoding() + if isinstance(path, bytes): + path = path.decode(encoding) + elif not isinstance(path, unicode): + raise TypeError('%s is not a byte type' + % type(path).__name__) + + return unicodedata.normalize('NFC', path) + +def consoledecode(text): + encoding = locale.getpreferredencoding() + return text.decode(encoding) + + +def parse_dir_entries(decoded_str): + '''Parse the entries from a recursive info xml''' + doc = xml.dom.pulldom.parseString(_get_xml_data(decoded_str)) entries = list() + for event, node in doc: if event == 'START_ELEMENT' and node.nodeName == 'entry': doc.expandNode(node) - entries.append(node) - - if entries: - return [ - _get_entry_name(element) - for element in entries[1:] - if _get_entry_schedule(element).lower() != 'deleted' - ] - else: - return [] + if not _get_entry_schedule(node).startswith('delete'): + entries.append((node.getAttribute('path'), + node.getAttribute('kind'))) + return entries[1:] # do not want the root directory -#--xml wasn't supported until 1.5.x need to do -R -#TODO: -R looks like directories are seperated by blank lines -# with dir - prepened to first directory -# what about directories with spaces? -# put quotes around them -# what about the URL's? -# same -# convert to UTF-8 and use csv -# delimiter = space -# -#-R without --xml parses a bit funny -def parse_externals(path): - try: - code, lines = _run_command(['svn', - 'propget', 'svn:externals', path]) - if code: - log.warn("svn propget failed") - return [] +def parse_externals_xml(decoded_str, prefix=''): + '''Parse a propget svn:externals xml''' + prefix = os.path.normpath(prefix) + + doc = xml.dom.pulldom.parseString(_get_xml_data(decoded_str)) + externals = list() + + for event, node in doc: + if event == 'START_ELEMENT' and node.nodeName == 'target': + doc.expandNode(node) + path = os.path.normpath(node.getAttribute('path')) + if os.path.normcase(path).startswith(prefix): + path = path[len(prefix)+1:] - lines = [line for line in lines.splitlines() if line] - except ValueError: - lines = [] + data = _get_target_property(node) + for external in parse_external_prop(data): + externals.append(joinpath(path, external)) + return externals # do not want the root directory + + +def parse_external_prop(lines): + """ + Parse the value of a retrieved svn:externals entry. + + possible token setups (with quotng and backscaping in laters versions) + URL[@#] EXT_FOLDERNAME + [-r#] URL EXT_FOLDERNAME + EXT_FOLDERNAME [-r#] URL + """ externals = [] - for line in lines: - line = line.split() + for line in lines.splitlines(): + line = line.lstrip() #there might be a "\ " if not line: continue + if sys.version_info < (3, 0): + #shlex handles NULLs just fine and shlex in 2.7 tries to encode + #as ascii automatiically + line = line.encode('utf-8') + line = shlex.split(line) + if sys.version_info < (3, 0): + line = [x.decode('utf-8') for x in line] + + #EXT_FOLDERNAME is either the first or last depending on where + #the URL falls if urlparse.urlsplit(line[-1])[0]: - externals.append(line[0]) + external = line[0] else: - externals.append(line[-1]) + external = line[-1] + + externals.append(os.path.normpath(external)) return externals -def get_svn_tool_version(): - _, data = _run_command(['svn', '--version', '--quiet']) - if data: - return data.strip() - else: - return '' +class SvnInfo(object): + ''' + Generic svn_info object. No has little knowledge of how to extract + information. Use cls.load to instatiate according svn version. -if __name__ == '__main__': - def entries_externals_finder(dirname): - for record in parse_dir_entries(dirname): - yield os.path.join(dirname, record) + Paths are not filesystem encoded. + ''' - for name in parse_externals(dirname): - yield os.path.join(dirname, name) + @staticmethod + def get_svn_version(): + code, data = _run_command(['svn', '--version', '--quiet']) + if code == 0 and data: + return unicode(data).strip() + else: + return unicode('') + + #svnversion return values (previous implementations return max revision) + # 4123:4168 mixed revision working copy + # 4168M modified working copy + # 4123S switched working copy + # 4123:4168MS mixed revision, modified, switched working copy + revision_re = re.compile(r'(?:([\-0-9]+):)?(\d+)([a-z]*)\s*$', re.I) + + @classmethod + def load(cls, dirname=''): + code, data = _run_command(['svn', 'info', os.path.normpath(dirname)]) + svn_version = tuple(cls.get_svn_version().split('.')) + base_svn_version = tuple(int(x) for x in svn_version[:2]) + if code and base_svn_version: + #Not an SVN repository or compatible one + return SvnInfo(dirname) + elif base_svn_version < (1, 3): + log.warn('Insufficent version of SVN found') + return SvnInfo(dirname) + elif base_svn_version < (1, 5): + return Svn13Info(dirname) + else: + return Svn15Info(dirname) + + def __init__(self, path=''): + self.path = path + self._entries = None + self._externals = None + + def get_revision(self): + 'Retrieve the directory revision informatino using svnversion' + code, data = _run_command(['svnversion', '-c', self.path]) + if code: + log.warn("svnversion failed") + return 0 - for name in entries_externals_finder(sys.argv[1]): + parsed = self.revision_re.match(data) + if parsed: + return int(parsed.group(2)) + else: + return 0 + + @property + def entries(self): + if self._entries is None: + self._entries = self.get_entries() + return self._entries + + @property + def externals(self): + if self._externals is None: + self._externals = self.get_externals() + return self._externals + + def iter_externals(self): + ''' + Iterate over the svn:external references in the repository path. + ''' + for item in self.externals: + yield item + + def iter_files(self): + ''' + Iterate over the non-deleted file entries in the repository path + ''' + for item, kind in self.entries: + if kind.lower()=='file': + yield item + + def iter_dirs(self, include_root=True): + ''' + Iterate over the non-deleted file entries in the repository path + ''' + if include_root: + yield self.path + for item, kind in self.entries: + if kind.lower()=='dir': + yield item + + def get_entries(self): + return [] + + def get_externals(self): + return [] + +class Svn13Info(SvnInfo): + def get_entries(self): + code, data = _run_command(['svn', 'info', '-R', '--xml', self.path]) + + if code: + log.debug("svn info failed") + return [] + + return parse_dir_entries(data) + + def get_externals(self): + #Previous to 1.5 --xml was not supported for svn propget and the -R + #output format breaks the shlex compatible semantics. + cmd = ['svn', 'propget', 'svn:externals'] + result = [] + for folder in self.iter_dirs(): + code, lines = _run_command(cmd + [folder]) + if code != 0: + log.warn("svn propget failed") + return [] + for external in parse_external_prop(lines): + if folder: + external = os.path.join(folder, external) + result.append(os.path.normpath(external)) + + return result + + +class Svn15Info(Svn13Info): + def get_externals(self): + cmd = ['svn', 'propget', 'svn:externals', self.path, '-R', '--xml'] + code, lines = _run_command(cmd) + if code: + log.debug("svn propget failed") + return [] + return parse_externals_xml(lines, prefix=os.path.abspath(self.path)) + + +def svn_finder(dirname=''): + #combined externals due to common interface + #combined externals and entries due to lack of dir_props in 1.7 + info = SvnInfo.load(dirname) + for path in info.iter_files(): + yield fsencode(path) + + for path in info.iter_externals(): + sub_info = SvnInfo.load(path) + for sub_path in sub_info.iter_files(): + yield fsencode(sub_path) + +if __name__ == '__main__': + for name in svn_finder(sys.argv[1]): print(name) \ No newline at end of file -- cgit v1.2.1