From 9424d8177020db5fd15e48e95226e9e53ef7ce34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 18 May 2012 11:47:38 +0200 Subject: Added comments to separate each option --- sqlparse/formatter.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'sqlparse') diff --git a/sqlparse/formatter.py b/sqlparse/formatter.py index 5be6652..f8563a6 100644 --- a/sqlparse/formatter.py +++ b/sqlparse/formatter.py @@ -11,34 +11,43 @@ from sqlparse import filters def validate_options(options): """Validates options.""" + + # keyword_case kwcase = options.get('keyword_case', None) if kwcase not in [None, 'upper', 'lower', 'capitalize']: raise SQLParseError('Invalid value for keyword_case: %r' % kwcase) + # identifier_case idcase = options.get('identifier_case', None) if idcase not in [None, 'upper', 'lower', 'capitalize']: raise SQLParseError('Invalid value for identifier_case: %r' % idcase) + # output_format ofrmt = options.get('output_format', None) if ofrmt not in [None, 'sql', 'python', 'php']: raise SQLParseError('Unknown output format: %r' % ofrmt) + # strip_comments strip_comments = options.get('strip_comments', False) if strip_comments not in [True, False]: raise SQLParseError('Invalid value for strip_comments: %r' % strip_comments) + # strip_whitespace strip_ws = options.get('strip_whitespace', False) if strip_ws not in [True, False]: raise SQLParseError('Invalid value for strip_whitespace: %r' % strip_ws) + # reindent reindent = options.get('reindent', False) if reindent not in [True, False]: raise SQLParseError('Invalid value for reindent: %r' % reindent) elif reindent: options['strip_whitespace'] = True + + # indent_tabs indent_tabs = options.get('indent_tabs', False) if indent_tabs not in [True, False]: raise SQLParseError('Invalid value for indent_tabs: %r' % indent_tabs) @@ -46,15 +55,20 @@ def validate_options(options): options['indent_char'] = '\t' else: options['indent_char'] = ' ' + + # indent_width indent_width = options.get('indent_width', 2) try: indent_width = int(indent_width) except (TypeError, ValueError): raise SQLParseError('indent_width requires an integer') + if indent_width < 1: raise SQLParseError('indent_width requires an positive integer') + options['indent_width'] = indent_width + # right_margin right_margin = options.get('right_margin', None) if right_margin is not None: try: @@ -65,6 +79,7 @@ def validate_options(options): raise SQLParseError('right_margin requires an integer > 10') options['right_margin'] = right_margin + # return the processed options return options -- cgit v1.2.1 From de495d42d3be77e7ca70dd7dab4488a3a719f4c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 18 May 2012 11:59:07 +0200 Subject: Removed useless option (was checked on validate_options) --- sqlparse/formatter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/formatter.py b/sqlparse/formatter.py index f8563a6..f182850 100644 --- a/sqlparse/formatter.py +++ b/sqlparse/formatter.py @@ -104,8 +104,7 @@ def build_filter_stack(stack, options): stack.enable_grouping() stack.stmtprocess.append(filters.StripCommentsFilter()) - if (options.get('strip_whitespace', False) - or options.get('reindent', False)): + if options.get('strip_whitespace', False): stack.enable_grouping() stack.stmtprocess.append(filters.StripWhitespaceFilter()) -- cgit v1.2.1 From 02999ed9af5dac5e1131b1e61332bcab6a686c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 18 May 2012 12:20:35 +0200 Subject: Clean-up of process --- sqlparse/filters.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 2c1f0eb..f69cea8 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -405,25 +405,33 @@ class ReindentFilter: self._split_statements(tlist) if kwds: self._split_kwds(tlist) - [self._process(sgroup) for sgroup in tlist.get_sublists()] + + for sgroup in tlist.get_sublists(): + self._process(sgroup) def process(self, stack, stmt): warn("Deprecated, use callable objects. This will be removed at 0.2.0", DeprecationWarning) + # If we are processing a statement, set it as the current one if isinstance(stmt, sql.Statement): self._curr_stmt = stmt + + # Process the statement self._process(stmt) + + # If we are processing a statement, check if we should add a new line if isinstance(stmt, sql.Statement): - if self._last_stmt is not None: + if self._last_stmt != None: if unicode(self._last_stmt).endswith('\n'): nl = '\n' else: nl = '\n\n' - stmt.tokens.insert(0, - sql.Token(T.Whitespace, nl)) - if self._last_stmt != stmt: - self._last_stmt = stmt + + stmt.tokens.insert(0, sql.Token(T.Whitespace, nl)) + + # Set the statement as the current one + self._last_stmt = stmt # FIXME: Doesn't work ;) -- cgit v1.2.1 From c0a7444281757056d962158dbb935596b2eb3572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 18 May 2012 14:09:07 +0200 Subject: Cleaned and added comments --- sqlparse/filters.py | 80 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 18 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index f69cea8..a3fbf06 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -260,6 +260,9 @@ class StripWhitespaceFilter: class ReindentFilter: + """ + Filter that return a correctly indented version of the SQL string + """ def __init__(self, width=2, char=' ', line_width=None): self.width = width @@ -281,7 +284,7 @@ class ReindentFilter: def nl(self): # TODO: newline character should be configurable - ws = '\n' + (self.char * ((self.indent * self.width) + self.offset)) + ws = '\n' + self.char * (self.indent * self.width + self.offset) return sql.Token(T.Whitespace, ws) def _split_kwds(self, tlist): @@ -331,37 +334,60 @@ class ReindentFilter: (T.Keyword.DDL, T.Keyword.DML)) def _process(self, tlist): + """ + Proxy to other methods based on `tlist` class + """ func_name = '_process_%s' % tlist.__class__.__name__.lower() func = getattr(self, func_name, self._process_default) func(tlist) def _process_where(self, tlist): + """ + Process WHERE statement + """ + # Look for the next WHERE keyword and add a new line token = tlist.token_next_match(0, T.Keyword, 'WHERE') tlist.insert_before(token, self.nl()) + + # Indent and process the (indented) WHERE statement as usual self.indent += 1 self._process_default(tlist) self.indent -= 1 def _process_parenthesis(self, tlist): + """ + Process parenthesis + """ + # Omit the 'open parenthesis' token + # and check if the next one require say us we should indent first = tlist.token_next(0) - indented = False - if first and first.ttype in (T.Keyword.DML, T.Keyword.DDL): + indented = first and first.ttype in (T.Keyword.DML, T.Keyword.DDL) + + # If we should indent, increase indent and add a new line + if indented: self.indent += 1 tlist.tokens.insert(0, self.nl()) - indented = True - num_offset = self._get_offset(tlist.token_next_match(0, - T.Punctuation, '(')) + + # Get indentation offset + token = tlist.token_next_match(0, T.Punctuation, '(') + num_offset = self._get_offset(token) + + # Increase indentation offset and process the statement as usual self.offset += num_offset self._process_default(tlist, stmts=not indented) + self.offset -= num_offset + + # If we indented, decrease indent to previous state if indented: self.indent -= 1 - self.offset -= num_offset def _process_identifierlist(self, tlist): identifiers = list(tlist.get_identifiers()) + if len(identifiers) > 1 and not tlist.within(sql.Function): first = list(identifiers[0].flatten())[0] num_offset = self._get_offset(first) - len(first.value) + self.offset += num_offset for token in identifiers[1:]: tlist.insert_before(token, self.nl()) @@ -369,38 +395,56 @@ class ReindentFilter: if isinstance(token, sql.Comment): tlist.insert_after(token, self.nl()) self.offset -= num_offset + self._process_default(tlist) def _process_case(self, tlist): - is_first = True - num_offset = None + """ + Process a CASE statement + """ + # Increase the offset the size of the CASE keyword case = tlist.tokens[0] outer_offset = self._get_offset(case) - len(case.value) self.offset += outer_offset - for cond, value in tlist.get_cases(): - if is_first: - tcond = list(cond[0].flatten())[0] - is_first = False - num_offset = self._get_offset(tcond) - len(tcond.value) - self.offset += num_offset - continue + + # Get the case conditions + cases = tlist.get_cases() + + # Get and increase the offset the size of the condition selector + cond, value = cases[0] + tcond = list(cond[0].flatten())[0] + num_offset = self._get_offset(tcond) - len(tcond.value) + self.offset += num_offset + + # Insert a new line before each condition + for cond, value in cases[1:]: if cond is None: token = value[0] else: token = cond[0] + tlist.insert_before(token, self.nl()) + # Line breaks on group level are done. Now let's add an offset of # 5 (=length of "when", "then", "else") and process subgroups. self.offset += 5 self._process_default(tlist) self.offset -= 5 - if num_offset is not None: - self.offset -= num_offset + + # Decrease the offset the size of the condition selector + self.offset -= num_offset + + # Insert a new line before the case END keyword end = tlist.token_next_match(0, T.Keyword, 'END') tlist.insert_before(end, self.nl()) + + # Decrease the offset the size of the CASE keyword self.offset -= outer_offset def _process_default(self, tlist, stmts=True, kwds=True): + """ + Generic processing of `tlist` statements + """ if stmts: self._split_statements(tlist) if kwds: -- cgit v1.2.1 From 980a9b70c32da9863af6ca9600cce9a5142ad60f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 18 May 2012 14:23:50 +0200 Subject: Clean-up and comments --- sqlparse/filters.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index a3fbf06..006bccc 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -278,11 +278,15 @@ class ReindentFilter: idx = all_.index(token) raw = ''.join(unicode(x) for x in all_[:idx + 1]) line = raw.splitlines()[-1] + # Now take current offset into account and return relative offset. - full_offset = len(line) - len(self.char * (self.width * self.indent)) + full_offset = len(line) - len(self.char * self.width * self.indent) return full_offset - self.offset def nl(self): + """ + Return an indented new line token + """ # TODO: newline character should be configurable ws = '\n' + self.char * (self.indent * self.width + self.offset) return sql.Token(T.Whitespace, ws) @@ -301,14 +305,14 @@ class ReindentFilter: t = _next_token(tlist.token_index(t) + 1) return t - idx = 0 - token = _next_token(idx) + token = _next_token(0) while token: prev = tlist.token_prev(tlist.token_index(token), False) offset = 1 if prev and prev.is_whitespace(): tlist.tokens.pop(tlist.token_index(prev)) offset += 1 + if (prev and isinstance(prev, sql.Comment) and (str(prev).endswith('\n') @@ -317,6 +321,7 @@ class ReindentFilter: else: nl = self.nl() tlist.insert_before(token, nl) + token = _next_token(tlist.token_index(nl) + offset) def _split_statements(self, tlist): -- cgit v1.2.1 From a6d6641e5feb0507f08d9167158d4ca033eb940f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 18 May 2012 22:41:01 +0200 Subject: Added comments and documentation to _get_offset() --- sqlparse/filters.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 006bccc..a6a34e9 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -274,6 +274,10 @@ class ReindentFilter: self._last_stmt = None def _get_offset(self, token): + """ + Return the offset where the token should be indented + """ + # Get last processed line (the current one) up to the next token all_ = list(self._curr_stmt.flatten()) idx = all_.index(token) raw = ''.join(unicode(x) for x in all_[:idx + 1]) -- cgit v1.2.1 From e78ff47592f593448b4ca6ef12869dbeba13278b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 18 May 2012 22:53:52 +0200 Subject: Add comments and docstring for _process_identifierlist() --- sqlparse/filters.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index a6a34e9..e306de4 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -391,12 +391,17 @@ class ReindentFilter: self.indent -= 1 def _process_identifierlist(self, tlist): + """ + Process an identifier list + """ + # If there are more than an identifier, put each on a line identifiers = list(tlist.get_identifiers()) - if len(identifiers) > 1 and not tlist.within(sql.Function): + # Get offset size to increase first = list(identifiers[0].flatten())[0] num_offset = self._get_offset(first) - len(first.value) + # Increase offset and insert new lines self.offset += num_offset for token in identifiers[1:]: tlist.insert_before(token, self.nl()) @@ -405,6 +410,7 @@ class ReindentFilter: tlist.insert_after(token, self.nl()) self.offset -= num_offset + # Process the identifier list as usual self._process_default(tlist) def _process_case(self, tlist): -- cgit v1.2.1 From c16219bf97150339590136dcf81fb7976e47bf63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 18 May 2012 23:02:22 +0200 Subject: Comments and documentation for _split_statements() --- sqlparse/filters.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index e306de4..d32b558 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -301,8 +301,7 @@ class ReindentFilter: 'SET', 'BETWEEN') def _next_token(i): - t = tlist.token_next_match(i, T.Keyword, split_words, - regex=True) + t = tlist.token_next_match(i, T.Keyword, split_words, regex=True) if t and t.value.upper() == 'BETWEEN': t = _next_token(tlist.token_index(t) + 1) if t and t.value.upper() == 'AND': @@ -329,16 +328,23 @@ class ReindentFilter: token = _next_token(tlist.token_index(nl) + offset) def _split_statements(self, tlist): - idx = 0 - token = tlist.token_next_by_type(idx, (T.Keyword.DDL, T.Keyword.DML)) + """ + Split tlist on statements + """ + # Search for the first statement + token = tlist.token_next_by_type(0, (T.Keyword.DDL, T.Keyword.DML)) + while token: prev = tlist.token_prev(tlist.token_index(token), False) - if prev and prev.is_whitespace(): - tlist.tokens.pop(tlist.token_index(prev)) - # only break if it's not the first token if prev: + if prev.is_whitespace(): + tlist.tokens.pop(tlist.token_index(prev)) + + # only break if it's not the first token nl = self.nl() tlist.insert_before(token, nl) + + # Go to the next statement token = tlist.token_next_by_type(tlist.token_index(token) + 1, (T.Keyword.DDL, T.Keyword.DML)) -- cgit v1.2.1 From 2e84066066059891270ea27e3397ed3eb39d419f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 00:30:00 +0200 Subject: Commented and documented _split_kwds() --- sqlparse/filters.py | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index d32b558..1ba97f5 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -296,35 +296,55 @@ class ReindentFilter: return sql.Token(T.Whitespace, ws) def _split_kwds(self, tlist): + """ + Split `tlist` by its keywords + """ split_words = ('FROM', 'JOIN$', 'AND', 'OR', 'GROUP', 'ORDER', 'UNION', 'VALUES', 'SET', 'BETWEEN') def _next_token(i): + """ + Get next keyword where to split + """ + # Search for the first keyword token t = tlist.token_next_match(i, T.Keyword, split_words, regex=True) + + # Use the BETWEEN ... AND ... struct as an unsplitable statement if t and t.value.upper() == 'BETWEEN': t = _next_token(tlist.token_index(t) + 1) if t and t.value.upper() == 'AND': t = _next_token(tlist.token_index(t) + 1) + + # Return the token return t + # Get first token token = _next_token(0) while token: - prev = tlist.token_prev(tlist.token_index(token), False) offset = 1 - if prev and prev.is_whitespace(): - tlist.tokens.pop(tlist.token_index(prev)) - offset += 1 - - if (prev - and isinstance(prev, sql.Comment) - and (str(prev).endswith('\n') - or str(prev).endswith('\r'))): - nl = tlist.token_next(token) - else: + nl = None + + # Check if we have any token before + prev = tlist.token_prev(tlist.token_index(token), False) + if prev: + # Previous token was a whitespace, increase offset + if prev.is_whitespace(): + tlist.tokens.pop(tlist.token_index(prev)) + offset += 1 + + # Previous token was a comment, add new line if necessary + if isinstance(prev, sql.Comment): + prev = str(prev) + if prev.endswith('\n') or prev.endswith('\r'): + nl = tlist.token_next(token) + + # New line was not added, set it now + if nl == None: nl = self.nl() tlist.insert_before(token, nl) + # Add token now token = _next_token(tlist.token_index(nl) + offset) def _split_statements(self, tlist): -- cgit v1.2.1 From 493d04a082adc0ffd02547ef373dd3156d46f8c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 15:42:09 +0200 Subject: Improved ColumnsSelect --- sqlparse/filters.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 92d7bd5..bc2af95 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -458,17 +458,21 @@ class ColumnsSelect: mode = 1 # We have detected a SELECT statement - elif mode == 1: - if value == 'FROM': + elif mode in (1, 3): + if value in ('FROM', 'WHERE', 'GROUP'): if oldValue: yield oldValue + oldValue = "" - mode = 3 # Columns have been checked + break # Columns have been checked elif value == 'AS': oldValue = "" mode = 2 + elif token_type in Whitespace: + mode = 3 + elif (token_type == Punctuation and value == ',' and not parenthesis): if oldValue: @@ -481,7 +485,11 @@ class ColumnsSelect: elif value == ')': parenthesis -= 1 - oldValue += value + if mode == 3: + oldValue = value + mode = 1 + else: + oldValue += value # We are processing an AS keyword elif mode == 2: @@ -490,6 +498,9 @@ class ColumnsSelect: yield value mode = 1 + if oldValue: + yield oldValue + # --------------------------- # postprocess -- cgit v1.2.1 From 42f9f77e07efc1c4f248aa9e48401e11c0ae4601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 17:36:56 +0200 Subject: Cleaned group_identifier_list() --- sqlparse/engine/grouping.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 1487c24..8f5ccd9 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -185,9 +185,10 @@ def group_identifier(tlist): def group_identifier_list(tlist): - [group_identifier_list(sgroup) for sgroup in tlist.get_sublists() - if not isinstance(sgroup, sql.IdentifierList)] - idx = 0 + for sgroup in tlist.get_sublists(): + if not isinstance(sgroup, sql.IdentifierList): + group_identifier_list(sgroup) + # Allowed list items fend1_funcs = [lambda t: isinstance(t, (sql.Identifier, sql.Function, sql.Case)), @@ -202,36 +203,40 @@ def group_identifier_list(tlist): lambda t: isinstance(t, sql.Comparison), lambda t: isinstance(t, sql.Comment), ] - tcomma = tlist.token_next_match(idx, T.Punctuation, ',') + + tcomma = tlist.token_next_match(0, T.Punctuation, ',') start = None - while tcomma is not None: + while tcomma: before = tlist.token_prev(tcomma) after = tlist.token_next(tcomma) + # Check if the tokens around tcomma belong to a list bpassed = apassed = False for func in fend1_funcs: - if before is not None and func(before): + if before and func(before): bpassed = True - if after is not None and func(after): + if after and func(after): apassed = True - if not bpassed or not apassed: - # Something's wrong here, skip ahead to next "," - start = None - tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, - T.Punctuation, ',') - else: + + if bpassed and apassed: if start is None: start = before + next_ = tlist.token_next(after) - if next_ is None or not next_.match(T.Punctuation, ','): + if next_ and next_.match(T.Punctuation, ','): + tcomma = next_ + else: # Reached the end of the list tokens = tlist.tokens_between(start, after) group = tlist.group_tokens(sql.IdentifierList, tokens) start = None tcomma = tlist.token_next_match(tlist.token_index(group) + 1, T.Punctuation, ',') - else: - tcomma = next_ + else: + # Something's wrong here, skip ahead to next "," + start = None + tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, + T.Punctuation, ',') def group_parenthesis(tlist): -- cgit v1.2.1 From 1b1810e4eaa5af17fe8d4f40717ae8f0a30554d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 18:00:43 +0200 Subject: Commented group_identifier_list() --- sqlparse/engine/grouping.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 8f5ccd9..37e029b 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -185,6 +185,7 @@ def group_identifier(tlist): def group_identifier_list(tlist): + # First group the `tlist` sublists for sgroup in tlist.get_sublists(): if not isinstance(sgroup, sql.IdentifierList): group_identifier_list(sgroup) @@ -204,13 +205,14 @@ def group_identifier_list(tlist): lambda t: isinstance(t, sql.Comment), ] - tcomma = tlist.token_next_match(0, T.Punctuation, ',') start = None + + tcomma = tlist.token_next_match(0, T.Punctuation, ',') while tcomma: before = tlist.token_prev(tcomma) after = tlist.token_next(tcomma) - # Check if the tokens around tcomma belong to a list + # Check if the tokens around tcomma belong to an identifier list bpassed = apassed = False for func in fend1_funcs: if before and func(before): @@ -218,22 +220,31 @@ def group_identifier_list(tlist): if after and func(after): apassed = True + # Both tokens around tcomma belong to a list if bpassed and apassed: - if start is None: + # Set the start of the identifier list if not defined before + if start == None: start = before + # Look if the next token is another comma next_ = tlist.token_next(after) if next_ and next_.match(T.Punctuation, ','): tcomma = next_ + + # Reached the end of the list else: - # Reached the end of the list + # Create and group the identifiers list tokens = tlist.tokens_between(start, after) group = tlist.group_tokens(sql.IdentifierList, tokens) + + # Skip ahead to next "," start = None tcomma = tlist.token_next_match(tlist.token_index(group) + 1, T.Punctuation, ',') + + # At least one of the tokens around tcomma don't belong to an + # identifier list. Something's wrong here, skip ahead to next "," else: - # Something's wrong here, skip ahead to next "," start = None tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, T.Punctuation, ',') -- cgit v1.2.1 From 9255cbc838b1fada24b2360ec9a12e311b9ced63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 19 May 2012 18:36:36 +0200 Subject: Improvements on identifierlist --- sqlparse/engine/grouping.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 37e029b..095827c 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -228,19 +228,25 @@ def group_identifier_list(tlist): # Look if the next token is another comma next_ = tlist.token_next(after) - if next_ and next_.match(T.Punctuation, ','): - tcomma = next_ + if next_: + if next_.match(T.Punctuation, ','): + tcomma = next_ + continue + + elif(next_.ttype == T.Keyword + and next_.value not in ('FROM', 'WHERE', 'GROUP')): + tcomma = next_ + continue # Reached the end of the list - else: - # Create and group the identifiers list - tokens = tlist.tokens_between(start, after) - group = tlist.group_tokens(sql.IdentifierList, tokens) + # Create and group the identifiers list + tokens = tlist.tokens_between(start, after) + group = tlist.group_tokens(sql.IdentifierList, tokens) - # Skip ahead to next "," - start = None - tcomma = tlist.token_next_match(tlist.token_index(group) + 1, - T.Punctuation, ',') + # Skip ahead to next "," + start = None + tcomma = tlist.token_next_match(tlist.token_index(group) + 1, + T.Punctuation, ',') # At least one of the tokens around tcomma don't belong to an # identifier list. Something's wrong here, skip ahead to next "," @@ -249,6 +255,12 @@ def group_identifier_list(tlist): tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, T.Punctuation, ',') + # There's an open identifier list + if start: + # Create and group the identifiers list + tokens = tlist.tokens_between(start, after) + group = tlist.group_tokens(sql.IdentifierList, tokens) + def group_parenthesis(tlist): _group_matching(tlist, T.Punctuation, '(', T.Punctuation, ')', -- cgit v1.2.1 From cd42b68188970376bc1e77473837083a4cd72226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 1 Jun 2012 22:32:58 +0200 Subject: Added comments --- sqlparse/filters.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index bc2af95..5919045 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -336,17 +336,32 @@ class ReindentFilter: self.offset -= num_offset def _process_identifierlist(self, tlist): + # Get identifiers from the tlist identifiers = list(tlist.get_identifiers()) + + # Split the identifier list if we have more than one identifier + # and its not from a function if len(identifiers) > 1 and not tlist.within(sql.Function): + # Get first token first = list(identifiers[0].flatten())[0] + + # Increase offset the size of the first token num_offset = self._get_offset(first) - len(first.value) self.offset += num_offset + + # Insert a new line between the tokens for token in identifiers[1:]: tlist.insert_before(token, self.nl()) + + # Imsert another new line after comment tokens for token in tlist.tokens: if isinstance(token, sql.Comment): tlist.insert_after(token, self.nl()) + + # Decrease offset the size of the first token self.offset -= num_offset + + # Process default tokens over tlist as usual self._process_default(tlist) def _process_case(self, tlist): @@ -383,7 +398,9 @@ class ReindentFilter: self._split_statements(tlist) if kwds: self._split_kwds(tlist) - [self._process(sgroup) for sgroup in tlist.get_sublists()] + + for sgroup in tlist.get_sublists(): + self._process(sgroup) def process(self, stack, stmt): if isinstance(stmt, sql.Statement): -- cgit v1.2.1 From 46014daed27b6cb0f042b6820937d391a2b2d424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 1 Jun 2012 22:59:54 +0200 Subject: Little clean-up --- sqlparse/filters.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 5919045..55af970 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -405,17 +405,19 @@ class ReindentFilter: def process(self, stack, stmt): if isinstance(stmt, sql.Statement): self._curr_stmt = stmt + self._process(stmt) + if isinstance(stmt, sql.Statement): - if self._last_stmt is not None: + if self._last_stmt: if unicode(self._last_stmt).endswith('\n'): nl = '\n' else: nl = '\n\n' - stmt.tokens.insert(0, - sql.Token(T.Whitespace, nl)) - if self._last_stmt != stmt: - self._last_stmt = stmt + + stmt.tokens.insert(0, sql.Token(T.Whitespace, nl)) + + self._last_stmt = stmt # FIXME: Doesn't work ;) -- cgit v1.2.1 From 7768a1a1bd67578ef2a591ac779782098716825d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Fri, 1 Jun 2012 23:05:56 +0200 Subject: Fixed bug on identifiers --- sqlparse/engine/grouping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 095827c..4120fc3 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -234,7 +234,7 @@ def group_identifier_list(tlist): continue elif(next_.ttype == T.Keyword - and next_.value not in ('FROM', 'WHERE', 'GROUP')): + and next_.value.upper() not in ('FROM', 'WHERE', 'GROUP')): tcomma = next_ continue -- cgit v1.2.1 From c7115f84fea3fd16922b5a3012d04a4ec185bb3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 2 Jun 2012 00:20:20 +0200 Subject: Optimized --- sqlparse/engine/grouping.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 4120fc3..bab932d 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -205,9 +205,16 @@ def group_identifier_list(tlist): lambda t: isinstance(t, sql.Comment), ] - start = None + def group(start, after): + """ + Create and group the identifiers list + """ + tokens = tlist.tokens_between(start, after) + return tlist.group_tokens(sql.IdentifierList, tokens) + start = None tcomma = tlist.token_next_match(0, T.Punctuation, ',') + while tcomma: before = tlist.token_prev(tcomma) after = tlist.token_next(tcomma) @@ -240,26 +247,16 @@ def group_identifier_list(tlist): # Reached the end of the list # Create and group the identifiers list - tokens = tlist.tokens_between(start, after) - group = tlist.group_tokens(sql.IdentifierList, tokens) + tcomma = group(start, after) - # Skip ahead to next "," - start = None - tcomma = tlist.token_next_match(tlist.token_index(group) + 1, - T.Punctuation, ',') + # Skip ahead to next "," + start = None + tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, + T.Punctuation, ',') - # At least one of the tokens around tcomma don't belong to an - # identifier list. Something's wrong here, skip ahead to next "," - else: - start = None - tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, - T.Punctuation, ',') - - # There's an open identifier list + # There's an open identifier list, create and group the identifiers list if start: - # Create and group the identifiers list - tokens = tlist.tokens_between(start, after) - group = tlist.group_tokens(sql.IdentifierList, tokens) + group(start, after) def group_parenthesis(tlist): -- cgit v1.2.1 From b89d03804d16c064d237e26340853bf4b3ed41f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 2 Jun 2012 00:36:54 +0200 Subject: Moved check for Function --- sqlparse/filters.py | 54 +++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 26 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 3ee2607..271bb75 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -422,32 +422,34 @@ class ReindentFilter: If there are more than an identifier, put each on a line """ - # Get identifiers from the tlist - identifiers = list(tlist.get_identifiers()) - - # Split the identifier list if we have more than one identifier - # and its not from a function - if len(identifiers) > 1 and not tlist.within(sql.Function): - # Get first token - first = list(identifiers[0].flatten())[0] - - # Increase offset the size of the first token - num_offset = self._get_offset(first) - len(first.value) - - # Increase offset and insert new lines - self.offset += num_offset - - # Insert a new line between the tokens - for token in identifiers[1:]: - tlist.insert_before(token, self.nl()) - - # Imsert another new line after comment tokens - for token in tlist.tokens: - if isinstance(token, sql.Comment): - tlist.insert_after(token, self.nl()) - - # Decrease offset the size of the first token - self.offset -= num_offset + # Split the identifier list if we are not in a function + if not tlist.within(sql.Function): + # Get identifiers from the tlist + identifiers = list(tlist.get_identifiers()) + print identifiers + + # Split the identifier list if we have more than one identifier + if len(identifiers) > 1: + # Get first token + first = list(identifiers[0].flatten())[0] + + # Increase offset the size of the first token + num_offset = self._get_offset(first) - len(first.value) + + # Increase offset and insert new lines + self.offset += num_offset + + # Insert a new line between the tokens + for token in identifiers[1:]: + tlist.insert_before(token, self.nl()) + + # Imsert another new line after comment tokens + for token in tlist.tokens: + if isinstance(token, sql.Comment): + tlist.insert_after(token, self.nl()) + + # Decrease offset the size of the first token + self.offset -= num_offset # Process the identifier list as usual self._process_default(tlist) -- cgit v1.2.1 From 56996812a9753fcf94117fd8c37eee60debeaf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 2 Jun 2012 00:37:11 +0200 Subject: Renamed function --- sqlparse/engine/grouping.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index bab932d..046ec9b 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -205,7 +205,7 @@ def group_identifier_list(tlist): lambda t: isinstance(t, sql.Comment), ] - def group(start, after): + def group_identifierlist(start, after): """ Create and group the identifiers list """ @@ -247,7 +247,7 @@ def group_identifier_list(tlist): # Reached the end of the list # Create and group the identifiers list - tcomma = group(start, after) + tcomma = group_identifierlist(start, after) # Skip ahead to next "," start = None @@ -256,7 +256,7 @@ def group_identifier_list(tlist): # There's an open identifier list, create and group the identifiers list if start: - group(start, after) + group_identifierlist(start, after) def group_parenthesis(tlist): -- cgit v1.2.1 From 05fb6421329c85dc8afaf400a5df2186da1971e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 2 Jun 2012 19:22:08 +0200 Subject: Fixed issue_50 (with some pending regressions) --- sqlparse/engine/grouping.py | 26 +++++++++++++++----------- sqlparse/filters.py | 23 ++++++++++++++++++++++- 2 files changed, 37 insertions(+), 12 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 046ec9b..66ae807 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -209,9 +209,11 @@ def group_identifier_list(tlist): """ Create and group the identifiers list """ + print "group_identifierlist", start, after tokens = tlist.tokens_between(start, after) return tlist.group_tokens(sql.IdentifierList, tokens) + # Search for the first identifier list start = None tcomma = tlist.token_next_match(0, T.Punctuation, ',') @@ -233,23 +235,25 @@ def group_identifier_list(tlist): if start == None: start = before - # Look if the next token is another comma - next_ = tlist.token_next(after) - if next_: - if next_.match(T.Punctuation, ','): - tcomma = next_ - continue + def continue_next(): + # Check the next token + next_ = tlist.token_next(after) + while next_: + # Next token is another comma or an identifier list keyword + if next_.match(T.Punctuation, ','): + return next_ - elif(next_.ttype == T.Keyword - and next_.value.upper() not in ('FROM', 'WHERE', 'GROUP')): - tcomma = next_ - continue + next_ = tlist.token_next(next_) + + tcomma = continue_next() + if tcomma: + continue # Reached the end of the list # Create and group the identifiers list tcomma = group_identifierlist(start, after) - # Skip ahead to next "," + # Skip ahead to next identifier list start = None tcomma = tlist.token_next_match(tlist.token_index(tcomma) + 1, T.Punctuation, ',') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 271bb75..bcfb776 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -438,16 +438,37 @@ class ReindentFilter: # Increase offset and insert new lines self.offset += num_offset + offset = 0 # Insert a new line between the tokens + ignore = False for token in identifiers[1:]: - tlist.insert_before(token, self.nl()) + if not ignore: + tlist.insert_before(token, self.nl()) + ignore = token.ttype + + # Check identifiers offset + if token.ttype: + l = len(token.value) + if offset < l: + offset = l # Imsert another new line after comment tokens for token in tlist.tokens: if isinstance(token, sql.Comment): tlist.insert_after(token, self.nl()) + # Update identifiers offset + if offset: + offset += 1 + + ignore = False + for token in identifiers: + if not ignore and not token.ttype: + tlist.insert_before(token, sql.Token(T.Whitespace, + " " * offset)) + ignore = token.ttype + # Decrease offset the size of the first token self.offset -= num_offset -- cgit v1.2.1 From c879535d316bfd914a89ce810b2926e4004121b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sun, 3 Jun 2012 02:01:33 +0200 Subject: Started new design for group_identifier_list() --- sqlparse/engine/grouping.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 66ae807..7dfc219 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -209,7 +209,6 @@ def group_identifier_list(tlist): """ Create and group the identifiers list """ - print "group_identifierlist", start, after tokens = tlist.tokens_between(start, after) return tlist.group_tokens(sql.IdentifierList, tokens) @@ -235,20 +234,27 @@ def group_identifier_list(tlist): if start == None: start = before - def continue_next(): # Check the next token next_ = tlist.token_next(after) while next_: - # Next token is another comma or an identifier list keyword - if next_.match(T.Punctuation, ','): - return next_ - +# # Next token is another comma or an identifier list keyword +# if next_.match(T.Punctuation, ','): +# return next_ +# +# next_ = tlist.token_next(next_) + + passed = False + for func in fend1_funcs: + if func(next_): + passed = True + break + + if not passed: + break + + after = next_ next_ = tlist.token_next(next_) - tcomma = continue_next() - if tcomma: - continue - # Reached the end of the list # Create and group the identifiers list tcomma = group_identifierlist(start, after) -- cgit v1.2.1 From b97a974e1c3b77118b50bd72a29c7046c7432450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sun, 3 Jun 2012 13:22:59 +0200 Subject: Finally fixed Issue #50 :-) --- sqlparse/engine/grouping.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 7dfc219..7b8b6b4 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -243,15 +243,17 @@ def group_identifier_list(tlist): # # next_ = tlist.token_next(next_) - passed = False - for func in fend1_funcs: - if func(next_): - passed = True + if next_.value != ',': + passed = False + for func in fend1_funcs: + if func(next_): + passed = True + break + + if not passed: + print "not passed", repr(next_) break - if not passed: - break - after = next_ next_ = tlist.token_next(next_) -- cgit v1.2.1 From 49b41027a3cb88b74f9d4ffb1adb34367c7763ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sun, 3 Jun 2012 13:25:32 +0200 Subject: Clean-up --- sqlparse/engine/grouping.py | 7 ------- sqlparse/filters.py | 1 - 2 files changed, 8 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 7b8b6b4..b82a317 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -237,12 +237,6 @@ def group_identifier_list(tlist): # Check the next token next_ = tlist.token_next(after) while next_: -# # Next token is another comma or an identifier list keyword -# if next_.match(T.Punctuation, ','): -# return next_ -# -# next_ = tlist.token_next(next_) - if next_.value != ',': passed = False for func in fend1_funcs: @@ -251,7 +245,6 @@ def group_identifier_list(tlist): break if not passed: - print "not passed", repr(next_) break after = next_ diff --git a/sqlparse/filters.py b/sqlparse/filters.py index bcfb776..50c7911 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -426,7 +426,6 @@ class ReindentFilter: if not tlist.within(sql.Function): # Get identifiers from the tlist identifiers = list(tlist.get_identifiers()) - print identifiers # Split the identifier list if we have more than one identifier if len(identifiers) > 1: -- cgit v1.2.1