From 896774cb5298924abbcea81b9b90f1c7c10b3d6a Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 2 Jun 2016 07:38:27 +0200 Subject: Special-case group_tokens(..., tokens_between()) When having been guaranteed that the tokens form a range, it is possible to get rid of a lot of calls to `Token.tokens.remove(...)` which are expensive. --- sqlparse/sql.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 9afdac3..81cd8e9 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -329,6 +329,29 @@ class TokenList(Token): end_idx = include_end + self.token_index(end) return self.tokens[start_idx:end_idx] + def group_tokens_between(self, grp_cls, start, end, include_end=True, extend=False): + """Replace tokens by an instance of *grp_cls*.""" + start_idx = self.token_index(start) + end_idx = self.token_index(end) + include_end + tokens = self.tokens[start_idx:end_idx] + + if extend and isinstance(start, grp_cls): + subtokens = self.tokens[start_idx+1:end_idx] + + grp = start + grp.tokens.extend(subtokens) + del self.tokens[start_idx+1:end_idx] + grp.value = start.__str__() + else: + subtokens = self.tokens[start_idx:end_idx] + grp = grp_cls(tokens) + self.tokens[start_idx:end_idx] = [grp] + grp.parent = self + + for token in subtokens: + token.parent = grp + + return grp def group_tokens(self, grp_cls, tokens, ignore_ws=False, extend=False): """Replace tokens by an instance of *grp_cls*.""" if ignore_ws: -- cgit v1.2.1 From d4cc0644c8348da5e49c58df5e26a3e969045249 Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 2 Jun 2016 08:30:27 +0200 Subject: Replace _group_matching with an inward-out grouping algorithm All the matching between open/close was done all the time, first finding the matching closing token, and then grouping the tokens in between, and recurse over the newly created list. Instead, it is more efficient to look for the previous open-token on finding a closing-token, group these two together, and then continue on. squashed: Handle token indices in group_tokens_between and find_matching. --- sqlparse/sql.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 81cd8e9..dfe0430 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -331,9 +331,14 @@ class TokenList(Token): def group_tokens_between(self, grp_cls, start, end, include_end=True, extend=False): """Replace tokens by an instance of *grp_cls*.""" - start_idx = self.token_index(start) - end_idx = self.token_index(end) + include_end - tokens = self.tokens[start_idx:end_idx] + if isinstance(start, int): + start_idx = start + start = self.tokens[start_idx] + else: + start_idx = self.token_index(start) + + end_idx = self.token_index(end) if not isinstance(end, int) else end + end_idx += include_end if extend and isinstance(start, grp_cls): subtokens = self.tokens[start_idx+1:end_idx] @@ -344,7 +349,7 @@ class TokenList(Token): grp.value = start.__str__() else: subtokens = self.tokens[start_idx:end_idx] - grp = grp_cls(tokens) + grp = grp_cls(subtokens) self.tokens[start_idx:end_idx] = [grp] grp.parent = self -- cgit v1.2.1 From 67dc823e1174eee9ea2159674c8eb016b2f95b54 Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 2 Jun 2016 10:08:00 +0200 Subject: Use specialized token_idx_next_by in group_aliased. The method group_aliased was making a lot of calls to token_index. By specializing token_next_by to token_idx_next_by, the calls to token_index became superfluous. Also use token_idx_next_by in group_identifier_list. It was making a lot of calls, which is now more than reduced in half. --- sqlparse/sql.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index dfe0430..928b784 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -225,6 +225,22 @@ class TokenList(Token): def _groupable_tokens(self): return self.tokens + def _token_idx_matching(self, funcs, start=0, end=None, reverse=False): + """next token that match functions""" + if start is None: + return None + + if not isinstance(funcs, (list, tuple)): + funcs = (funcs,) + + iterable = enumerate(self.tokens[start:end], start=start) + + for idx, token in iterable: + for func in funcs: + if func(token): + return idx, token + return None, None + def _token_matching(self, funcs, start=0, end=None, reverse=False): """next token that match functions""" if start is None: @@ -259,6 +275,10 @@ class TokenList(Token): (ignore_comments and imt(tk, i=Comment))) return self._token_matching(funcs) + def token_idx_next_by(self, i=None, m=None, t=None, idx=0, end=None): + funcs = lambda tk: imt(tk, i, m, t) + return self._token_idx_matching(funcs, idx, end) + def token_next_by(self, i=None, m=None, t=None, idx=0, end=None): funcs = lambda tk: imt(tk, i, m, t) return self._token_matching(funcs, idx, end) -- cgit v1.2.1 From 8f7968ed5c649e5227e605ee272f59dd5ca75adb Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 2 Jun 2016 10:28:54 +0200 Subject: Index-based token_idx_prev Prevent some more calls to token_index in group_identifier_list. They are now all gone. --- sqlparse/sql.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 928b784..9782c33 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -233,12 +233,18 @@ class TokenList(Token): if not isinstance(funcs, (list, tuple)): funcs = (funcs,) - iterable = enumerate(self.tokens[start:end], start=start) - - for idx, token in iterable: - for func in funcs: - if func(token): - return idx, token + if reverse: + assert end is None + for idx in range(start - 2, -1, -1): + token = self.tokens[idx] + for func in funcs: + if func(token): + return idx, token + else: + for idx, token in enumerate(self.tokens[start:end], start=start): + for func in funcs: + if func(token): + return idx, token return None, None def _token_matching(self, funcs, start=0, end=None, reverse=False): @@ -312,6 +318,16 @@ class TokenList(Token): def token_matching(self, idx, funcs): return self._token_matching(funcs, idx) + def token_idx_prev(self, idx, skip_ws=True): + """Returns the previous token relative to *idx*. + + If *skip_ws* is ``True`` (the default) whitespace tokens are ignored. + ``None`` is returned if there's no previous token. + """ + idx += 1 # alot of code usage current pre-compensates for this + funcs = lambda tk: not (tk.is_whitespace() and skip_ws) + return self._token_idx_matching(funcs, idx, reverse=True) + def token_prev(self, idx, skip_ws=True): """Returns the previous token relative to *idx*. -- cgit v1.2.1 From 89d4f68ba5bbe78a9dd89257cbe4a9f3cfa76433 Mon Sep 17 00:00:00 2001 From: Sjoerd Job Postmus Date: Thu, 2 Jun 2016 11:58:19 +0200 Subject: Use a specialized token_idx_next. Prevent calling token_index. --- sqlparse/sql.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 9782c33..f3ef642 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -350,6 +350,26 @@ class TokenList(Token): funcs = lambda tk: not (tk.is_whitespace() and skip_ws) return self._token_matching(funcs, idx) + def token_idx_next(self, idx, skip_ws=True): + """Returns the next token relative to *idx*. + + If *skip_ws* is ``True`` (the default) whitespace tokens are ignored. + ``None`` is returned if there's no next token. + """ + if isinstance(idx, int): + idx += 1 # alot of code usage current pre-compensates for this + try: + if not skip_ws: + return idx, self.tokens[idx] + else: + while True: + token = self.tokens[idx] + if not token.is_whitespace(): + return idx, token + idx += 1 + except IndexError: + return None, None + def token_index(self, token, start=0): """Return list index of token.""" start = self.token_index(start) if not isinstance(start, int) else start -- cgit v1.2.1 From 0bcb34cc1514d77446a29c2c636a3f9a653588f2 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Sun, 12 Jun 2016 21:02:11 -0700 Subject: Remove unused code from sql.py and style up some changes --- sqlparse/sql.py | 45 +++++++-------------------------------------- 1 file changed, 7 insertions(+), 38 deletions(-) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 54f7d4f..027228d 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -338,16 +338,6 @@ class TokenList(Token): start = start if isinstance(start, int) else self.token_index(start) return start + self.tokens[start:].index(token) - def tokens_between(self, start, end, include_end=True): - """Return all tokens between (and including) start and end. - - If *include_end* is ``False`` (default is ``True``) the end token - is excluded. - """ - start_idx = self.token_index(start) - end_idx = include_end + self.token_index(end) - return self.tokens[start_idx:end_idx] - def group_tokens_between(self, grp_cls, start, end, include_end=True, extend=False): """Replace tokens by an instance of *grp_cls*.""" @@ -357,8 +347,12 @@ class TokenList(Token): else: start_idx = self.token_index(start) - end_idx = self.token_index(end) if not isinstance(end, int) else end - end_idx += include_end + end = end if isinstance(end, int) else self.token_index(end, start_idx) + end_idx = end + include_end + + # will be needed later for new group_clauses + # while skip_ws and tokens and tokens[-1].is_whitespace(): + # tokens = tokens[:-1] if extend and isinstance(start, grp_cls): subtokens = self.tokens[start_idx + 1:end_idx] @@ -366,7 +360,7 @@ class TokenList(Token): grp = start grp.tokens.extend(subtokens) del self.tokens[start_idx + 1:end_idx] - grp.value = start.__str__() + grp.value = text_type(start) else: subtokens = self.tokens[start_idx:end_idx] grp = grp_cls(subtokens) @@ -378,31 +372,6 @@ class TokenList(Token): return grp - def group_tokens(self, grp_cls, tokens, skip_ws=False, extend=False): - """Replace tokens by an instance of *grp_cls*.""" - - while skip_ws and tokens and tokens[-1].is_whitespace(): - tokens = tokens[:-1] - - left = tokens[0] - idx = self.token_index(left) - - if extend and isinstance(left, grp_cls): - grp = left - grp.tokens.extend(tokens[1:]) - else: - grp = grp_cls(tokens) - - for token in tokens: - token.parent = grp - - # Improve performance. LOOP(list.remove()) is O(n**2) operation - self.tokens = [token for token in self.tokens if token not in tokens] - - self.tokens.insert(idx, grp) - grp.parent = self - return grp - def insert_before(self, where, token): """Inserts *token* before *where*.""" token.parent = self -- cgit v1.2.1 From 997f95b8b6ec5129362dcfe5deedaf50800e3afc Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Mon, 13 Jun 2016 10:50:58 -0700 Subject: Change argument order to match order of all other functions --- sqlparse/sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 027228d..e0ac81d 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -269,12 +269,12 @@ class TokenList(Token): funcs = lambda tk: imt(tk, i, m, t) return self._token_matching(funcs, idx, end) - def token_not_matching(self, idx, funcs): + def token_not_matching(self, funcs, idx): funcs = (funcs,) if not isinstance(funcs, (list, tuple)) else funcs funcs = [lambda tk: not func(tk) for func in funcs] return self._token_matching(funcs, idx) - def token_matching(self, idx, funcs): + def token_matching(self, funcs, idx): return self._token_matching(funcs, idx) def token_idx_prev(self, idx, skip_ws=True): -- cgit v1.2.1 From a795be1a70a241e177227b742269fb2df88af962 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Mon, 13 Jun 2016 13:21:20 -0700 Subject: Change token_ funcs to token_idx funcs --- sqlparse/sql.py | 64 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 26 deletions(-) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index e0ac81d..d1d8e3e 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -259,7 +259,7 @@ class TokenList(Token): # this on is inconsistent, using Comment instead of T.Comment... funcs = lambda tk: not ((skip_ws and tk.is_whitespace()) or (skip_cm and imt(tk, t=T.Comment, i=Comment))) - return self._token_matching(funcs) + return self._token_idx_matching(funcs)[1] def token_idx_next_by(self, i=None, m=None, t=None, idx=0, end=None): funcs = lambda tk: imt(tk, i, m, t) @@ -272,19 +272,22 @@ class TokenList(Token): def token_not_matching(self, funcs, idx): funcs = (funcs,) if not isinstance(funcs, (list, tuple)) else funcs funcs = [lambda tk: not func(tk) for func in funcs] - return self._token_matching(funcs, idx) + return self._token_idx_matching(funcs, idx)[1] def token_matching(self, funcs, idx): - return self._token_matching(funcs, idx) + return self._token_idx_matching(funcs, idx)[1] - def token_idx_prev(self, idx, skip_ws=True): + def token_idx_prev(self, idx, skip_ws=True, skip_cm=False): """Returns the previous token relative to *idx*. If *skip_ws* is ``True`` (the default) whitespace tokens are ignored. ``None`` is returned if there's no previous token. """ + if idx is None: + return None, None idx += 1 # alot of code usage current pre-compensates for this - funcs = lambda tk: not (tk.is_whitespace() and skip_ws) + funcs = lambda tk: not ((skip_ws and tk.is_whitespace()) or + (skip_cm and imt(tk, t=T.Comment, i=Comment))) return self._token_idx_matching(funcs, idx, reverse=True) def token_prev(self, idx=0, skip_ws=True, skip_cm=False): @@ -313,14 +316,17 @@ class TokenList(Token): (skip_cm and imt(tk, t=T.Comment, i=Comment))) return self._token_matching(funcs, idx) - def token_idx_next(self, idx, skip_ws=True): + # TODO: May need to implement skip_cm for upstream changes. + # TODO: May need to re-add default value to idx + def token_idx_next(self, idx, skip_ws=True, skip_cm=False): """Returns the next token relative to *idx*. If *skip_ws* is ``True`` (the default) whitespace tokens are ignored. ``None`` is returned if there's no next token. """ - if isinstance(idx, int): - idx += 1 # alot of code usage current pre-compensates for this + if idx is None: + return None, None + idx += 1 # alot of code usage current pre-compensates for this try: if not skip_ws: return idx, self.tokens[idx] @@ -374,17 +380,21 @@ class TokenList(Token): def insert_before(self, where, token): """Inserts *token* before *where*.""" + if not isinstance(where, int): + where = self.token_index(where) token.parent = self - self.tokens.insert(self.token_index(where), token) + self.tokens.insert(where, token) def insert_after(self, where, token, skip_ws=True): """Inserts *token* after *where*.""" - next_token = self.token_next(where, skip_ws=skip_ws) + if not isinstance(where, int): + where = self.token_index(where) + nidx, next_ = self.token_idx_next(where, skip_ws=skip_ws) token.parent = self - if next_token is None: + if next_ is None: self.tokens.append(token) else: - self.insert_before(next_token, token) + self.tokens.insert(nidx, token) def has_alias(self): """Returns ``True`` if an alias is present.""" @@ -394,12 +404,13 @@ class TokenList(Token): """Returns the alias for this identifier or ``None``.""" # "name AS alias" - kw = self.token_next_by(m=(T.Keyword, 'AS')) + kw_idx, kw = self.token_idx_next_by(m=(T.Keyword, 'AS')) if kw is not None: - return self._get_first_name(kw, keywords=True) + return self._get_first_name(kw_idx + 1, keywords=True) # "name alias" or "complicated column expression alias" - if len(self.tokens) > 2 and self.token_next_by(t=T.Whitespace): + _, ws = self.token_idx_next_by(t=T.Whitespace) + if len(self.tokens) > 2 and ws is not None: return self._get_first_name(reverse=True) def get_name(self): @@ -414,16 +425,16 @@ class TokenList(Token): def get_real_name(self): """Returns the real name (object name) of this identifier.""" # a.b - dot = self.token_next_by(m=(T.Punctuation, '.')) - return self._get_first_name(dot) + dot_idx, _ = self.token_idx_next_by(m=(T.Punctuation, '.')) + return self._get_first_name(dot_idx) def get_parent_name(self): """Return name of the parent object if any. A parent object is identified by the first occuring dot. """ - dot = self.token_next_by(m=(T.Punctuation, '.')) - prev_ = self.token_prev(dot) + dot_idx, _ = self.token_idx_next_by(m=(T.Punctuation, '.')) + _, prev_ = self.token_idx_prev(dot_idx) return remove_quotes(prev_.value) if prev_ is not None else None def _get_first_name(self, idx=None, reverse=False, keywords=False): @@ -472,9 +483,10 @@ class Statement(TokenList): # The WITH keyword should be followed by either an Identifier or # an IdentifierList containing the CTE definitions; the actual # DML keyword (e.g. SELECT, INSERT) will follow next. - token = self.token_next(first_token, skip_ws=True) + fidx = self.token_index(first_token) + tidx, token = self.token_idx_next(fidx, skip_ws=True) if isinstance(token, (Identifier, IdentifierList)): - dml_keyword = self.token_next(token, skip_ws=True) + _, dml_keyword = self.token_idx_next(tidx, skip_ws=True) if dml_keyword.ttype == T.Keyword.DML: return dml_keyword.normalized @@ -491,18 +503,18 @@ class Identifier(TokenList): def is_wildcard(self): """Return ``True`` if this identifier contains a wildcard.""" - token = self.token_next_by(t=T.Wildcard) + _, token = self.token_idx_next_by(t=T.Wildcard) return token is not None def get_typecast(self): """Returns the typecast or ``None`` of this object as a string.""" - marker = self.token_next_by(m=(T.Punctuation, '::')) - next_ = self.token_next(marker, skip_ws=False) + midx, marker = self.token_idx_next_by(m=(T.Punctuation, '::')) + nidx, next_ = self.token_idx_next(midx, skip_ws=False) return next_.value if next_ else None def get_ordering(self): """Returns the ordering or ``None`` as uppercase string.""" - ordering = self.token_next_by(t=T.Keyword.Order) + _, ordering = self.token_idx_next_by(t=T.Keyword.Order) return ordering.normalized if ordering else None def get_array_indices(self): @@ -649,7 +661,7 @@ class Function(TokenList): """Return a list of parameters.""" parenthesis = self.tokens[-1] for token in parenthesis.tokens: - if imt(token, i=IdentifierList): + if isinstance(token, IdentifierList): return token.get_identifiers() elif imt(token, i=(Function, Identifier), t=T.Literal): return [token, ] -- cgit v1.2.1 From 711744d1664f8244d8ab0b090cbf12e923101cce Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Mon, 13 Jun 2016 21:50:17 -0700 Subject: Remove functions no-longer used --- sqlparse/sql.py | 51 --------------------------------------------------- 1 file changed, 51 deletions(-) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index d1d8e3e..af282a3 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -226,27 +226,6 @@ class TokenList(Token): return idx, token return None, None - def _token_matching(self, funcs, start=0, end=None, reverse=False): - """next token that match functions""" - if start is None: - return None - - if not isinstance(start, int): - start = self.token_index(start) + 1 - - if not isinstance(funcs, (list, tuple)): - funcs = (funcs,) - - if reverse: - iterable = reversed(self.tokens[end:start - 1]) - else: - iterable = self.tokens[start:end] - - for token in iterable: - for func in funcs: - if func(token): - return token - def token_first(self, skip_ws=True, skip_cm=False): """Returns the first child token. @@ -265,10 +244,6 @@ class TokenList(Token): funcs = lambda tk: imt(tk, i, m, t) return self._token_idx_matching(funcs, idx, end) - def token_next_by(self, i=None, m=None, t=None, idx=0, end=None): - funcs = lambda tk: imt(tk, i, m, t) - return self._token_matching(funcs, idx, end) - def token_not_matching(self, funcs, idx): funcs = (funcs,) if not isinstance(funcs, (list, tuple)) else funcs funcs = [lambda tk: not func(tk) for func in funcs] @@ -290,32 +265,6 @@ class TokenList(Token): (skip_cm and imt(tk, t=T.Comment, i=Comment))) return self._token_idx_matching(funcs, idx, reverse=True) - def token_prev(self, idx=0, skip_ws=True, skip_cm=False): - """Returns the previous token relative to *idx*. - - If *skip_ws* is ``True`` (the default) whitespace tokens are ignored. - ``None`` is returned if there's no previous token. - """ - if isinstance(idx, int): - idx += 1 # alot of code usage current pre-compensates for this - funcs = lambda tk: not ((skip_ws and tk.is_whitespace()) or - (skip_cm and imt(tk, t=T.Comment, i=Comment))) - return self._token_matching(funcs, idx, reverse=True) - - def token_next(self, idx=0, skip_ws=True, skip_cm=False): - """Returns the next token relative to *idx*. - - If called with idx = 0. Returns the first child token. - If *skip_ws* is ``True`` (the default) whitespace tokens are ignored. - If *skip_cm* is ``True`` (default: ``False``), comments are ignored. - ``None`` is returned if there's no next token. - """ - if isinstance(idx, int): - idx += 1 # alot of code usage current pre-compensates for this - funcs = lambda tk: not ((skip_ws and tk.is_whitespace()) or - (skip_cm and imt(tk, t=T.Comment, i=Comment))) - return self._token_matching(funcs, idx) - # TODO: May need to implement skip_cm for upstream changes. # TODO: May need to re-add default value to idx def token_idx_next(self, idx, skip_ws=True, skip_cm=False): -- cgit v1.2.1 From 4f922d9b6fb68b8281c6b3d93a57a4c84860e06a Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Mon, 13 Jun 2016 22:01:53 -0700 Subject: Rename token_idx_ funcs to simply token_ funcs --- sqlparse/sql.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index af282a3..ed56793 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -204,7 +204,7 @@ class TokenList(Token): def _groupable_tokens(self): return self.tokens - def _token_idx_matching(self, funcs, start=0, end=None, reverse=False): + def _token_matching(self, funcs, start=0, end=None, reverse=False): """next token that match functions""" if start is None: return None @@ -238,21 +238,21 @@ class TokenList(Token): # this on is inconsistent, using Comment instead of T.Comment... funcs = lambda tk: not ((skip_ws and tk.is_whitespace()) or (skip_cm and imt(tk, t=T.Comment, i=Comment))) - return self._token_idx_matching(funcs)[1] + return self._token_matching(funcs)[1] - def token_idx_next_by(self, i=None, m=None, t=None, idx=0, end=None): + def token_next_by(self, i=None, m=None, t=None, idx=0, end=None): funcs = lambda tk: imt(tk, i, m, t) - return self._token_idx_matching(funcs, idx, end) + return self._token_matching(funcs, idx, end) def token_not_matching(self, funcs, idx): funcs = (funcs,) if not isinstance(funcs, (list, tuple)) else funcs funcs = [lambda tk: not func(tk) for func in funcs] - return self._token_idx_matching(funcs, idx)[1] + return self._token_matching(funcs, idx)[1] def token_matching(self, funcs, idx): - return self._token_idx_matching(funcs, idx)[1] + return self._token_matching(funcs, idx)[1] - def token_idx_prev(self, idx, skip_ws=True, skip_cm=False): + def token_prev(self, idx, skip_ws=True, skip_cm=False): """Returns the previous token relative to *idx*. If *skip_ws* is ``True`` (the default) whitespace tokens are ignored. @@ -263,11 +263,11 @@ class TokenList(Token): idx += 1 # alot of code usage current pre-compensates for this funcs = lambda tk: not ((skip_ws and tk.is_whitespace()) or (skip_cm and imt(tk, t=T.Comment, i=Comment))) - return self._token_idx_matching(funcs, idx, reverse=True) + return self._token_matching(funcs, idx, reverse=True) # TODO: May need to implement skip_cm for upstream changes. # TODO: May need to re-add default value to idx - def token_idx_next(self, idx, skip_ws=True, skip_cm=False): + def token_next(self, idx, skip_ws=True, skip_cm=False): """Returns the next token relative to *idx*. If *skip_ws* is ``True`` (the default) whitespace tokens are ignored. @@ -293,8 +293,8 @@ class TokenList(Token): start = start if isinstance(start, int) else self.token_index(start) return start + self.tokens[start:].index(token) - def group_tokens_between(self, grp_cls, start, end, include_end=True, - extend=False): + def group_tokens(self, grp_cls, start, end, include_end=True, + extend=False): """Replace tokens by an instance of *grp_cls*.""" if isinstance(start, int): start_idx = start @@ -338,7 +338,7 @@ class TokenList(Token): """Inserts *token* after *where*.""" if not isinstance(where, int): where = self.token_index(where) - nidx, next_ = self.token_idx_next(where, skip_ws=skip_ws) + nidx, next_ = self.token_next(where, skip_ws=skip_ws) token.parent = self if next_ is None: self.tokens.append(token) @@ -353,12 +353,12 @@ class TokenList(Token): """Returns the alias for this identifier or ``None``.""" # "name AS alias" - kw_idx, kw = self.token_idx_next_by(m=(T.Keyword, 'AS')) + kw_idx, kw = self.token_next_by(m=(T.Keyword, 'AS')) if kw is not None: return self._get_first_name(kw_idx + 1, keywords=True) # "name alias" or "complicated column expression alias" - _, ws = self.token_idx_next_by(t=T.Whitespace) + _, ws = self.token_next_by(t=T.Whitespace) if len(self.tokens) > 2 and ws is not None: return self._get_first_name(reverse=True) @@ -374,7 +374,7 @@ class TokenList(Token): def get_real_name(self): """Returns the real name (object name) of this identifier.""" # a.b - dot_idx, _ = self.token_idx_next_by(m=(T.Punctuation, '.')) + dot_idx, _ = self.token_next_by(m=(T.Punctuation, '.')) return self._get_first_name(dot_idx) def get_parent_name(self): @@ -382,8 +382,8 @@ class TokenList(Token): A parent object is identified by the first occuring dot. """ - dot_idx, _ = self.token_idx_next_by(m=(T.Punctuation, '.')) - _, prev_ = self.token_idx_prev(dot_idx) + dot_idx, _ = self.token_next_by(m=(T.Punctuation, '.')) + _, prev_ = self.token_prev(dot_idx) return remove_quotes(prev_.value) if prev_ is not None else None def _get_first_name(self, idx=None, reverse=False, keywords=False): @@ -433,9 +433,9 @@ class Statement(TokenList): # an IdentifierList containing the CTE definitions; the actual # DML keyword (e.g. SELECT, INSERT) will follow next. fidx = self.token_index(first_token) - tidx, token = self.token_idx_next(fidx, skip_ws=True) + tidx, token = self.token_next(fidx, skip_ws=True) if isinstance(token, (Identifier, IdentifierList)): - _, dml_keyword = self.token_idx_next(tidx, skip_ws=True) + _, dml_keyword = self.token_next(tidx, skip_ws=True) if dml_keyword.ttype == T.Keyword.DML: return dml_keyword.normalized @@ -452,18 +452,18 @@ class Identifier(TokenList): def is_wildcard(self): """Return ``True`` if this identifier contains a wildcard.""" - _, token = self.token_idx_next_by(t=T.Wildcard) + _, token = self.token_next_by(t=T.Wildcard) return token is not None def get_typecast(self): """Returns the typecast or ``None`` of this object as a string.""" - midx, marker = self.token_idx_next_by(m=(T.Punctuation, '::')) - nidx, next_ = self.token_idx_next(midx, skip_ws=False) + midx, marker = self.token_next_by(m=(T.Punctuation, '::')) + nidx, next_ = self.token_next(midx, skip_ws=False) return next_.value if next_ else None def get_ordering(self): """Returns the ordering or ``None`` as uppercase string.""" - _, ordering = self.token_idx_next_by(t=T.Keyword.Order) + _, ordering = self.token_next_by(t=T.Keyword.Order) return ordering.normalized if ordering else None def get_array_indices(self): -- cgit v1.2.1 From 5002bfa36c4fa2ee72eff18648b6ddc616b718f0 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Mon, 13 Jun 2016 22:20:29 -0700 Subject: Normalize behavior between token_next and token_next_by both will now return the "next" token and not itself when passing own index --- sqlparse/sql.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index ed56793..4b6abf1 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -240,8 +240,9 @@ class TokenList(Token): (skip_cm and imt(tk, t=T.Comment, i=Comment))) return self._token_matching(funcs)[1] - def token_next_by(self, i=None, m=None, t=None, idx=0, end=None): + def token_next_by(self, i=None, m=None, t=None, idx=-1, end=None): funcs = lambda tk: imt(tk, i, m, t) + idx += 1 return self._token_matching(funcs, idx, end) def token_not_matching(self, funcs, idx): -- cgit v1.2.1 From 56b28dc15023d36bab8764bea6df75e28651646e Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Tue, 14 Jun 2016 04:15:27 -0700 Subject: Make use of token_index more obvious --- sqlparse/sql.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'sqlparse/sql.py') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 4b6abf1..9656390 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -248,7 +248,7 @@ class TokenList(Token): def token_not_matching(self, funcs, idx): funcs = (funcs,) if not isinstance(funcs, (list, tuple)) else funcs funcs = [lambda tk: not func(tk) for func in funcs] - return self._token_matching(funcs, idx)[1] + return self._token_matching(funcs, idx) def token_matching(self, funcs, idx): return self._token_matching(funcs, idx)[1] @@ -297,13 +297,9 @@ class TokenList(Token): def group_tokens(self, grp_cls, start, end, include_end=True, extend=False): """Replace tokens by an instance of *grp_cls*.""" - if isinstance(start, int): - start_idx = start - start = self.tokens[start_idx] - else: - start_idx = self.token_index(start) + start_idx = start + start = self.tokens[start_idx] - end = end if isinstance(end, int) else self.token_index(end, start_idx) end_idx = end + include_end # will be needed later for new group_clauses @@ -390,9 +386,6 @@ class TokenList(Token): def _get_first_name(self, idx=None, reverse=False, keywords=False): """Returns the name of the first token with a name""" - if idx and not isinstance(idx, int): - idx = self.token_index(idx) + 1 - tokens = self.tokens[idx:] if idx else self.tokens tokens = reversed(tokens) if reverse else tokens types = [T.Name, T.Wildcard, T.String.Symbol] -- cgit v1.2.1