diff options
Diffstat (limited to 'sqlparse/lexer.py')
| -rw-r--r-- | sqlparse/lexer.py | 52 |
1 files changed, 34 insertions, 18 deletions
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index 6e17fca..9d25c9e 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -7,6 +7,7 @@ """SQL Lexer""" import re + # This code is based on the SqlLexer in pygments. # http://pygments.org/ # It's separated from the rest of pygments to increase performance @@ -18,21 +19,39 @@ from sqlparse import tokens, keywords from sqlparse.utils import consume -class _LexerSingletonMetaclass(type): - _lexer_instance = None - - def __call__(cls, *args, **kwargs): - if _LexerSingletonMetaclass._lexer_instance is None: - _LexerSingletonMetaclass._lexer_instance = super( - _LexerSingletonMetaclass, cls - ).__call__(*args, **kwargs) - return _LexerSingletonMetaclass._lexer_instance - - -class Lexer(metaclass=_LexerSingletonMetaclass): +class Lexer: """The Lexer supports configurable syntax. To add support for additional keywords, use the `add_keywords` method.""" + _default_intance = None + + # Development notes: + # - This class is prepared to be able to support additional SQL dialects + # in the future by adding additional functions that take the place of + # the function default_initialization() + # - The lexer class uses an explicit singleton behavior with the + # instance-getter method get_default_instance(). This mechanism has + # the advantage that the call signature of the entry-points to the + # sqlparse library are not affected. Also, usage of sqlparse in third + # party code does not need to be adapted. On the other hand, singleton + # behavior is not thread safe, and the current implementation does not + # easily allow for multiple SQL dialects to be parsed in the same + # process. Such behavior can be supported in the future by passing a + # suitably initialized lexer object as an additional parameter to the + # entry-point functions (such as `parse`). Code will need to be written + # to pass down and utilize such an object. The current implementation + # is prepared to support this thread safe approach without the + # default_instance part needing to change interface. + + @classmethod + def get_default_instance(cls): + """Returns the lexer instance used internally + by the sqlparse core functions.""" + if cls._default_intance is None: + cls._default_intance = cls() + cls._default_intance.default_initialization() + return cls._default_intance + def default_initialization(self): """Initialize the lexer with default dictionaries. Useful if you need to revert custom syntax settings.""" @@ -45,13 +64,10 @@ class Lexer(metaclass=_LexerSingletonMetaclass): self.add_keywords(keywords.KEYWORDS_MSACCESS) self.add_keywords(keywords.KEYWORDS) - def __init__(self): - self.default_initialization() - def clear(self): """Clear all syntax configurations. Useful if you want to load a reduced set of syntax configurations. - After this call, reg-exps and keyword dictionaries need to be loaded + After this call, regexps and keyword dictionaries need to be loaded to make the lexer functional again.""" self._SQL_REGEX = [] self._keywords = [] @@ -73,7 +89,7 @@ class Lexer(metaclass=_LexerSingletonMetaclass): """Checks for a keyword. If the given value is in one of the KEYWORDS_* dictionary - it's considered a keyword. Otherwise tokens.Name is returned. + it's considered a keyword. Otherwise, tokens.Name is returned. """ val = value.upper() for kwdict in self._keywords: @@ -136,4 +152,4 @@ def tokenize(sql, encoding=None): Tokenize *sql* using the :class:`Lexer` and return a 2-tuple stream of ``(token type, value)`` items. """ - return Lexer().get_tokens(sql, encoding) + return Lexer.get_default_instance().get_tokens(sql, encoding) |