Merged birkenfeld/pygments-main into default

4 files changed, 24 insertions, 39 deletions

diff --git a/pygments/lexer.py b/pygments/lexer.py
index 567e85f8..205b7495 100644
--- a/pygments/lexer.py
+++ b/pygments/lexer.py
@@ -18,7 +18,7 @@ from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
 
 
 __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
-           'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this']
+           'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this', 'default']
 
 
 _encoding_map = [(b'\xef\xbb\xbf', 'utf-8'),
@@ -383,6 +383,16 @@ def using(_other, **kwargs):
     return callback
 
 
+class default:
+    """
+    Indicates a state or state action (e.g. #pop) to apply.
+    For example default('#pop') is equivalent to ('', Token, '#pop')
+    Note that state tuples may be used as well
+    """
+    def __init__(self, state):
+        self.state = state
+
+
 class RegexLexerMeta(LexerMeta):
     """
     Metaclass for RegexLexer, creates the self._tokens attribute from
@@ -452,6 +462,10 @@ class RegexLexerMeta(LexerMeta):
             if isinstance(tdef, _inherit):
                 # processed already
                 continue
+            if isinstance(tdef, default):
+                new_state = cls._process_new_state(tdef.state, unprocessed, processed)
+                tokens.append((re.compile('').match, None, new_state))
+                continue
 
             assert type(tdef) is tuple, "wrong rule def %r" % tdef
 
@@ -582,11 +596,12 @@ class RegexLexer(Lexer):
             for rexmatch, action, new_state in statetokens:
                 m = rexmatch(text, pos)
                 if m:
-                    if type(action) is _TokenType:
-                        yield pos, action, m.group()
-                    else:
-                        for item in action(self, m):
-                            yield item
+                    if action is not None:
+                        if type(action) is _TokenType:
+                            yield pos, action, m.group()
+                        else:
+                            for item in action(self, m):
+                                yield item
                     pos = m.end()
                     if new_state is not None:
                         # state transition
diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py
index ec3015a5..c2c24cd2 100644
--- a/pygments/lexers/compiled.py
+++ b/pygments/lexers/compiled.py
@@ -13,7 +13,7 @@ import re
 from string import Template
 
 from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \
-     this, combined, inherit, do_insertions
+     this, combined, inherit, do_insertions, default
 from pygments.util import get_bool_opt, get_list_opt
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
      Number, Punctuation, Error, Literal, Generic
@@ -107,7 +107,7 @@ class CFamilyLexer(RegexLexer):
              r'(' + _ws + r')?(;)',
              bygroups(using(this), Name.Function, using(this), using(this),
                       Punctuation)),
-            ('', Text, 'statement'),
+            default('statement'),
         ],
         'statement' : [
             include('whitespace'),
diff --git a/tests/test_clexer.py b/tests/test_clexer.py
index 188569dc..4aac6d39 100644
--- a/tests/test_clexer.py
+++ b/tests/test_clexer.py
@@ -28,7 +28,7 @@ class CLexerTest(unittest.TestCase):
                          Number.Float, Number.Float], code.split()):
             wanted.append(item)
             wanted.append((Text, ' '))
-        wanted = [(Text, '')] + wanted[:-1] + [(Text, '\n')]
+        wanted = wanted[:-1] + [(Text, '\n')]
         self.assertEqual(list(self.lexer.get_tokens(code)), wanted)
 
     def testSwitch(self):
@@ -44,15 +44,12 @@ class CLexerTest(unittest.TestCase):
         }
         '''
         tokens = [
-            (Token.Text, u''),
             (Token.Keyword.Type, u'int'),
             (Token.Text, u' '),
             (Token.Name.Function, u'main'),
-            (Token.Text, u''),
             (Token.Punctuation, u'('),
             (Token.Punctuation, u')'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
             (Token.Punctuation, u'{'),
             (Token.Text, u'\n'),
             (Token.Text, u'    '),
@@ -83,7 +80,6 @@ class CLexerTest(unittest.TestCase):
             (Token.Text, u'\n'),
             (Token.Punctuation, u'}'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
 
@@ -100,15 +96,12 @@ class CLexerTest(unittest.TestCase):
         }
         '''
         tokens = [
-            (Token.Text, u''),
             (Token.Keyword.Type, u'int'),
             (Token.Text, u' '),
             (Token.Name.Function, u'main'),
-            (Token.Text, u''),
             (Token.Punctuation, u'('),
             (Token.Punctuation, u')'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
             (Token.Punctuation, u'{'),
             (Token.Text, u'\n'),
             (Token.Text, u'    '),
@@ -141,7 +134,6 @@ class CLexerTest(unittest.TestCase):
             (Token.Text, u'\n'),
             (Token.Punctuation, u'}'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
 
@@ -154,15 +146,12 @@ class CLexerTest(unittest.TestCase):
         }
         '''
         tokens = [
-            (Token.Text, u''),
             (Token.Keyword.Type, u'int'),
             (Token.Text, u' '),
             (Token.Name.Function, u'main'),
-            (Token.Text, u''),
             (Token.Punctuation, u'('),
             (Token.Punctuation, u')'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
             (Token.Punctuation, u'{'),
             (Token.Text, u'\n'),
             (Token.Name.Label, u'foo'),
@@ -176,7 +165,6 @@ class CLexerTest(unittest.TestCase):
             (Token.Text, u'\n'),
             (Token.Punctuation, u'}'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
 
@@ -189,15 +177,12 @@ class CLexerTest(unittest.TestCase):
         }
         '''
         tokens = [
-            (Token.Text, u''),
             (Token.Keyword.Type, u'int'),
             (Token.Text, u' '),
             (Token.Name.Function, u'main'),
-            (Token.Text, u''),
             (Token.Punctuation, u'('),
             (Token.Punctuation, u')'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
             (Token.Punctuation, u'{'),
             (Token.Text, u'\n'),
             (Token.Name.Label, u'foo'),
@@ -212,7 +197,6 @@ class CLexerTest(unittest.TestCase):
             (Token.Text, u'\n'),
             (Token.Punctuation, u'}'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
 
@@ -225,15 +209,12 @@ class CLexerTest(unittest.TestCase):
         }
         '''
         tokens = [
-            (Token.Text, u''),
             (Token.Keyword.Type, u'int'),
             (Token.Text, u' '),
             (Token.Name.Function, u'main'),
-            (Token.Text, u''),
             (Token.Punctuation, u'('),
             (Token.Punctuation, u')'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
             (Token.Punctuation, u'{'),
             (Token.Text, u'\n'),
             (Token.Name.Label, u'foo'),
@@ -251,6 +232,5 @@ class CLexerTest(unittest.TestCase):
             (Token.Text, u'\n'),
             (Token.Punctuation, u'}'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
diff --git a/tests/test_objectiveclexer.py b/tests/test_objectiveclexer.py
index 46fdb6d2..7339f6f7 100644
--- a/tests/test_objectiveclexer.py
+++ b/tests/test_objectiveclexer.py
@@ -22,20 +22,17 @@ class ObjectiveCLexerTest(unittest.TestCase):
     def testLiteralNumberInt(self):
         fragment = u'@(1);\n'
         expected = [
-            (Token.Text, u''),
             (Token.Literal, u'@('),
             (Token.Literal.Number.Integer, u'1'),
             (Token.Literal, u')'),
             (Token.Punctuation, u';'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
         ]
         self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
 
     def testLiteralNumberExpression(self):
         fragment = u'@(1+2);\n'
         expected = [
-            (Token.Text, u''),
             (Token.Literal, u'@('),
             (Token.Literal.Number.Integer, u'1'),
             (Token.Operator, u'+'),
@@ -43,14 +40,12 @@ class ObjectiveCLexerTest(unittest.TestCase):
             (Token.Literal, u')'),
             (Token.Punctuation, u';'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
         ]
         self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
 
     def testLiteralNumberNestedExpression(self):
         fragment = u'@(1+(2+3));\n'
         expected = [
-            (Token.Text, u''),
             (Token.Literal, u'@('),
             (Token.Literal.Number.Integer, u'1'),
             (Token.Operator, u'+'),
@@ -62,30 +57,25 @@ class ObjectiveCLexerTest(unittest.TestCase):
             (Token.Literal, u')'),
             (Token.Punctuation, u';'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
         ]
         self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
 
     def testLiteralNumberBool(self):
         fragment = u'@NO;\n'
         expected = [
-            (Token.Text, u''),
             (Token.Literal.Number, u'@NO'),
             (Token.Punctuation, u';'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
         ]
         self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
 
     def testLieralNumberBoolExpression(self):
         fragment = u'@(YES);\n'
         expected = [
-            (Token.Text, u''),
             (Token.Literal, u'@('),
             (Token.Name.Builtin, u'YES'),
             (Token.Literal, u')'),
             (Token.Punctuation, u';'),
             (Token.Text, u'\n'),
-            (Token.Text, u''),
         ]
         self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))