[svn] implemented filters for pygments (first approach, api might change), it's impossible at the moment to add filters by using pygmentize

16 files changed, 72 insertions, 19 deletions

diff --git a/pygments/__init__.py b/pygments/__init__.py
index 027eafd3..eafa867f 100644
--- a/pygments/__init__.py
+++ b/pygments/__init__.py
@@ -64,7 +64,8 @@ def format(tokens, formatter, outfile=None):
 def highlight(code, lexer, formatter, outfile=None):
     """
     Lex ``code`` with ``lexer`` and format it with the formatter
-    ``formatter``.
+    ``formatter``. If ``filters`` are given they will be applied
+    on the token stream.
 
     If ``outfile`` is given and a valid file object (an object
     with a ``write`` method), the result will be written to it, otherwise
diff --git a/pygments/lexer.py b/pygments/lexer.py
index 16a6b8ab..7e141887 100644
--- a/pygments/lexer.py
+++ b/pygments/lexer.py
@@ -15,8 +15,11 @@ try:
 except NameError:
     from sets import Set as set
 
+from pygments.filter import apply_filters, Filter
+from pygments.filters import find_filter
 from pygments.token import Error, Text, Other, _TokenType
-from pygments.util import get_bool_opt, get_int_opt, make_analysator
+from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
+     make_analysator
 
 
 __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
@@ -81,6 +84,9 @@ class Lexer(object):
         self.stripall = get_bool_opt(options, 'stripall', False)
         self.tabsize = get_int_opt(options, 'tabsize', 0)
         self.encoding = options.get('encoding', 'latin1')
+        self.filters = []
+        for filter in get_list_opt(options, 'filters', ()):
+            self.add_filter(filter)
 
     def __repr__(self):
         if self.options:
@@ -89,6 +95,14 @@ class Lexer(object):
         else:
             return '<pygments.lexers.%s>' % self.__class__.__name__
 
+    def add_filter(self, filter, **options):
+        """
+        Add a new stream filter to this lexer.
+        """
+        if not isinstance(filter, Filter):
+            filter = find_filter(filter, **options)
+        self.filters.append(filter)
+
     def analyse_text(text):
         """
         Has to return a float between ``0`` and ``1`` that indicates
@@ -103,11 +117,14 @@ class Lexer(object):
         it's the same as if the return values was ``0.0``.
         """
 
-    def get_tokens(self, text):
+    def get_tokens(self, text, unfiltered=False):
         """
-        Return an iterable of (tokentype, value) pairs generated from ``text``.
+        Return an iterable of (tokentype, value) pairs generated from
+        `text`. If `unfiltered` is set to `True` the filtering mechanism
+        is bypassed, even if filters are defined.
 
-        Also preprocess the text, i.e. expand tabs and strip it if wanted.
+        Also preprocess the text, i.e. expand tabs and strip it if
+        wanted and applies registered filters.
         """
         if isinstance(text, unicode):
             text = u'\n'.join(text.splitlines())
@@ -122,9 +139,9 @@ class Lexer(object):
                 try:
                     import chardet
                 except ImportError:
-                    raise ImportError('To enable chardet encoding guessing, please '
-                                      'install the chardet library from '
-                                      'http://chardet.feedparser.org/')
+                    raise ImportError('To enable chardet encoding guessing, '
+                                      'please install the chardet library '
+                                      'from http://chardet.feedparser.org/')
                 enc = chardet.detect(text)
                 text = text.decode(enc['encoding'])
             else:
@@ -138,8 +155,13 @@ class Lexer(object):
         if not text.endswith('\n'):
             text += '\n'
 
-        for i, t, v in self.get_tokens_unprocessed(text):
-            yield t, v
+        def streamer():
+            for i, t, v in self.get_tokens_unprocessed(text):
+                yield t, v
+        stream = streamer()
+        if not unfiltered:
+            stream = apply_filters(stream, self.filters, self)
+        return stream
 
     def get_tokens_unprocessed(self, text):
         """
diff --git a/pygments/plugin.py b/pygments/plugin.py
index 4422cbb5..89300677 100644
--- a/pygments/plugin.py
+++ b/pygments/plugin.py
@@ -26,6 +26,11 @@
         [pygments.styles]
         yourstyle = yourstyle:YourStyle
 
+    filter plugin::
+
+        [pygments.filter]
+        yourfilter = yourfilter:YourFilter
+
 
     :copyright: 2006 by Armin Ronacher.
     :license: BSD, see LICENSE for more details.
@@ -38,6 +43,7 @@ except ImportError:
 LEXER_ENTRY_POINT = 'pygments.lexers'
 FORMATTER_ENTRY_POINT = 'pygments.formatters'
 STYLE_ENTRY_POINT = 'pygments.styles'
+FILTER_ENTRY_POINT = 'pygments.filters'
 
 
 def find_plugin_lexers():
@@ -59,3 +65,10 @@ def find_plugin_styles():
         return
     for entrypoint in pkg_resources.iter_entry_points(STYLE_ENTRY_POINT):
         yield entrypoint.name, entrypoint.load()
+
+
+def find_plugin_filters():
+    if pkg_recources is None:
+        return
+    for entrypoint in pkg_resources.iter_entry_points(FILTER_ENTRY_POINT):
+        yield entrypoint.name, entrypoint.load()
diff --git a/pygments/styles/autumn.py b/pygments/styles/autumn.py
index 0e9c64d2..1862c3ad 100644
--- a/pygments/styles/autumn.py
+++ b/pygments/styles/autumn.py
@@ -21,6 +21,7 @@ class AutumnStyle(Style):
     styles = {
         Comment:                    "italic #aaaaaa",
         Comment.Preproc:            "noitalic #4c8317",
+        Comment.Special:            "italic #0000aa",
 
         Keyword:                    "#0000aa",
         Keyword.Type:               "#00aaaa",
diff --git a/pygments/styles/borland.py b/pygments/styles/borland.py
index e6c14b19..b798ebe0 100644
--- a/pygments/styles/borland.py
+++ b/pygments/styles/borland.py
@@ -21,6 +21,7 @@ class BorlandStyle(Style):
     styles = {
         Comment:                'italic #008800',
         Comment.Preproc:        'noitalic',
+        Comment.Special:        'noitalic bold',
 
         String:                 '#0000FF',
         Number:                 '#0000FF',
diff --git a/pygments/styles/colorful.py b/pygments/styles/colorful.py
index 0e3bc5e6..f542d3fc 100644
--- a/pygments/styles/colorful.py
+++ b/pygments/styles/colorful.py
@@ -21,6 +21,7 @@ class ColorfulStyle(Style):
     styles = {
         Comment:                   "#888",
         Comment.Preproc:           "#579",
+        Comment.Special:           "bold #cc0000",
 
         Keyword:                   "bold #080",
         Keyword.Pseudo:            "#038",
diff --git a/pygments/styles/default.py b/pygments/styles/default.py
index 277c323f..a7ba6eb8 100644
--- a/pygments/styles/default.py
+++ b/pygments/styles/default.py
@@ -25,6 +25,7 @@ class DefaultStyle(Style):
     styles = {
         Comment:                   "italic #008800",
         Comment.Preproc:           "noitalic",
+        Comment.Special:           "noitalic bold",
 
         Keyword:                   "bold #AA22FF",
         Keyword.Pseudo:            "nobold",
diff --git a/pygments/styles/friendly.py b/pygments/styles/friendly.py
index af983909..e6762091 100644
--- a/pygments/styles/friendly.py
+++ b/pygments/styles/friendly.py
@@ -22,6 +22,7 @@ class FriendlyStyle(Style):
     styles = {
         Comment:                   "italic #60a0b0",
         Comment.Preproc:           "noitalic #007020",
+        Comment.Special:           "noitalic bg:#fff0f0",
 
         Keyword:                   "bold #007020",
         Keyword.Pseudo:            "nobold",
diff --git a/pygments/styles/manni.py b/pygments/styles/manni.py
index 61e041fc..1f99ad1f 100644
--- a/pygments/styles/manni.py
+++ b/pygments/styles/manni.py
@@ -24,6 +24,7 @@ class ManniStyle(Style):
     styles = {
         Comment:            'italic #0099FF',
         Comment.Preproc:    'noitalic #009999',
+        Comment.Special:    'bold',
 
         Keyword:            'bold #006699',
         Keyword.Pseudo:     'nobold',
diff --git a/pygments/styles/murphy.py b/pygments/styles/murphy.py
index 425a1683..82ae6d15 100644
--- a/pygments/styles/murphy.py
+++ b/pygments/styles/murphy.py
@@ -21,6 +21,7 @@ class MurphyStyle(Style):
     styles = {
         Comment:                   "#666 italic",
         Comment.Preproc:           "#579 noitalic",
+        Comment.Special:           "#c00 bold",
 
         Keyword:                   "bold #289",
         Keyword.Pseudo:            "#08f",
diff --git a/pygments/styles/native.py b/pygments/styles/native.py
index 057e66ce..2f1e6576 100644
--- a/pygments/styles/native.py
+++ b/pygments/styles/native.py
@@ -23,6 +23,7 @@ class NativeStyle(Style):
 
         Comment:            'italic #999999',
         Comment.Preproc:    'noitalic bold #cd2828',
+        Comment.Special:    'noitalic bold #e50808 bg:#520000',
 
         Keyword:            'bold #6ab825',
         Keyword.Pseudo:     'nobold',
diff --git a/pygments/styles/pastie.py b/pygments/styles/pastie.py
index a6b3a444..3cfe781d 100644
--- a/pygments/styles/pastie.py
+++ b/pygments/styles/pastie.py
@@ -23,6 +23,7 @@ class PastieStyle(Style):
     styles = {
         Comment:                '#888888',
         Comment.Preproc:        'bold #cc0000',
+        Comment.Special:        'bg:#fff0f0 bold #cc0000',
 
         String:                 'bg:#fff0f0 #dd2200',
         String.Regex:           'bg:#fff0ff #008800',
@@ -41,7 +42,7 @@ class PastieStyle(Style):
         Name.Exception:         'bold #bb0066',
         Name.Function:          'bold #0066bb',
         Name.Property:          'bold #336699',
-        Name.Module:            'bold #bb0066',
+        Name.Namespace:         'bold #bb0066',
         Name.Builtin:           '#003388',
         Name.Variable:          '#336699',
         Name.Variable.Class:    '#336699',
diff --git a/pygments/styles/perldoc.py b/pygments/styles/perldoc.py
index 300d8d25..8cd21a93 100644
--- a/pygments/styles/perldoc.py
+++ b/pygments/styles/perldoc.py
@@ -24,6 +24,7 @@ class PerldocStyle(Style):
     styles = {
         Comment:                '#228B22',
         Comment.Preproc:        '#1e889b',
+        Comment.Special:        '#8B008B bold',
 
         String:                 '#CD5555',
         String.Heredoc:         '#1c7e71 italic',
diff --git a/pygments/styles/trac.py b/pygments/styles/trac.py
index 158b1f82..df28045a 100644
--- a/pygments/styles/trac.py
+++ b/pygments/styles/trac.py
@@ -21,6 +21,7 @@ class TracStyle(Style):
     styles = {
         Comment:                'italic #999988',
         Comment.Preproc:        'bold noitalic #999999',
+        Comment.Special:        'bold #999999',
 
         Operator:               'bold',
 
@@ -37,8 +38,8 @@ class TracStyle(Style):
         Name.Class:             'bold #445588',
         Name.Exception:         'bold #990000',
         Name.Namespace:         '#555555',
-        Name.Variable:          '#ff99ff',
-        Name.Constant:          '#ff99ff',
+        Name.Variable:          '#008080',
+        Name.Constant:          '#008080',
         Name.Tag:               '#000080',
         Name.Attribute:         '#008080',
         Name.Entity:            '#800080',
diff --git a/pygments/token.py b/pygments/token.py
index f9ab7442..64c67be4 100644
--- a/pygments/token.py
+++ b/pygments/token.py
@@ -21,6 +21,12 @@ class _TokenType(tuple):
         buf.reverse()
         return buf
 
+    def __contains__(self, val):
+        return self is val or (
+            type(val) is self.__class__ and
+            val[:len(self)] == self
+        )
+
     def __getattr__(self, val):
         if not val or not val[0].isupper():
             return tuple.__getattr__(self, val)
diff --git a/pygments/util.py b/pygments/util.py
index d66eabd6..87ee817a 100644
--- a/pygments/util.py
+++ b/pygments/util.py
@@ -37,8 +37,8 @@ def get_bool_opt(options, optname, default=None):
         return False
     else:
         raise OptionError('Invalid value %r for option %s; use '
-                          '1/0, yes/no, true/false, on/off' %
-                          string, optname)
+                          '1/0, yes/no, true/false, on/off' % (
+                          string, optname))
 
 
 def get_int_opt(options, optname, default=None):
@@ -47,8 +47,8 @@ def get_int_opt(options, optname, default=None):
         return int(string)
     except ValueError:
         raise OptionError('Invalid value %r for option %s; you '
-                          'must give an integer value' %
-                          string, optname)
+                          'must give an integer value' % (
+                          string, optname))
 
 
 def get_list_opt(options, optname, default=None):
@@ -59,8 +59,8 @@ def get_list_opt(options, optname, default=None):
         return list(val)
     else:
         raise OptionError('Invalid value %r for option %s; you '
-                          'must give a list value' %
-                          val, optname)
+                          'must give a list value' % (
+                          val, optname))
 
 
 def make_analysator(f):