[svn] Add whitespace filter. Add a few more filter unit tests.

17 files changed, 218 insertions, 80 deletions

diff --git a/pygments/filters/__init__.py b/pygments/filters/__init__.py
index 4b4bd28d..3df10a4f 100644
--- a/pygments/filters/__init__.py
+++ b/pygments/filters/__init__.py
@@ -15,10 +15,11 @@ except NameError:
     from sets import Set as set
 
 import re
-from pygments.token import String, Comment, Keyword, Name, Error, \
+from pygments.token import String, Comment, Keyword, Name, Error, Whitespace, \
     string_to_tokentype
 from pygments.filter import Filter
-from pygments.util import get_list_opt, ClassNotFound
+from pygments.util import get_list_opt, get_int_opt, get_bool_opt, \
+     ClassNotFound, OptionError
 from pygments.plugin import find_plugin_filters
 
 
@@ -56,14 +57,30 @@ def get_all_filters():
         yield name
 
 
+def _replace_special(ttype, value, regex, specialttype,
+                     replacefunc=lambda x: x):
+    last = 0
+    for match in regex.finditer(value):
+        start, end = match.start(), match.end()
+        if start != last:
+            yield ttype, value[last:start]
+        yield specialttype, replacefunc(value[start:end])
+        last = end
+    if last != len(value):
+        yield ttype, value[last:]
+
+
 class CodeTagFilter(Filter):
     """
     Highlight special code tags in comments and docstrings.
 
-    Per default, the list of highlighted tags is ``XXX``, ``TODO``, ``BUG`` and
-    ``NOTE``. You can override this list by specifying a `codetags` parameter
-    that takes a list of words.
+    Options accepted:
+
+    `codetags` : list of strings
+       A list of strings that are flagged as code tags.  The default is to
+       highlight ``XXX``, ``TODO``, ``BUG`` and ``NOTE``.
     """
+
     def __init__(self, **options):
         Filter.__init__(self, **options)
         tags = get_list_opt(options, 'codetags',
@@ -73,39 +90,38 @@ class CodeTagFilter(Filter):
         ]))
 
     def filter(self, lexer, stream):
+        regex = self.tag_re
         for ttype, value in stream:
             if ttype in String.Doc or \
                ttype in Comment and \
                ttype not in Comment.Preproc:
-                last = 0
-                for match in self.tag_re.finditer(value):
-                    start = match.start()
-                    end = match.end()
-                    if start != last:
-                        yield ttype, value[last:start]
-                    yield Comment.Special, value[start:end]
-                    last = end
-                if last != len(value):
-                    yield ttype, value[last:]
-                continue
-            yield ttype, value
+                for sttype, svalue in _replace_special(ttype, value, regex,
+                                                       Comment.Special):
+                    yield sttype, svalue
+            else:
+                yield ttype, value
 
 
 class KeywordCaseFilter(Filter):
     """
-    Convert keywords to ``lower``, ``upper`` or ``capitalize`` which means
-    first letter uppercase, rest lowercase.
+    Convert keywords to lowercase or uppercase or capitalize them, which
+    means first letter uppercase, rest lowercase.
 
     This can be useful e.g. if you highlight Pascal code and want to adapt the
-    code to your styleguide. The default is ``lower``, override that by
-    providing the `case` parameter.
+    code to your styleguide.
+
+    Options accepted:
+
+    `case` : string
+       The casing to convert keywords to. Must be one of ``'lower'``,
+       ``'upper'`` or ``'capitalize'``.  The default is ``'lower'``.
     """
 
     def __init__(self, **options):
         Filter.__init__(self, **options)
         case = options.get('case', 'lower')
         if case not in ('lower', 'upper', 'capitalize'):
-            raise TypeError('unknown conversion method %r' % case)
+            raise OptionError('unknown conversion method %r' % case)
         self.convert = getattr(unicode, case)
 
     def filter(self, lexer, stream):
@@ -129,6 +145,16 @@ class NameHighlightFilter(Filter):
 
     This would highlight the names "foo", "bar" and "baz"
     as functions. `Name.Function` is the default token type.
+
+    Options accepted:
+
+    `names` : list of strings
+      A list of names that should be given the different token type.
+      There is no default.
+    `tokentype` : TokenType or string
+      A token type or a string containing a token type name that is
+      used for highlighting the strings in `names`.  The default is
+      `Name.Function`.
     """
 
     def __init__(self, **options):
@@ -153,22 +179,115 @@ class ErrorToken(Exception):
 
 class RaiseOnErrorTokenFilter(Filter):
     """
-    Raise an ``pygments.filters.ErrorToken`` exception when the lexer
-    generates an error token.
+    Raise an exception when the lexer generates an error token.
+
+    Options accepted:
+
+    `excclass` : Exception class
+      The exception class to raise.
+      The default is `pygments.filters.ErrorToken`.
 
     *New in Pygments 0.8.*
     """
 
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        self.exception = options.get('excclass', ErrorToken)
+        try:
+            # issubclass() will raise TypeError if first argument is not a class
+            if not issubclass(self.exception, Exception):
+                raise TypeError
+        except TypeError:
+            raise OptionError('excclass option is not an exception class')
+
     def filter(self, lexer, stream):
         for ttype, value in stream:
             if ttype is Error:
-                raise ErrorToken(value)
+                raise self.exception(value)
             yield ttype, value
 
 
+class VisibleWhitespaceFilter(Filter):
+    """
+    Convert tabs, newlines and/or spaces to visible characters.
+
+    Options accepted:
+
+    `spaces` : string or bool
+      If this is a one-character string, spaces will be replaces by this string.
+      If it is another true value, spaces will be replaced by ``·`` (unicode
+      MIDDLE DOT).  If it is a false value, spaces will not be replaced.  The
+      default is ``False``.
+    `tabs` : string or bool
+      The same as for `spaces`, but the default replacement character is ``»``
+      (unicode RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK).  The default value
+      is ``False``.  Note: this will not work if the `tabsize` option for the
+      lexer is nonzero, as tabs will already have been expanded then.
+    `tabsize` : int
+      If tabs are to be replaced by this filter, this is the total number of
+      characters that a tab should be expanded to.  The default is ``8``.
+    `newlines` : string or bool
+      The same as for `spaces`, but the default replacement character is ``¶``
+      (unicode PILCROW SIGN).  The default value is ``False``.
+    `wstokentype` : bool
+      If true, give whitespace the special `Whitespace` token type.  This allows
+      styling the visible whitespace differently (e.g. greyed out), but it can
+      disrupt background colors.  The default is ``True``
+
+    *New in Pygments 0.8.*
+    """
+
+    def __init__(self, **options):
+        Filter.__init__(self, **options)
+        for name, default in {'spaces': u'·', 'tabs': u'»', 'newlines': u'¶'}.items():
+            opt = options.get(name, False)
+            if isinstance(opt, basestring) and len(opt) == 1:
+                setattr(self, name, opt)
+            else:
+                setattr(self, name, (opt and default or ''))
+        tabsize = get_int_opt(options, 'tabsize', 8)
+        if self.tabs:
+            self.tabs += ' '*(tabsize-1)
+        if self.newlines:
+            self.newlines += '\n'
+        self.wstt = get_bool_opt(options, 'wstokentype', True)
+
+    def filter(self, lexer, stream):
+        if self.wstt:
+            spaces = self.spaces or ' '
+            tabs = self.tabs or '\t'
+            newlines = self.newlines or '\n'
+            regex = re.compile(r'\s')
+            def replacefunc(wschar):
+                if wschar == ' ':
+                    return spaces
+                elif wschar == '\t':
+                    return tabs
+                elif wschar == '\n':
+                    return newlines
+                return wschar
+
+            for ttype, value in stream:
+                for sttype, svalue in _replace_special(ttype, value, regex,
+                                                       Whitespace, replacefunc):
+                    yield sttype, svalue
+        else:
+            spaces, tabs, newlines = self.spaces, self.tabs, self.newlines
+            # simpler processing
+            for ttype, value in stream:
+                if spaces:
+                    value = value.replace(' ', spaces)
+                if tabs:
+                    value = value.replace('\t', tabs)
+                if newlines:
+                    value = value.replace('\n', newlines)
+                yield ttype, value
+
+
 FILTERS = {
     'codetagify':     CodeTagFilter,
     'keywordcase':    KeywordCaseFilter,
     'highlight':      NameHighlightFilter,
     'raiseonerror':   RaiseOnErrorTokenFilter,
+    'whitespace':     VisibleWhitespaceFilter,
 }
diff --git a/pygments/formatters/terminal.py b/pygments/formatters/terminal.py
index cad37c7d..adac90df 100644
--- a/pygments/formatters/terminal.py
+++ b/pygments/formatters/terminal.py
@@ -11,7 +11,7 @@
 
 from pygments.formatter import Formatter
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic, Token
+     Number, Operator, Generic, Token, Whitespace
 from pygments.console import ansiformat
 
 
@@ -23,6 +23,7 @@ __all__ = ['TerminalFormatter']
 TERMINAL_COLORS = {
     Token:              ('',            ''),
 
+    Whitespace:         ('lightgray',   'darkgray'),
     Comment:            ('lightgray',   'darkgray'),
     Keyword:            ('darkblue',    'blue'),
     Keyword.Type:       ('teal',        'turquoise'),
diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py
index e1e442e2..a248b476 100644
--- a/pygments/lexers/functional.py
+++ b/pygments/lexers/functional.py
@@ -16,9 +16,9 @@ try:
 except NameError:
     from sets import Set as set
 
-from pygments.lexer import Lexer, RegexLexer
-from pygments.token import Error, Text, \
-     Comment, Operator, Keyword, Name, String, Number, Generic, Punctuation
+from pygments.lexer import RegexLexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, \
+     String, Number, Punctuation
 
 
 __all__ = ['SchemeLexer', 'HaskellLexer']
diff --git a/pygments/styles/autumn.py b/pygments/styles/autumn.py
index 4508db17..b0862e41 100644
--- a/pygments/styles/autumn.py
+++ b/pygments/styles/autumn.py
@@ -11,7 +11,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class AutumnStyle(Style):
@@ -22,42 +22,44 @@ class AutumnStyle(Style):
     default_style = ""
 
     styles = {
-        Comment:                    "italic #aaaaaa",
-        Comment.Preproc:            "noitalic #4c8317",
-        Comment.Special:            "italic #0000aa",
-
-        Keyword:                    "#0000aa",
-        Keyword.Type:               "#00aaaa",
-
-        Operator.Word:              "#0000aa",
-
-        Name.Builtin:               "#00aaaa",
-        Name.Function:              "#00aa00",
-        Name.Class:                 "underline #00aa00",
-        Name.Namespace:             "underline #00aaaa",
-        Name.Variable:              "#aa0000",
-        Name.Constant:              "#aa0000",
-        Name.Entity:                "bold #800",
-        Name.Attribute:             "#1e90ff",
-        Name.Tag:                   "bold #1e90ff",
-        Name.Decorator:             "#888888",
-
-        String:                     "#aa5500",
-        String.Symbol:              "#0000aa",
-        String.Regex:               "#009999",
-
-        Number:                     "#009999",
-
-        Generic.Heading:            "bold #000080",
-        Generic.Subheading:         "bold #800080",
-        Generic.Deleted:            "#aa0000",
-        Generic.Inserted:           "#00aa00",
-        Generic.Error:              "#aa0000",
-        Generic.Emph:               "italic",
-        Generic.Strong:             "bold",
-        Generic.Prompt:             "#555555",
-        Generic.Output:             "#888888",
-        Generic.Traceback:          "#aa0000",
-
-        Error:                      "#F00 bg:#FAA"
+        Whitespace:                 '#bbbbbb',
+
+        Comment:                    'italic #aaaaaa',
+        Comment.Preproc:            'noitalic #4c8317',
+        Comment.Special:            'italic #0000aa',
+
+        Keyword:                    '#0000aa',
+        Keyword.Type:               '#00aaaa',
+
+        Operator.Word:              '#0000aa',
+
+        Name.Builtin:               '#00aaaa',
+        Name.Function:              '#00aa00',
+        Name.Class:                 'underline #00aa00',
+        Name.Namespace:             'underline #00aaaa',
+        Name.Variable:              '#aa0000',
+        Name.Constant:              '#aa0000',
+        Name.Entity:                'bold #800',
+        Name.Attribute:             '#1e90ff',
+        Name.Tag:                   'bold #1e90ff',
+        Name.Decorator:             '#888888',
+
+        String:                     '#aa5500',
+        String.Symbol:              '#0000aa',
+        String.Regex:               '#009999',
+
+        Number:                     '#009999',
+
+        Generic.Heading:            'bold #000080',
+        Generic.Subheading:         'bold #800080',
+        Generic.Deleted:            '#aa0000',
+        Generic.Inserted:           '#00aa00',
+        Generic.Error:              '#aa0000',
+        Generic.Emph:               'italic',
+        Generic.Strong:             'bold',
+        Generic.Prompt:             '#555555',
+        Generic.Output:             '#888888',
+        Generic.Traceback:          '#aa0000',
+
+        Error:                      '#F00 bg:#FAA'
     }
diff --git a/pygments/styles/borland.py b/pygments/styles/borland.py
index 696178d3..5763073b 100644
--- a/pygments/styles/borland.py
+++ b/pygments/styles/borland.py
@@ -11,7 +11,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class BorlandStyle(Style):
@@ -22,6 +22,8 @@ class BorlandStyle(Style):
     default_style = ''
 
     styles = {
+        Whitespace:             '#bbbbbb',
+
         Comment:                'italic #008800',
         Comment.Preproc:        'noitalic',
         Comment.Special:        'noitalic bold',
diff --git a/pygments/styles/colorful.py b/pygments/styles/colorful.py
index 16f084e6..83988291 100644
--- a/pygments/styles/colorful.py
+++ b/pygments/styles/colorful.py
@@ -11,7 +11,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class ColorfulStyle(Style):
@@ -22,6 +22,8 @@ class ColorfulStyle(Style):
     default_style = ""
 
     styles = {
+        Whitespace:                "#bbbbbb",
+
         Comment:                   "#888",
         Comment.Preproc:           "#579",
         Comment.Special:           "bold #cc0000",
diff --git a/pygments/styles/default.py b/pygments/styles/default.py
index bfa40099..b66d4c95 100644
--- a/pygments/styles/default.py
+++ b/pygments/styles/default.py
@@ -12,7 +12,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class DefaultStyle(Style):
@@ -24,6 +24,7 @@ class DefaultStyle(Style):
     default_style = ""
 
     styles = {
+        Whitespace:                "#bbbbbb",
         Comment:                   "italic #408080",
         Comment.Preproc:           "noitalic #BC7A00",
 
diff --git a/pygments/styles/emacs.py b/pygments/styles/emacs.py
index cf7b2dc8..c27268b8 100644
--- a/pygments/styles/emacs.py
+++ b/pygments/styles/emacs.py
@@ -11,7 +11,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class EmacsStyle(Style):
@@ -23,6 +23,7 @@ class EmacsStyle(Style):
     default_style = ""
 
     styles = {
+        Whitespace:                "#bbbbbb",
         Comment:                   "italic #008800",
         Comment.Preproc:           "noitalic",
         Comment.Special:           "noitalic bold",
diff --git a/pygments/styles/friendly.py b/pygments/styles/friendly.py
index b1e95037..95ee3b37 100644
--- a/pygments/styles/friendly.py
+++ b/pygments/styles/friendly.py
@@ -11,7 +11,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class FriendlyStyle(Style):
@@ -23,6 +23,7 @@ class FriendlyStyle(Style):
     default_style = ""
 
     styles = {
+        Whitespace:                "#bbbbbb",
         Comment:                   "italic #60a0b0",
         Comment.Preproc:           "noitalic #007020",
         Comment.Special:           "noitalic bg:#fff0f0",
diff --git a/pygments/styles/fruity.py b/pygments/styles/fruity.py
index ba1f388a..95fbcc3e 100644
--- a/pygments/styles/fruity.py
+++ b/pygments/styles/fruity.py
@@ -11,7 +11,7 @@
 
 from pygments.style import Style
 from pygments.token import Token, Comment, Name, Keyword, \
-    Generic, Number, String
+    Generic, Number, String, Whitespace
 
 class FruityStyle(Style):
     """
@@ -21,6 +21,7 @@ class FruityStyle(Style):
     background_color = '#111111'
 
     styles = {
+        Whitespace:         '#888888',
         Token:              '#ffffff',
         Generic.Output:     '#444444 bg:#222222',
         Keyword:            '#fb660a bold',
diff --git a/pygments/styles/manni.py b/pygments/styles/manni.py
index bdc979e2..c160f873 100644
--- a/pygments/styles/manni.py
+++ b/pygments/styles/manni.py
@@ -14,7 +14,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class ManniStyle(Style):
@@ -25,6 +25,7 @@ class ManniStyle(Style):
     background_color = '#f0f3f3'
 
     styles = {
+        Whitespace:         '#bbbbbb',
         Comment:            'italic #0099FF',
         Comment.Preproc:    'noitalic #009999',
         Comment.Special:    'bold',
diff --git a/pygments/styles/murphy.py b/pygments/styles/murphy.py
index 06cc795b..5152fe68 100644
--- a/pygments/styles/murphy.py
+++ b/pygments/styles/murphy.py
@@ -11,7 +11,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class MurphyStyle(Style):
@@ -22,6 +22,7 @@ class MurphyStyle(Style):
     default_style = ""
 
     styles = {
+        Whitespace:                "#bbbbbb",
         Comment:                   "#666 italic",
         Comment.Preproc:           "#579 noitalic",
         Comment.Special:           "#c00 bold",
diff --git a/pygments/styles/native.py b/pygments/styles/native.py
index c9b290b0..76021a53 100644
--- a/pygments/styles/native.py
+++ b/pygments/styles/native.py
@@ -11,7 +11,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic, Token
+     Number, Operator, Generic, Token, Whitespace
 
 
 class NativeStyle(Style):
@@ -23,6 +23,7 @@ class NativeStyle(Style):
 
     styles = {
         Token:              '#d0d0d0',
+        Whitespace:         '#666666',
 
         Comment:            'italic #999999',
         Comment.Preproc:    'noitalic bold #cd2828',
diff --git a/pygments/styles/pastie.py b/pygments/styles/pastie.py
index 49ab9cee..b0332252 100644
--- a/pygments/styles/pastie.py
+++ b/pygments/styles/pastie.py
@@ -13,7 +13,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class PastieStyle(Style):
@@ -24,6 +24,7 @@ class PastieStyle(Style):
     default_style = ''
 
     styles = {
+        Whitespace:             '#bbbbbb',
         Comment:                '#888888',
         Comment.Preproc:        'bold #cc0000',
         Comment.Special:        'bg:#fff0f0 bold #cc0000',
diff --git a/pygments/styles/perldoc.py b/pygments/styles/perldoc.py
index 6ce06624..91e63074 100644
--- a/pygments/styles/perldoc.py
+++ b/pygments/styles/perldoc.py
@@ -13,7 +13,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class PerldocStyle(Style):
@@ -25,6 +25,7 @@ class PerldocStyle(Style):
     default_style = ''
 
     styles = {
+        Whitespace:             '#bbbbbb',
         Comment:                '#228B22',
         Comment.Preproc:        '#1e889b',
         Comment.Special:        '#8B008B bold',
diff --git a/pygments/styles/trac.py b/pygments/styles/trac.py
index 0e32986f..d6bd4cf1 100644
--- a/pygments/styles/trac.py
+++ b/pygments/styles/trac.py
@@ -11,7 +11,7 @@
 
 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic
+     Number, Operator, Generic, Whitespace
 
 
 class TracStyle(Style):
@@ -22,6 +22,7 @@ class TracStyle(Style):
     default_style = ''
 
     styles = {
+        Whitespace:             '#bbbbbb',
         Comment:                'italic #999988',
         Comment.Preproc:        'bold noitalic #999999',
         Comment.Special:        'bold #999999',
diff --git a/pygments/token.py b/pygments/token.py
index e4b81e55..66183955 100644
--- a/pygments/token.py
+++ b/pygments/token.py
@@ -56,6 +56,7 @@ Token       = _TokenType()
 
 # Special token types
 Text        = Token.Text
+Whitespace  = Text.Whitespace
 Error       = Token.Error
 # Text that doesn't belong to this lexer (e.g. HTML in PHP)
 Other       = Token.Other
@@ -122,6 +123,7 @@ STANDARD_TYPES = {
     Token:                         '',
 
     Text:                          '',
+    Whitespace:                    'w',
     Error:                         'err',
     Other:                         'x',