[svn] Generate lexer, formatter and filter docs from docstrings.

There is the problem of ordering, though.

7 files changed, 190 insertions, 90 deletions

diff --git a/pygments/formatters/__init__.py b/pygments/formatters/__init__.py
index be20c0ce..8aefe103 100644
--- a/pygments/formatters/__init__.py
+++ b/pygments/formatters/__init__.py
@@ -34,18 +34,18 @@ def _doc_desc(obj):
 FORMATTERS = {
     HtmlFormatter:        ('HTML', ('html',), ('.htm', '.html'),
                            _doc_desc(HtmlFormatter)),
-    TerminalFormatter:    ('Terminal', ('terminal', 'console'), (),
-                           _doc_desc(TerminalFormatter)),
     LatexFormatter:       ('LaTeX', ('latex', 'tex'), ('.tex',),
                            _doc_desc(LatexFormatter)),
     RtfFormatter:         ('RTF', ('rtf',), ('.rtf',),
                            _doc_desc(RtfFormatter)),
+    TerminalFormatter:    ('Terminal', ('terminal', 'console'), (),
+                           _doc_desc(TerminalFormatter)),
+    BBCodeFormatter:      ('BBcode', ('bbcode', 'bb'), (),
+                           _doc_desc(BBCodeFormatter)),
     RawTokenFormatter:    ('Raw tokens', ('raw', 'tokens'), ('.raw',),
                            _doc_desc(RawTokenFormatter)),
     NullFormatter:        ('Text only', ('text', 'null'), ('.txt',),
                            _doc_desc(NullFormatter)),
-    BBCodeFormatter:      ('BBcode', ('bbcode', 'bb'), (),
-                           _doc_desc(BBCodeFormatter))
 }
 
 
diff --git a/pygments/formatters/bbcode.py b/pygments/formatters/bbcode.py
index d9b96485..2f15884a 100644
--- a/pygments/formatters/bbcode.py
+++ b/pygments/formatters/bbcode.py
@@ -18,12 +18,14 @@ __all__ = ['BBCodeFormatter']
 
 class BBCodeFormatter(Formatter):
     """
-    Output BBCode tags with appropiate colors and formatting.
+    Formats tokens with BBcodes. These formatting codes are used by many
+    bulletin boards, so you can highlight your sourcecode with pygments before
+    posting it there.
 
-    This formatter doesn't support background colors and borders, as there are
-    no common BBcodes for that.
+    This formatter has no support for background colors and borders, as there
+    are no common BBcode tags for that.
 
-    Some board systems (e.g. phpBB) don't support markup in their [code] tag,
+    Some board systems (e.g. phpBB) don't support colors in their [code] tag,
     so you can't use the highlighting together with that tag.
     Text in a [code] tag usually is shown with a monospace font (which this
     formatter can do with the ``monofont`` option) and no spaces (which you
@@ -31,12 +33,13 @@ class BBCodeFormatter(Formatter):
 
     Additional options accepted:
 
-    ``codetag``
-        If set to true, put the output into [code] tags (default: false).
+    `codetag`
+        If set to true, put the output into ``[code]`` tags (default:
+        ``false``)
 
-    ``monofont``
+    `monofont`
         If set to true, add a tag to show the code with a monospace font
-        (default: false).
+        (default: ``false``).
     """
 
     def __init__(self, **options):
diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py
index f72046ef..d323c781 100644
--- a/pygments/formatters/html.py
+++ b/pygments/formatters/html.py
@@ -103,43 +103,106 @@ td.linenos { background-color: #f0f0f0; padding-right: 10px; }
 
 class HtmlFormatter(Formatter):
     """
-    Output HTML <span> tags with appropriate classes.
+    Format tokens as HTML 4 ``<span>`` tags within a ``<pre>`` tag, wrapped
+    in a ``<div>`` tag. The ``<div>``'s CSS class can be set by the `cssclass`
+    option.
+
+    If the `linenos` option is given and true, the ``<pre>`` is additionally
+    wrapped inside a ``<table>`` which has one row and two cells: one
+    containing the line numbers and one containing the code. Example:
+
+    .. sourcecode:: html
+
+        <div class="highlight" >
+        <table><tr>
+          <td class="linenos" title="click to toggle"
+            onclick="with (this.firstChild.style)
+                     { display = (display == '') ? 'none' : '' }">
+            <pre>1
+            2</pre>
+          </td>
+          <td class="code">
+            <pre><span class="Ke">def </span><span class="NaFu">foo</span>(bar):
+              <span class="Ke">pass</span>
+            </pre>
+          </td>
+        </tr></table></div>
+
+    (whitespace added to improve clarity). Wrapping can be disabled using the
+    `nowrap` option.
+
+    With the `full` option, a complete HTML 4 document is output, including
+    the style definitions inside a ``<style>`` tag, or in a separate file if
+    the `cssfile` option is given.
+
+    The `get_style_defs(arg='')` method of a `HtmlFormatter` returns a string
+    containing CSS rules for the CSS classes used by the formatter. The
+    argument `arg` can be used to specify additional CSS selectors that
+    are prepended to the classes. A call `fmter.get_style_defs('td .code')`
+    would result in the following CSS classes:
+
+    .. sourcecode:: css
+
+        td .code .kw { font-weight: bold; color: #00FF00 }
+        td .code .cm { color: #999999 }
+        ...
+
+    If you have pygments 0.6 or higher you can also pass a list of tuple to the
+    `get_style_defs` method to request multiple prefixes for the tokens:
+
+    .. sourcecode:: python
+
+        formatter.get_style_defs(['div.syntax pre', 'pre.syntax'])
+
+    The output would then look like this:
+
+    .. sourcecode:: css
+
+        div.syntax pre .kw,
+        pre.syntax .kw { font-weight: bold; color: #00FF00 }
+        div.syntax pre .cm,
+        pre.syntax .cm { color: #999999 }
+        ...
 
     Additional options accepted:
 
-    ``nowrap``
-        If set to true, don't wrap the tokens at all. This disables
-        all other options (default: False).
-    ``noclasses``
-        If set to true, token <span>s will not use CSS classes, but
-        inline styles.
-    ``classprefix``
-        Prefix for token CSS classes, is prepended to all token style
-        classes (e.g. class="o" -> class="_o" if classprefix == '_')
-        (default: '').
-    ``cssclass``
-        CSS class for the wrapping <div> (default: 'highlight').
-    ``cssstyles``
-        Inline CSS styles for the wrapping <div>. (default: '').
-    ``cssfile``
-        If the ``full`` option is ``True`` and this is not ``''``,
-        put the CSS in a separate file whose name is given by this option
-        (default: ''). New in 0.6.
-    ``linenos``
-        If set to ``True``, output line numbers (default: False).
-    ``linenostart``
-        The line number for the first line (default: 1).
-    ``linenostep``
-        If set to a number n > 1, only every nth line number is printed
-        (default: 1).
-    ``linenospecial``
-        If set to a number n > 0, every nth line number is given a special
-        CSS class ``special`` (default: 0).
-    ``nobackground``
-        If set to ``True`` the formatter won't output the background color
-        for the overall element (this automatically defaults to ``False``
-        when there is no overall element [eg: no argument for the
-        `get_syntax_defs` method given]) (default: ``False``). New in 0.6.
+    `nowrap`
+        If set to ``True``, don't wrap the tokens at all, not even in a ``<pre>``
+        tag. This disables all other options (default: ``False``).
+
+    `noclasses`
+        If set to true, token ``<span>`` tags will not use CSS classes, but
+        inline styles. This is not recommended for larger pieces of code since
+        it increases output size by quite a bit (default: ``False``).
+
+    `classprefix`
+        Since the token types use relatively short class names, they may clash
+        with some of your own class names. In this case you can use the
+        `classprefix` option to give a string to prepend to all Pygments-generated
+        CSS class names for token types.
+        Note that this option also affects the output of `get_style_defs()`.
+
+    `cssclass`
+        CSS class for the wrapping ``<div>`` tag (default: ``'highlight'``).
+
+    `cssstyles`
+        Inline CSS styles for the wrapping ``<div>`` tag (default: ``''``).
+
+    `cssfile`
+        If the `full` option is true and this option is given, it must be the
+        name of an external file. The stylesheet is then written to this file
+        instead of the HTML file. *New in Pygments 0.6.*
+
+    `linenospecial`
+        If set to a number n > 0, every nth line number is given the CSS
+        class ``"special"`` (default: ``0``).
+
+    `nobackground`
+        If set to ``True``, the formatter won't output the background color
+        for the wrapping element (this automatically defaults to ``False``
+        when there is no wrapping element [eg: no argument for the
+        `get_syntax_defs` method given]) (default: ``False``). *New in
+        Pygments 0.6.*
     """
 
     def __init__(self, **options):
diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py
index d3d4f0b5..d8bdb8a3 100644
--- a/pygments/formatters/latex.py
+++ b/pygments/formatters/latex.py
@@ -46,32 +46,48 @@ DOC_TEMPLATE = r'''
 
 
 class LatexFormatter(Formatter):
-    """
-    Output LaTeX "color" and "fancyvrb" control sequences.
+    r"""
+    Format tokens as LaTeX code. This needs the `fancyvrb` and `color`
+    standard packages.
+
+    Without the `full` option, code is formatted as one ``Verbatim``
+    environment, like this:
+
+    .. sourcecode:: latex
+
+        \begin{Verbatim}[commandchars=@\[\]]
+        @Can[def ]@Cax[foo](bar):
+            @Can[pass]
+        \end{Verbatim}
+
+    The command sequences used here (``@Can`` etc.) are generated from the given
+    `style` and can be retrieved using the `get_style_defs` method.
+
+    With the `full` option, a complete LaTeX document is output, including
+    the command definitions in the preamble.
+
+    The `get_style_defs(arg='')` method of a `LatexFormatter` returns a string
+    containing ``\newcommand`` commands defining the commands used inside the
+    ``Verbatim`` environments. If the argument `arg` is true,
+    ``\renewcommand`` is used instead.
+    
+    Additional options accepted:
+
+    `docclass`
+        If the `full` option is enabled, this is the document class to use
+        (default: ``'article'``).
+
+    `preamble`
+        If the `full` option is enabled, this can be further preamble commands,
+        e.g. ``\usepackage`` (default: ``''``).
+
+    `verboptions`
+        Additional options given to the Verbatim environment (see the *fancyvrb*
+        docs for possible values) (default: ``''``).
+    
     """
 
     def __init__(self, **options):
-        """
-        Additional options accepted:
-
-        ``docclass``
-            If ``full`` is true, this is the document class to use (default: 'article').
-        ``preamble``
-            If ``full`` is true, this can be further preamble commands (default: '').
-        ``linenos``
-            If true, output line numbers (default: False).
-        ``linenostart``
-            The line number for the first line (default: 1).
-        ``linenostep``
-            If set to a number n > 1, only every nth line number is printed (default: 1).
-        ``verboptions``
-            Additional options given to the Verbatim environment (default: '').
-        ``nobackground``
-            If set to ``True`` the formatter won't output the background color
-            for the overall element (default: ``False``)
-            Note that light colors on dark background with this option disabled
-            won't be readable very good.
-        """
         Formatter.__init__(self, **options)
         self.docclass = options.get('docclass', 'article')
         self.preamble = options.get('preamble', '')
diff --git a/pygments/formatters/other.py b/pygments/formatters/other.py
index f6101848..7ef39b4f 100644
--- a/pygments/formatters/other.py
+++ b/pygments/formatters/other.py
@@ -29,16 +29,18 @@ class NullFormatter(Formatter):
 
 
 class RawTokenFormatter(Formatter):
-    """
-    Output a raw token representation for storing token streams.
+    r"""
+    Formats tokens as a raw representation for storing token streams.
 
-    The format is ``tokentype<TAB>repr(tokenstring)``
+    The format is ``tokentype<TAB>repr(tokenstring)\n``. The output can later
+    be converted to a token stream with the `RawTokenLexer`, described in the
+    `lexer list <lexers.txt>`_.
 
-    Additional options accepted:
+    Only one option is accepted:
 
-    ``compress``
-        If set to "gz" or "bz2", compress the token stream with
-        the given compression algorithm (default: '').
+    `compress`
+        If set to ``'gz'`` or ``'bz2'``, compress the output with the given
+        compression algorithm after encoding (default: ``''``).
     """
 
     unicodeoutput = False
diff --git a/pygments/formatters/rtf.py b/pygments/formatters/rtf.py
index f6764d50..9bfed9d8 100644
--- a/pygments/formatters/rtf.py
+++ b/pygments/formatters/rtf.py
@@ -16,7 +16,19 @@ __all__ = ['RtfFormatter']
 
 
 class RtfFormatter(Formatter):
-    """Output RTF (Rich Text Format)."""
+    """
+    Formats tokens as RTF markup. This formatter automatically outputs full RTF
+    documents with color information and other useful stuff. Perfect for Copy and
+    Paste into Microsoft® Word® documents.
+    
+    *New in Pygments 0.6.*
+
+    Additional options accepted:
+
+    `fontface`
+        The used font famliy, for example ``Bitstream Vera Sans``. Defaults to
+        some generic font which is supposed to have fixed width.
+    """
 
     unicodeoutput = False
 
diff --git a/pygments/formatters/terminal.py b/pygments/formatters/terminal.py
index 4d6debd9..661031e7 100644
--- a/pygments/formatters/terminal.py
+++ b/pygments/formatters/terminal.py
@@ -51,22 +51,26 @@ TERMINAL_COLORS = {
 
 
 class TerminalFormatter(Formatter):
-    """
-    Output plain text with coloring ANSI sequences.
-    """
+    r"""
+    Formats tokens with ANSI color sequences, for output in a text console.
+    Color sequences are terminated at newlines, so that paging the output
+    works correctly.
 
-    def __init__(self, **options):
-        """
-        Accepted options:
+    The `get_style_defs()` method doesn't do anything special since there is
+    no support for common styles.
+
+    Options accepted:
 
-        ``bg``
-            Set to ``'light'`` or ``'dark'`` depending on the
-            terminal's background.
+    `bg`
+        Set to ``"light"`` or ``"dark"`` depending on the terminal's background
+        (default: ``"light"``).
 
-        ``colorscheme``
-            ``None`` or a dictionary mapping token types to
-            ``(lightbg, darkbg)`` color names.
-        """
+    `colorscheme`
+        A dictionary mapping token types to (lightbg, darkbg) color names or
+        ``None`` (default: ``None`` = use builtin colorscheme).
+    """
+
+    def __init__(self, **options):
         Formatter.__init__(self, **options)
         self.darkbg = options.get('bg', 'light') == 'dark'
         self.colorscheme = options.get('colorscheme', None) or TERMINAL_COLORS