From 70caebd582ffbd61ff0066911d80fb6afd960bdd Mon Sep 17 00:00:00 2001 From: murphy Date: Tue, 14 Apr 2009 04:03:40 +0000 Subject: New: *Python Scanner* (feature #41) * Based on pygment's implementation. * The goal is to highlight both Python 2 and 3. ** print and exec will need special treatment. * Still needs work; see TODOs in the code. * Example files: PLEAC, pygments. ** The pygments.unistring code has very long lines which bother TextMate, so I put this one in a separate file. * Rating: 1 / 5 stars, "Poor support", because of problems with numeric literals. * Added .py, .pyw and .py3 to FileType. Else: * Sorted lines in FileType::TypeFromExt. --- test/scanners/python/pygments.in.py | 25595 ++++++++++++++++++++++++++++++++++ 1 file changed, 25595 insertions(+) create mode 100644 test/scanners/python/pygments.in.py (limited to 'test/scanners/python/pygments.in.py') diff --git a/test/scanners/python/pygments.in.py b/test/scanners/python/pygments.in.py new file mode 100644 index 0000000..5953b4d --- /dev/null +++ b/test/scanners/python/pygments.in.py @@ -0,0 +1,25595 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + Generate Pygments Documentation + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Generates a bunch of html files containing the documentation. + + :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import os +import sys +from datetime import datetime +from cgi import escape + +from docutils import nodes +from docutils.parsers.rst import directives +from docutils.core import publish_parts +from docutils.writers import html4css1 + +from jinja import from_string + +from pygments import highlight +from pygments.lexers import get_lexer_by_name +from pygments.formatters import HtmlFormatter + + +LEXERDOC = ''' +`%s` +%s + :Short names: %s + :Filename patterns: %s + :Mimetypes: %s + +''' + +def generate_lexer_docs(): + from pygments.lexers import LEXERS + + out = [] + + modules = {} + moduledocstrings = {} + for classname, data in sorted(LEXERS.iteritems(), key=lambda x: x[0]): + module = data[0] + mod = __import__(module, None, None, [classname]) + cls = getattr(mod, classname) + if not cls.__doc__: + print "Warning: %s does not have a docstring." % classname + modules.setdefault(module, []).append(( + classname, + cls.__doc__, + ', '.join(data[2]) or 'None', + ', '.join(data[3]).replace('*', '\\*') or 'None', + ', '.join(data[4]) or 'None')) + if module not in moduledocstrings: + moduledocstrings[module] = mod.__doc__ + + for module, lexers in sorted(modules.iteritems(), key=lambda x: x[0]): + heading = moduledocstrings[module].splitlines()[4].strip().rstrip('.') + out.append('\n' + heading + '\n' + '-'*len(heading) + '\n') + for data in lexers: + out.append(LEXERDOC % data) + return ''.join(out) + +def generate_formatter_docs(): + from pygments.formatters import FORMATTERS + + out = [] + for cls, data in sorted(FORMATTERS.iteritems(), + key=lambda x: x[0].__name__): + heading = cls.__name__ + out.append('`' + heading + '`\n' + '-'*(2+len(heading)) + '\n') + out.append(cls.__doc__) + out.append(''' + :Short names: %s + :Filename patterns: %s + + +''' % (', '.join(data[1]) or 'None', ', '.join(data[2]).replace('*', '\\*') or 'None')) + return ''.join(out) + +def generate_filter_docs(): + from pygments.filters import FILTERS + + out = [] + for name, cls in FILTERS.iteritems(): + out.append(''' +`%s` +%s + :Name: %s +''' % (cls.__name__, cls.__doc__, name)) + return ''.join(out) + +def generate_changelog(): + fn = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', + 'CHANGES')) + f = file(fn) + result = [] + in_header = False + header = True + for line in f: + if header: + if not in_header and line.strip(): + in_header = True + elif in_header and not line.strip(): + header = False + else: + result.append(line.rstrip()) + f.close() + return '\n'.join(result) + +def generate_authors(): + fn = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', + 'AUTHORS')) + f = file(fn) + r = f.read().rstrip() + f.close() + return r + +LEXERDOCS = generate_lexer_docs() +FORMATTERDOCS = generate_formatter_docs() +FILTERDOCS = generate_filter_docs() +CHANGELOG = generate_changelog() +AUTHORS = generate_authors() + + +PYGMENTS_FORMATTER = HtmlFormatter(style='pastie', cssclass='syntax') + +USAGE = '''\ +Usage: %s [ ...] + +Generate either python or html files out of the documentation. + +Mode can either be python or html.\ +''' % sys.argv[0] + +TEMPLATE = '''\ + + + + {{ title }} — Pygments + + + + +
+

Pygments

+

{{ title }}

+ {% if file_id != "index" %} + « Back To Index + {% endif %} + {% if toc %} +
+

Contents

+
    + {% for key, value in toc %} +
  • {{ value }}
    • + {% endfor %} +
+
+ {% endif %} + {{ body }} +
+ + +\ +''' + +STYLESHEET = '''\ +body { + background-color: #f2f2f2; + margin: 0; + padding: 0; + font-family: 'Georgia', serif; + color: #111; +} + +#content { + background-color: white; + padding: 20px; + margin: 20px auto 20px auto; + max-width: 800px; + border: 4px solid #ddd; +} + +h1 { + font-weight: normal; + font-size: 40px; + color: #09839A; +} + +h2 { + font-weight: normal; + font-size: 30px; + color: #C73F00; +} + +h1.heading { + margin: 0 0 30px 0; +} + +h2.subheading { + margin: -30px 0 0 45px; +} + +h3 { + margin-top: 30px; +} + +table.docutils { + border-collapse: collapse; + border: 2px solid #aaa; + margin: 0.5em 1.5em 0.5em 1.5em; +} + +table.docutils td { + padding: 2px; + border: 1px solid #ddd; +} + +p, li, dd, dt, blockquote { + font-size: 15px; + color: #333; +} + +p { + line-height: 150%; + margin-bottom: 0; + margin-top: 10px; +} + +hr { + border-top: 1px solid #ccc; + border-bottom: 0; + border-right: 0; + border-left: 0; + margin-bottom: 10px; + margin-top: 20px; +} + +dl { + margin-left: 10px; +} + +li, dt { + margin-top: 5px; +} + +dt { + font-weight: bold; +} + +th { + text-align: left; +} + +a { + color: #990000; +} + +a:hover { + color: #c73f00; +} + +pre { + background-color: #f9f9f9; + border-top: 1px solid #ccc; + border-bottom: 1px solid #ccc; + padding: 5px; + font-size: 13px; + font-family: Bitstream Vera Sans Mono,monospace; +} + +tt { + font-size: 13px; + font-family: Bitstream Vera Sans Mono,monospace; + color: black; + padding: 1px 2px 1px 2px; + background-color: #f0f0f0; +} + +cite { + /* abusing , it's generated by ReST for `x` */ + font-size: 13px; + font-family: Bitstream Vera Sans Mono,monospace; + font-weight: bold; + font-style: normal; +} + +#backlink { + float: right; + font-size: 11px; + color: #888; +} + +div.toc { + margin: 0 0 10px 0; +} + +div.toc h2 { + font-size: 20px; +} +''' #' + + +def pygments_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + try: + lexer = get_lexer_by_name(arguments[0]) + except ValueError: + # no lexer found + lexer = get_lexer_by_name('text') + parsed = highlight(u'\n'.join(content), lexer, PYGMENTS_FORMATTER) + return [nodes.raw('', parsed, format="html")] +pygments_directive.arguments = (1, 0, 1) +pygments_directive.content = 1 +directives.register_directive('sourcecode', pygments_directive) + + +def create_translator(link_style): + class Translator(html4css1.HTMLTranslator): + def visit_reference(self, node): + refuri = node.get('refuri') + if refuri is not None and '/' not in refuri and refuri.endswith('.txt'): + node['refuri'] = link_style(refuri[:-4]) + html4css1.HTMLTranslator.visit_reference(self, node) + return Translator + + +class DocumentationWriter(html4css1.Writer): + + def __init__(self, link_style): + html4css1.Writer.__init__(self) + self.translator_class = create_translator(link_style) + + def translate(self): + html4css1.Writer.translate(self) + # generate table of contents + contents = self.build_contents(self.document) + contents_doc = self.document.copy() + contents_doc.children = contents + contents_visitor = self.translator_class(contents_doc) + contents_doc.walkabout(contents_visitor) + self.parts['toc'] = self._generated_toc + + def build_contents(self, node, level=0): + sections = [] + i = len(node) - 1 + while i >= 0 and isinstance(node[i], nodes.section): + sections.append(node[i]) + i -= 1 + sections.reverse() + toc = [] + for section in sections: + try: + reference = nodes.reference('', '', refid=section['ids'][0], *section[0]) + except IndexError: + continue + ref_id = reference['refid'] + text = escape(reference.astext().encode('utf-8')) + toc.append((ref_id, text)) + + self._generated_toc = [('#%s' % href, caption) for href, caption in toc] + # no further processing + return [] + + +def generate_documentation(data, link_style): + writer = DocumentationWriter(link_style) + data = data.replace('[builtin_lexer_docs]', LEXERDOCS).\ + replace('[builtin_formatter_docs]', FORMATTERDOCS).\ + replace('[builtin_filter_docs]', FILTERDOCS).\ + replace('[changelog]', CHANGELOG).\ + replace('[authors]', AUTHORS) + parts = publish_parts( + data, + writer=writer, + settings_overrides={ + 'initial_header_level': 3, + 'field_name_limit': 50, + } + ) + return { + 'title': parts['title'].encode('utf-8'), + 'body': parts['body'].encode('utf-8'), + 'toc': parts['toc'] + } + + +def handle_python(filename, fp, dst): + now = datetime.now() + title = os.path.basename(filename)[:-4] + content = fp.read() + def urlize(href): + # create links for the pygments webpage + if href == 'index.txt': + return '/docs/' + else: + return '/docs/%s/' % href + parts = generate_documentation(content, urlize) + result = file(os.path.join(dst, title + '.py'), 'w') + result.write('# -*- coding: utf-8 -*-\n') + result.write('"""\n Pygments Documentation - %s\n' % title) + result.write(' %s\n\n' % ('~' * (24 + len(title)))) + result.write(' Generated on: %s\n"""\n\n' % now) + result.write('import datetime\n') + result.write('DATE = %r\n' % now) + result.write('TITLE = %r\n' % parts['title']) + result.write('TOC = %r\n' % parts['toc']) + result.write('BODY = %r\n' % parts['body']) + result.close() + + +def handle_html(filename, fp, dst): + now = datetime.now() + title = os.path.basename(filename)[:-4] + content = fp.read() + c = generate_documentation(content, (lambda x: './%s.html' % x)) + result = file(os.path.join(dst, title + '.html'), 'w') + c['style'] = STYLESHEET + PYGMENTS_FORMATTER.get_style_defs('.syntax') + c['generation_date'] = now + c['file_id'] = title + t = from_string(TEMPLATE) + result.write(t.render(c).encode('utf-8')) + result.close() + + +def run(handle_file, dst, sources=()): + path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'src')) + if not sources: + sources = [os.path.join(path, fn) for fn in os.listdir(path)] + for fn in sources: + if not os.path.isfile(fn): + continue + print 'Processing %s' % fn + f = open(fn) + try: + handle_file(fn, f, dst) + finally: + f.close() + + +def main(mode, dst='build/', *sources): + try: + handler = { + 'html': handle_html, + 'python': handle_python + }[mode] + except KeyError: + print 'Error: unknown mode "%s"' % mode + sys.exit(1) + run(handler, os.path.realpath(dst), sources) + + +if __name__ == '__main__': + if len(sys.argv) == 1: + print USAGE + else: + main(*sys.argv[1:]) +# -*- coding: utf-8 -*- +""" + The Pygments Markdown Preprocessor + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This fragment is a Markdown_ preprocessor that renders source code + to HTML via Pygments. To use it, invoke Markdown like so:: + + from markdown import Markdown + + md = Markdown() + md.textPreprocessors.insert(0, CodeBlockPreprocessor()) + html = md.convert(someText) + + markdown is then a callable that can be passed to the context of + a template and used in that template, for example. + + This uses CSS classes by default, so use + ``pygmentize -S -f html > pygments.css`` + to create a stylesheet to be added to the website. + + You can then highlight source code in your markdown markup:: + + [sourcecode:lexer] + some code + [/sourcecode] + + .. _Markdown: http://www.freewisdom.org/projects/python-markdown/ + + :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Options +# ~~~~~~~ + +# Set to True if you want inline CSS styles instead of classes +INLINESTYLES = False + + +import re + +from markdown import TextPreprocessor + +from pygments import highlight +from pygments.formatters import HtmlFormatter +from pygments.lexers import get_lexer_by_name, TextLexer + + +class CodeBlockPreprocessor(TextPreprocessor): + + pattern = re.compile( + r'\[sourcecode:(.+?)\](.+?)\[/sourcecode\]', re.S) + + formatter = HtmlFormatter(noclasses=INLINESTYLES) + + def run(self, lines): + def repl(m): + try: + lexer = get_lexer_by_name(m.group(1)) + except ValueError: + lexer = TextLexer() + code = highlight(m.group(2), lexer, self.formatter) + code = code.replace('\n\n', '\n \n').replace('\n', '
') + return '\n\n
%s
\n\n' % code + return self.pattern.sub( + repl, lines)# -*- coding: utf-8 -*- +""" + The Pygments MoinMoin Parser + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This is a MoinMoin parser plugin that renders source code to HTML via + Pygments; you need Pygments 0.7 or newer for this parser to work. + + To use it, set the options below to match your setup and put this file in + the data/plugin/parser subdirectory of your Moin instance, and give it the + name that the parser directive should have. For example, if you name the + file ``code.py``, you can get a highlighted Python code sample with this + Wiki markup:: + + {{{ + #!code python + [...] + }}} + + Additionally, if you set ATTACHMENTS below to True, Pygments will also be + called for all attachments for whose filenames there is no other parser + registered. + + You are responsible for including CSS rules that will map the Pygments CSS + classes to colors. You can output a stylesheet file with `pygmentize`, put + it into the `htdocs` directory of your Moin instance and then include it in + the `stylesheets` configuration option in the Moin config, e.g.:: + + stylesheets = [('screen', '/htdocs/pygments.css')] + + If you do not want to do that and are willing to accept larger HTML + output, you can set the INLINESTYLES option below to True. + + :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Options +# ~~~~~~~ + +# Set to True if you want to highlight attachments, in addition to +# {{{ }}} blocks. +ATTACHMENTS = True + +# Set to True if you want inline CSS styles instead of classes +INLINESTYLES = False + + +import sys + +from pygments import highlight +from pygments.lexers import get_lexer_by_name, get_lexer_for_filename, TextLexer +from pygments.formatters import HtmlFormatter +from pygments.util import ClassNotFound + + +# wrap lines in s so that the Moin-generated line numbers work +class MoinHtmlFormatter(HtmlFormatter): + def wrap(self, source, outfile): + for line in source: + yield 1, '' + line[1] + '' + +htmlformatter = MoinHtmlFormatter(noclasses=INLINESTYLES) +textlexer = TextLexer() +codeid = [0] + + +class Parser: + """ + MoinMoin Pygments parser. + """ + if ATTACHMENTS: + extensions = '*' + else: + extensions = [] + + Dependencies = [] + + def __init__(self, raw, request, **kw): + self.raw = raw + self.req = request + if "format_args" in kw: + # called from a {{{ }}} block + try: + self.lexer = get_lexer_by_name(kw['format_args'].strip()) + except ClassNotFound: + self.lexer = textlexer + return + if "filename" in kw: + # called for an attachment + filename = kw['filename'] + else: + # called for an attachment by an older moin + # HACK: find out the filename by peeking into the execution + # frame which might not always work + try: + frame = sys._getframe(1) + filename = frame.f_locals['filename'] + except: + filename = 'x.txt' + try: + self.lexer = get_lexer_for_filename(filename) + except ClassNotFound: + self.lexer = textlexer + + def format(self, formatter): + codeid[0] += 1 + id = "pygments_%s" % codeid[0] + w = self.req.write + w(formatter.code_area(1, id, start=1, step=1)) + w(formatter.rawHTML(highlight(self.raw, self.lexer, htmlformatter))) + w(formatter.code_area(0, id)) +# -*- coding: utf-8 -*- +""" + The Pygments reStructuredText directive + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This fragment is a Docutils_ 0.4 directive that renders source code + (to HTML only, currently) via Pygments. + + To use it, adjust the options below and copy the code into a module + that you import on initialization. The code then automatically + registers a ``sourcecode`` directive that you can use instead of + normal code blocks like this:: + + .. sourcecode:: python + + My code goes here. + + If you want to have different code styles, e.g. one with line numbers + and one without, add formatters with their names in the VARIANTS dict + below. You can invoke them instead of the DEFAULT one by using a + directive option:: + + .. sourcecode:: python + :linenos: + + My code goes here. + + Look at the `directive documentation`_ to get all the gory details. + + .. _Docutils: http://docutils.sf.net/ + .. _directive documentation: + http://docutils.sourceforge.net/docs/howto/rst-directives.html + + :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Options +# ~~~~~~~ + +# Set to True if you want inline CSS styles instead of classes +INLINESTYLES = False + +from pygments.formatters import HtmlFormatter + +# The default formatter +DEFAULT = HtmlFormatter(noclasses=INLINESTYLES) + +# Add name -> formatter pairs for every variant you want to use +VARIANTS = { + # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True), +} + + +from docutils import nodes +from docutils.parsers.rst import directives + +from pygments import highlight +from pygments.lexers import get_lexer_by_name, TextLexer + +def pygments_directive(name, arguments, options, content, lineno, + content_offset, block_text, state, state_machine): + try: + lexer = get_lexer_by_name(arguments[0]) + except ValueError: + # no lexer found - use the text one instead of an exception + lexer = TextLexer() + # take an arbitrary option if more than one is given + formatter = options and VARIANTS[options.keys()[0]] or DEFAULT + parsed = highlight(u'\n'.join(content), lexer, formatter) + return [nodes.raw('', parsed, format='html')] + +pygments_directive.arguments = (1, 0, 1) +pygments_directive.content = 1 +pygments_directive.options = dict([(key, directives.flag) for key in VARIANTS]) + +directives.register_directive('sourcecode', pygments_directive) +#!python +"""Bootstrap setuptools installation + +If you want to use setuptools in your package's setup.py, just include this +file in the same directory with it, and add this to the top of your setup.py:: + + from ez_setup import use_setuptools + use_setuptools() + +If you want to require a specific version of setuptools, set a download +mirror, or use an alternate download directory, you can do so by supplying +the appropriate options to ``use_setuptools()``. + +This file can also be run as a script to install or upgrade setuptools. +""" +import sys +DEFAULT_VERSION = "0.6c9" +DEFAULT_URL = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3] + +md5_data = { + 'setuptools-0.6b1-py2.3.egg': '8822caf901250d848b996b7f25c6e6ca', + 'setuptools-0.6b1-py2.4.egg': 'b79a8a403e4502fbb85ee3f1941735cb', + 'setuptools-0.6b2-py2.3.egg': '5657759d8a6d8fc44070a9d07272d99b', + 'setuptools-0.6b2-py2.4.egg': '4996a8d169d2be661fa32a6e52e4f82a', + 'setuptools-0.6b3-py2.3.egg': 'bb31c0fc7399a63579975cad9f5a0618', + 'setuptools-0.6b3-py2.4.egg': '38a8c6b3d6ecd22247f179f7da669fac', + 'setuptools-0.6b4-py2.3.egg': '62045a24ed4e1ebc77fe039aa4e6f7e5', + 'setuptools-0.6b4-py2.4.egg': '4cb2a185d228dacffb2d17f103b3b1c4', + 'setuptools-0.6c1-py2.3.egg': 'b3f2b5539d65cb7f74ad79127f1a908c', + 'setuptools-0.6c1-py2.4.egg': 'b45adeda0667d2d2ffe14009364f2a4b', + 'setuptools-0.6c2-py2.3.egg': 'f0064bf6aa2b7d0f3ba0b43f20817c27', + 'setuptools-0.6c2-py2.4.egg': '616192eec35f47e8ea16cd6a122b7277', + 'setuptools-0.6c3-py2.3.egg': 'f181fa125dfe85a259c9cd6f1d7b78fa', + 'setuptools-0.6c3-py2.4.egg': 'e0ed74682c998bfb73bf803a50e7b71e', + 'setuptools-0.6c3-py2.5.egg': 'abef16fdd61955514841c7c6bd98965e', + 'setuptools-0.6c4-py2.3.egg': 'b0b9131acab32022bfac7f44c5d7971f', + 'setuptools-0.6c4-py2.4.egg': '2a1f9656d4fbf3c97bf946c0a124e6e2', + 'setuptools-0.6c4-py2.5.egg': '8f5a052e32cdb9c72bcf4b5526f28afc', + 'setuptools-0.6c5-py2.3.egg': 'ee9fd80965da04f2f3e6b3576e9d8167', + 'setuptools-0.6c5-py2.4.egg': 'afe2adf1c01701ee841761f5bcd8aa64', + 'setuptools-0.6c5-py2.5.egg': 'a8d3f61494ccaa8714dfed37bccd3d5d', + 'setuptools-0.6c6-py2.3.egg': '35686b78116a668847237b69d549ec20', + 'setuptools-0.6c6-py2.4.egg': '3c56af57be3225019260a644430065ab', + 'setuptools-0.6c6-py2.5.egg': 'b2f8a7520709a5b34f80946de5f02f53', + 'setuptools-0.6c7-py2.3.egg': '209fdf9adc3a615e5115b725658e13e2', + 'setuptools-0.6c7-py2.4.egg': '5a8f954807d46a0fb67cf1f26c55a82e', + 'setuptools-0.6c7-py2.5.egg': '45d2ad28f9750e7434111fde831e8372', + 'setuptools-0.6c8-py2.3.egg': '50759d29b349db8cfd807ba8303f1902', + 'setuptools-0.6c8-py2.4.egg': 'cba38d74f7d483c06e9daa6070cce6de', + 'setuptools-0.6c8-py2.5.egg': '1721747ee329dc150590a58b3e1ac95b', + 'setuptools-0.6c9-py2.3.egg': 'a83c4020414807b496e4cfbe08507c03', + 'setuptools-0.6c9-py2.4.egg': '260a2be2e5388d66bdaee06abec6342a', + 'setuptools-0.6c9-py2.5.egg': 'fe67c3e5a17b12c0e7c541b7ea43a8e6', + 'setuptools-0.6c9-py2.6.egg': 'ca37b1ff16fa2ede6e19383e7b59245a', +} + +import sys, os +try: from hashlib import md5 +except ImportError: from md5 import md5 + +def _validate_md5(egg_name, data): + if egg_name in md5_data: + digest = md5(data).hexdigest() + if digest != md5_data[egg_name]: + print >>sys.stderr, ( + "md5 validation of %s failed! (Possible download problem?)" + % egg_name + ) + sys.exit(2) + return data + +def use_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, + download_delay=15 +): + """Automatically find/download setuptools and make it available on sys.path + + `version` should be a valid setuptools version number that is available + as an egg for download under the `download_base` URL (which should end with + a '/'). `to_dir` is the directory where setuptools will be downloaded, if + it is not already available. If `download_delay` is specified, it should + be the number of seconds that will be paused before initiating a download, + should one be required. If an older version of setuptools is installed, + this routine will print a message to ``sys.stderr`` and raise SystemExit in + an attempt to abort the calling script. + """ + was_imported = 'pkg_resources' in sys.modules or 'setuptools' in sys.modules + def do_download(): + egg = download_setuptools(version, download_base, to_dir, download_delay) + sys.path.insert(0, egg) + import setuptools; setuptools.bootstrap_install_from = egg + try: + import pkg_resources + except ImportError: + return do_download() + try: + pkg_resources.require("setuptools>="+version); return + except pkg_resources.VersionConflict, e: + if was_imported: + print >>sys.stderr, ( + "The required version of setuptools (>=%s) is not available, and\n" + "can't be installed while this script is running. Please install\n" + " a more recent version first, using 'easy_install -U setuptools'." + "\n\n(Currently using %r)" + ) % (version, e.args[0]) + sys.exit(2) + else: + del pkg_resources, sys.modules['pkg_resources'] # reload ok + return do_download() + except pkg_resources.DistributionNotFound: + return do_download() + +def download_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, + delay = 15 +): + """Download setuptools from a specified location and return its filename + + `version` should be a valid setuptools version number that is available + as an egg for download under the `download_base` URL (which should end + with a '/'). `to_dir` is the directory where the egg will be downloaded. + `delay` is the number of seconds to pause before an actual download attempt. + """ + import urllib2, shutil + egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3]) + url = download_base + egg_name + saveto = os.path.join(to_dir, egg_name) + src = dst = None + if not os.path.exists(saveto): # Avoid repeated downloads + try: + from distutils import log + if delay: + log.warn(""" +--------------------------------------------------------------------------- +This script requires setuptools version %s to run (even to display +help). I will attempt to download it for you (from +%s), but +you may need to enable firewall access for this script first. +I will start the download in %d seconds. + +(Note: if this machine does not have network access, please obtain the file + + %s + +and place it in this directory before rerunning this script.) +---------------------------------------------------------------------------""", + version, download_base, delay, url + ); from time import sleep; sleep(delay) + log.warn("Downloading %s", url) + src = urllib2.urlopen(url) + # Read/write all in one block, so we don't create a corrupt file + # if the download is interrupted. + data = _validate_md5(egg_name, src.read()) + dst = open(saveto,"wb"); dst.write(data) + finally: + if src: src.close() + if dst: dst.close() + return os.path.realpath(saveto) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +def main(argv, version=DEFAULT_VERSION): + """Install or upgrade setuptools and EasyInstall""" + try: + import setuptools + except ImportError: + egg = None + try: + egg = download_setuptools(version, delay=0) + sys.path.insert(0,egg) + from setuptools.command.easy_install import main + return main(list(argv)+[egg]) # we're done here + finally: + if egg and os.path.exists(egg): + os.unlink(egg) + else: + if setuptools.__version__ == '0.0.1': + print >>sys.stderr, ( + "You have an obsolete version of setuptools installed. Please\n" + "remove it from your system entirely before rerunning this script." + ) + sys.exit(2) + + req = "setuptools>="+version + import pkg_resources + try: + pkg_resources.require(req) + except pkg_resources.VersionConflict: + try: + from setuptools.command.easy_install import main + except ImportError: + from easy_install import main + main(list(argv)+[download_setuptools(delay=0)]) + sys.exit(0) # try to force an exit + else: + if argv: + from setuptools.command.easy_install import main + main(argv) + else: + print "Setuptools version",version,"or greater has been installed." + print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)' + +def update_md5(filenames): + """Update our built-in md5 registry""" + + import re + + for name in filenames: + base = os.path.basename(name) + f = open(name,'rb') + md5_data[base] = md5(f.read()).hexdigest() + f.close() + + data = [" %r: %r,\n" % it for it in md5_data.items()] + data.sort() + repl = "".join(data) + + import inspect + srcfile = inspect.getsourcefile(sys.modules[__name__]) + f = open(srcfile, 'rb'); src = f.read(); f.close() + + match = re.search("\nmd5_data = {\n([^}]+)}", src) + if not match: + print >>sys.stderr, "Internal error!" + sys.exit(2) + + src = src[:match.start(1)] + repl + src[match.end(1):] + f = open(srcfile,'w') + f.write(src) + f.close() + + +if __name__=='__main__': + if len(sys.argv)>2 and sys.argv[1]=='--md5update': + update_md5(sys.argv[2:]) + else: + main(sys.argv[1:]) + + + + + + +# -*- coding: utf-8 -*- +""" + Pygments + ~~~~~~~~ + + Pygments is a syntax highlighting package written in Python. + + It is a generic syntax highlighter for general use in all kinds of software + such as forum systems, wikis or other applications that need to prettify + source code. Highlights are: + + * a wide range of common languages and markup formats is supported + * special attention is paid to details, increasing quality by a fair amount + * support for new languages and formats are added easily + * a number of output formats, presently HTML, LaTeX, RTF, SVG and ANSI sequences + * it is usable as a command-line tool and as a library + * ... and it highlights even Brainfuck! + + The `Pygments tip`_ is installable with ``easy_install Pygments==dev``. + + .. _Pygments tip: http://dev.pocoo.org/hg/pygments-main/archive/tip.tar.gz#egg=Pygments-dev + + :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +__version__ = '1.1' +__docformat__ = 'restructuredtext' + +__all__ = ['lex', 'format', 'highlight'] + + +import sys, os + +from pygments.util import StringIO, BytesIO + + +def lex(code, lexer): + """ + Lex ``code`` with ``lexer`` and return an iterable of tokens. + """ + try: + return lexer.get_tokens(code) + except TypeError, err: + if isinstance(err.args[0], str) and \ + 'unbound method get_tokens' in err.args[0]: + raise TypeError('lex() argument must be a lexer instance, ' + 'not a class') + raise + + +def format(tokens, formatter, outfile=None): + """ + Format a tokenlist ``tokens`` with the formatter ``formatter``. + + If ``outfile`` is given and a valid file object (an object + with a ``write`` method), the result will be written to it, otherwise + it is returned as a string. + """ + try: + if not outfile: + #print formatter, 'using', formatter.encoding + realoutfile = formatter.encoding and BytesIO() or StringIO() + formatter.format(tokens, realoutfile) + return realoutfile.getvalue() + else: + formatter.format(tokens, outfile) + except TypeError, err: + if isinstance(err.args[0], str) and \ + 'unbound method format' in err.args[0]: + raise TypeError('format() argument must be a formatter instance, ' + 'not a class') + raise + + +def highlight(code, lexer, formatter, outfile=None): + """ + Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``. + + If ``outfile`` is given and a valid file object (an object + with a ``write`` method), the result will be written to it, otherwise + it is returned as a string. + """ + return format(lex(code, lexer), formatter, outfile) + + +if __name__ == '__main__': + from pygments.cmdline import main + sys.exit(main(sys.argv)) +# -*- coding: utf-8 -*- +""" + pygments.cmdline + ~~~~~~~~~~~~~~~~ + + Command line interface. + + :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import sys +import getopt +from textwrap import dedent + +from pygments import __version__, highlight +from pygments.util import ClassNotFound, OptionError, docstring_headline +from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \ + find_lexer_class, guess_lexer, TextLexer +from pygments.formatters import get_all_formatters, get_formatter_by_name, \ + get_formatter_for_filename, find_formatter_class, \ + TerminalFormatter # pylint:disable-msg=E0611 +from pygments.filters import get_all_filters, find_filter_class +from pygments.styles import get_all_styles, get_style_by_name + + +USAGE = """\ +Usage: %s [-l | -g] [-F [:]] [-f ] + [-O ] [-P ] [-o ] [] + + %s -S + + +

%(title)s

+ +''' + +DOC_HEADER_EXTERNALCSS = '''\ + + + + + %(title)s + + + + +

%(title)s

+ +''' + +DOC_FOOTER = '''\ + + +''' + + +class HtmlFormatter(Formatter): + r""" + Format tokens as HTML 4 ```` tags within a ``
`` tag, wrapped
+    in a ``
`` tag. The ``
``'s CSS class can be set by the `cssclass`
+    option.
+
+    If the `linenos` option is set to ``"table"``, the ``
`` is
+    additionally wrapped inside a ```` which has one row and two
+    cells: one containing the line numbers and one containing the code.
+    Example:
+
+    .. sourcecode:: html
+
+        

+        

+          
+          
+        

+            
1
+            2

+          		
+            
def foo(bar):
+              pass
+            

+          

+
+    (whitespace added to improve clarity).
+
+    Wrapping can be disabled using the `nowrap` option.
+
+    A list of lines can be specified using the `hl_lines` option to make these
+    lines highlighted (as of Pygments 0.11).
+
+    With the `full` option, a complete HTML 4 document is output, including
+    the style definitions inside a ``
+
+
+
Code tags report for %s

+
+
+%s
+
LineTagWhoDescription

+
+
+'''
+
+    TABLE = '\nFile: %s\n'
+
+    TR = ('%%(lno)d'
+          '%%(tag)s'
+          '%%(who)s%%(what)s')
+
+    f = file(output, 'w')
+    table = '\n'.join(TABLE % fname +
+                      '\n'.join(TR % (no % 2,) % entry
+                                for no, entry in enumerate(store[fname]))
+                      for fname in sorted(store))
+    f.write(HTML % (', '.join(map(abspath, args)), table))
+    f.close()
+
+    print "Report written to %s." % output
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+    Lexing error finder
+    ~~~~~~~~~~~~~~~~~~~
+
+    For the source files given on the command line, display
+    the text where Error tokens are being generated, along
+    with some context.
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys, os
+
+try:
+    import pygments
+except ImportError:
+    # try parent path
+    sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+
+from pygments import highlight
+from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
+from pygments.token import Error
+
+def main(fn):
+    try:
+        lx = get_lexer_for_filename(fn)
+    except ValueError:
+        try:
+            name, rest = fn.split("_", 1)
+            lx = get_lexer_by_name(name)
+        except ValueError:
+            raise AssertionError('no lexer found for file %r' % fn)
+    text = file(fn, 'U').read()
+    text = text.strip('\n') + '\n'
+    text = text.decode('latin1')
+    ntext = []
+    for type, val in lx.get_tokens(text):
+        if type == Error:
+            print "Error parsing", fn
+            print "\n".join(['   ' + repr(x) for x in ntext[-num:]])
+            print `val` + "<<<"
+            return 1
+        ntext.append((type,val))
+    return 0
+
+
+num = 10
+
+if __name__ == "__main__":
+    if sys.argv[1][:2] == '-n':
+        num = int(sys.argv[1][2:])
+        del sys.argv[1]
+    ret = 0
+    for f in sys.argv[1:]:
+        ret += main(f)
+    sys.exit(bool(ret))
+import re
+from pprint import pprint
+
+r_line = re.compile(r"^(syn keyword vimCommand contained|syn keyword vimOption "
+                    r"contained|syn keyword vimAutoEvent contained)\s+(.*)")
+r_item = re.compile(r"(\w+)(?:\[(\w+)\])?")
+
+def getkw(input, output):
+    out = file(output, 'w')
+
+    output_info = {'command': [], 'option': [], 'auto': []}
+    for line in file(input):
+        m = r_line.match(line)
+        if m:
+            # Decide which output gets mapped to d
+            if 'vimCommand' in m.group(1):
+                d = output_info['command']
+            elif 'AutoEvent' in m.group(1):
+                d = output_info['auto']
+            else:
+                d = output_info['option']
+
+            # Extract all the shortened versions
+            for i in r_item.finditer(m.group(2)):
+                d.append((i.group(1), "%s%s" % (i.group(1), i.group(2) or '')))
+            d.sort()
+
+    for a, b in output_info.items():
+        print >>out, '%s=%r' % (a, b)
+
+def is_keyword(w, keywords):
+    for i in range(len(w), 0, -1):
+        if w[:i] in keywords:
+            return signals[w[:i]][:len(w)] == w
+    return False
+
+if __name__ == "__main__":
+    getkw("/usr/share/vim/vim70/syntax/vim.vim", "temp.py")
+#! /usr/bin/env python
+
+# Released to the public domain, by Tim Peters, 03 October 2000.
+# -B option added by Georg Brandl, 2006.
+
+"""reindent [-d][-r][-v] [ path ... ]
+
+-d (--dryrun)  Dry run.  Analyze, but don't make any changes to files.
+-r (--recurse) Recurse.  Search for all .py files in subdirectories too.
+-B (--no-backup)         Don't write .bak backup files.
+-v (--verbose) Verbose.  Print informative msgs; else only names of changed files.
+-h (--help)    Help.     Print this usage information and exit.
+
+Change Python (.py) files to use 4-space indents and no hard tab characters.
+Also trim excess spaces and tabs from ends of lines, and remove empty lines
+at the end of files.  Also ensure the last line ends with a newline.
+
+If no paths are given on the command line, reindent operates as a filter,
+reading a single source file from standard input and writing the transformed
+source to standard output.  In this case, the -d, -r and -v flags are
+ignored.
+
+You can pass one or more file and/or directory paths.  When a directory
+path, all .py files within the directory will be examined, and, if the -r
+option is given, likewise recursively for subdirectories.
+
+If output is not to standard output, reindent overwrites files in place,
+renaming the originals with a .bak extension.  If it finds nothing to
+change, the file is left alone.  If reindent does change a file, the changed
+file is a fixed-point for future runs (i.e., running reindent on the
+resulting .py file won't change it again).
+
+The hard part of reindenting is figuring out what to do with comment
+lines.  So long as the input files get a clean bill of health from
+tabnanny.py, reindent should do a good job.
+"""
+
+__version__ = "1"
+
+import tokenize
+import os
+import sys
+
+verbose = 0
+recurse = 0
+dryrun  = 0
+no_backup = 0
+
+def usage(msg=None):
+    if msg is not None:
+        print >> sys.stderr, msg
+    print >> sys.stderr, __doc__
+
+def errprint(*args):
+    sep = ""
+    for arg in args:
+        sys.stderr.write(sep + str(arg))
+        sep = " "
+    sys.stderr.write("\n")
+
+def main():
+    import getopt
+    global verbose, recurse, dryrun, no_backup
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "drvhB",
+                                   ["dryrun", "recurse", "verbose", "help",
+                                    "no-backup"])
+    except getopt.error, msg:
+        usage(msg)
+        return
+    for o, a in opts:
+        if o in ('-d', '--dryrun'):
+            dryrun += 1
+        elif o in ('-r', '--recurse'):
+            recurse += 1
+        elif o in ('-v', '--verbose'):
+            verbose += 1
+        elif o in ('-B', '--no-backup'):
+            no_backup += 1
+        elif o in ('-h', '--help'):
+            usage()
+            return
+    if not args:
+        r = Reindenter(sys.stdin)
+        r.run()
+        r.write(sys.stdout)
+        return
+    for arg in args:
+        check(arg)
+
+def check(file):
+    if os.path.isdir(file) and not os.path.islink(file):
+        if verbose:
+            print "listing directory", file
+        names = os.listdir(file)
+        for name in names:
+            fullname = os.path.join(file, name)
+            if ((recurse and os.path.isdir(fullname) and
+                 not os.path.islink(fullname))
+                or name.lower().endswith(".py")):
+                check(fullname)
+        return
+
+    if verbose:
+        print "checking", file, "...",
+    try:
+        f = open(file)
+    except IOError, msg:
+        errprint("%s: I/O Error: %s" % (file, str(msg)))
+        return
+
+    r = Reindenter(f)
+    f.close()
+    if r.run():
+        if verbose:
+            print "changed."
+            if dryrun:
+                print "But this is a dry run, so leaving it alone."
+        else:
+            print "reindented", file, (dryrun and "(dry run => not really)" or "")
+        if not dryrun:
+            if not no_backup:
+                bak = file + ".bak"
+                if os.path.exists(bak):
+                    os.remove(bak)
+                os.rename(file, bak)
+                if verbose:
+                    print "renamed", file, "to", bak
+            f = open(file, "w")
+            r.write(f)
+            f.close()
+            if verbose:
+                print "wrote new", file
+    else:
+        if verbose:
+            print "unchanged."
+
+
+class Reindenter:
+
+    def __init__(self, f):
+        self.find_stmt = 1  # next token begins a fresh stmt?
+        self.level = 0      # current indent level
+
+        # Raw file lines.
+        self.raw = f.readlines()
+
+        # File lines, rstripped & tab-expanded.  Dummy at start is so
+        # that we can use tokenize's 1-based line numbering easily.
+        # Note that a line is all-blank iff it's "\n".
+        self.lines = [line.rstrip('\n \t').expandtabs() + "\n"
+                      for line in self.raw]
+        self.lines.insert(0, None)
+        self.index = 1  # index into self.lines of next line
+
+        # List of (lineno, indentlevel) pairs, one for each stmt and
+        # comment line.  indentlevel is -1 for comment lines, as a
+        # signal that tokenize doesn't know what to do about them;
+        # indeed, they're our headache!
+        self.stats = []
+
+    def run(self):
+        tokenize.tokenize(self.getline, self.tokeneater)
+        # Remove trailing empty lines.
+        lines = self.lines
+        while lines and lines[-1] == "\n":
+            lines.pop()
+        # Sentinel.
+        stats = self.stats
+        stats.append((len(lines), 0))
+        # Map count of leading spaces to # we want.
+        have2want = {}
+        # Program after transformation.
+        after = self.after = []
+        # Copy over initial empty lines -- there's nothing to do until
+        # we see a line with *something* on it.
+        i = stats[0][0]
+        after.extend(lines[1:i])
+        for i in range(len(stats)-1):
+            thisstmt, thislevel = stats[i]
+            nextstmt = stats[i+1][0]
+            have = getlspace(lines[thisstmt])
+            want = thislevel * 4
+            if want < 0:
+                # A comment line.
+                if have:
+                    # An indented comment line.  If we saw the same
+                    # indentation before, reuse what it most recently
+                    # mapped to.
+                    want = have2want.get(have, -1)
+                    if want < 0:
+                        # Then it probably belongs to the next real stmt.
+                        for j in xrange(i+1, len(stats)-1):
+                            jline, jlevel = stats[j]
+                            if jlevel >= 0:
+                                if have == getlspace(lines[jline]):
+                                    want = jlevel * 4
+                                break
+                    if want < 0:           # Maybe it's a hanging
+                                           # comment like this one,
+                        # in which case we should shift it like its base
+                        # line got shifted.
+                        for j in xrange(i-1, -1, -1):
+                            jline, jlevel = stats[j]
+                            if jlevel >= 0:
+                                want = have + getlspace(after[jline-1]) - \
+                                       getlspace(lines[jline])
+                                break
+                    if want < 0:
+                        # Still no luck -- leave it alone.
+                        want = have
+                else:
+                    want = 0
+            assert want >= 0
+            have2want[have] = want
+            diff = want - have
+            if diff == 0 or have == 0:
+                after.extend(lines[thisstmt:nextstmt])
+            else:
+                for line in lines[thisstmt:nextstmt]:
+                    if diff > 0:
+                        if line == "\n":
+                            after.append(line)
+                        else:
+                            after.append(" " * diff + line)
+                    else:
+                        remove = min(getlspace(line), -diff)
+                        after.append(line[remove:])
+        return self.raw != self.after
+
+    def write(self, f):
+        f.writelines(self.after)
+
+    # Line-getter for tokenize.
+    def getline(self):
+        if self.index >= len(self.lines):
+            line = ""
+        else:
+            line = self.lines[self.index]
+            self.index += 1
+        return line
+
+    # Line-eater for tokenize.
+    def tokeneater(self, type, token, (sline, scol), end, line,
+                   INDENT=tokenize.INDENT,
+                   DEDENT=tokenize.DEDENT,
+                   NEWLINE=tokenize.NEWLINE,
+                   COMMENT=tokenize.COMMENT,
+                   NL=tokenize.NL):
+
+        if type == NEWLINE:
+            # A program statement, or ENDMARKER, will eventually follow,
+            # after some (possibly empty) run of tokens of the form
+            #     (NL | COMMENT)* (INDENT | DEDENT+)?
+            self.find_stmt = 1
+
+        elif type == INDENT:
+            self.find_stmt = 1
+            self.level += 1
+
+        elif type == DEDENT:
+            self.find_stmt = 1
+            self.level -= 1
+
+        elif type == COMMENT:
+            if self.find_stmt:
+                self.stats.append((sline, -1))
+                # but we're still looking for a new stmt, so leave
+                # find_stmt alone
+
+        elif type == NL:
+            pass
+
+        elif self.find_stmt:
+            # This is the first "real token" following a NEWLINE, so it
+            # must be the first token of the next program statement, or an
+            # ENDMARKER.
+            self.find_stmt = 0
+            if line:   # not endmarker
+                self.stats.append((sline, self.level))
+
+# Count number of leading blanks.
+def getlspace(line):
+    i, n = 0, len(line)
+    while i < n and line[i] == " ":
+        i += 1
+    return i
+
+if __name__ == '__main__':
+    main()
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    Vim Colorscheme Converter
+    ~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    This script converts vim colorscheme files to valid pygments
+    style classes meant for putting into modules.
+
+    :copyright 2006 by Armin Ronacher.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys
+import re
+from os import path
+from cStringIO import StringIO
+
+split_re = re.compile(r'(? 2 and \
+           len(parts[0]) >= 2 and \
+           'highlight'.startswith(parts[0]):
+            token = parts[1].lower()
+            if token not in TOKENS:
+                continue
+            for item in parts[2:]:
+                p = item.split('=', 1)
+                if not len(p) == 2:
+                    continue
+                key, value = p
+                if key in ('ctermfg', 'guifg'):
+                    color = get_vim_color(value)
+                    if color:
+                        set('color', color)
+                elif key in ('ctermbg', 'guibg'):
+                    color = get_vim_color(value)
+                    if color:
+                        set('bgcolor', color)
+                elif key in ('term', 'cterm', 'gui'):
+                    items = value.split(',')
+                    for item in items:
+                        item = item.lower()
+                        if item == 'none':
+                            set('noinherit', True)
+                        elif item == 'bold':
+                            set('bold', True)
+                        elif item == 'underline':
+                            set('underline', True)
+                        elif item == 'italic':
+                            set('italic', True)
+
+    if bg_color is not None and not colors['Normal'].get('bgcolor'):
+        colors['Normal']['bgcolor'] = bg_color
+
+    color_map = {}
+    for token, styles in colors.iteritems():
+        if token in TOKENS:
+            tmp = []
+            if styles.get('noinherit'):
+                tmp.append('noinherit')
+            if 'color' in styles:
+                tmp.append(styles['color'])
+            if 'bgcolor' in styles:
+                tmp.append('bg:' + styles['bgcolor'])
+            if styles.get('bold'):
+                tmp.append('bold')
+            if styles.get('italic'):
+                tmp.append('italic')
+            if styles.get('underline'):
+                tmp.append('underline')
+            tokens = TOKENS[token]
+            if not isinstance(tokens, tuple):
+                tokens = (tokens,)
+            for token in tokens:
+                color_map[token] = ' '.join(tmp)
+
+    default_token = color_map.pop('')
+    return default_token, color_map
+
+
+class StyleWriter(object):
+
+    def __init__(self, code, name):
+        self.code = code
+        self.name = name.lower()
+
+    def write_header(self, out):
+        out.write('# -*- coding: utf-8 -*-\n"""\n')
+        out.write('    %s Colorscheme\n' % self.name.title())
+        out.write('    %s\n\n' % ('~' * (len(self.name) + 12)))
+        out.write('    Converted by %s\n' % SCRIPT_NAME)
+        out.write('"""\nfrom pygments.style import Style\n')
+        out.write('from pygments.token import Token, %s\n\n' % ', '.join(TOKEN_TYPES))
+        out.write('class %sStyle(Style):\n\n' % self.name.title())
+
+    def write(self, out):
+        self.write_header(out)
+        default_token, tokens = find_colors(self.code)
+        tokens = tokens.items()
+        tokens.sort(lambda a, b: cmp(len(a[0]), len(a[1])))
+        bg_color = [x[3:] for x in default_token.split() if x.startswith('bg:')]
+        if bg_color:
+            out.write('    background_color = %r\n' % bg_color[0])
+        out.write('    styles = {\n')
+        out.write('        %-20s%r\n' % ('Token:', default_token))
+        for token, definition in tokens:
+            if definition:
+                out.write('        %-20s%r\n' % (token + ':', definition))
+        out.write('    }')
+
+    def __repr__(self):
+        out = StringIO()
+        self.write_style(out)
+        return out.getvalue()
+
+
+def convert(filename, stream=None):
+    name = path.basename(filename)
+    if name.endswith('.vim'):
+        name = name[:-4]
+    f = file(filename)
+    code = f.read()
+    f.close()
+    writer = StyleWriter(code, name)
+    if stream is not None:
+        out = stream
+    else:
+        out = StringIO()
+    writer.write(out)
+    if stream is None:
+        return out.getvalue()
+
+
+def main():
+    if len(sys.argv) != 2 or sys.argv[1] in ('-h', '--help'):
+        print 'Usage: %s ' % sys.argv[0]
+        return 2
+    if sys.argv[1] in ('-v', '--version'):
+        print '%s %s' % (SCRIPT_NAME, SCRIPT_VERSION)
+        return
+    filename = sys.argv[1]
+    if not (path.exists(filename) and path.isfile(filename)):
+        print 'Error: %s not found' % filename
+        return 1
+    convert(filename, sys.stdout)
+    sys.stdout.write('\n')
+
+
+if __name__ == '__main__':
+    sys.exit(main() or 0)
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    Pygments
+    ~~~~~~~~
+
+    Pygments is a syntax highlighting package written in Python.
+
+    It is a generic syntax highlighter for general use in all kinds of software
+    such as forum systems, wikis or other applications that need to prettify
+    source code. Highlights are:
+
+    * a wide range of common languages and markup formats is supported
+    * special attention is paid to details, increasing quality by a fair amount
+    * support for new languages and formats are added easily
+    * a number of output formats, presently HTML, LaTeX, RTF, SVG and ANSI sequences
+    * it is usable as a command-line tool and as a library
+    * ... and it highlights even Brainfuck!
+
+    The `Pygments tip`_ is installable with ``easy_install Pygments==dev``.
+
+    .. _Pygments tip: http://dev.pocoo.org/hg/pygments-main/archive/tip.tar.gz#egg=Pygments-dev
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+try:
+    from setuptools import setup, find_packages
+except ImportError:
+    from distutils.core import setup
+    def find_packages():
+        return [
+            'pygments',
+            'pygments.lexers',
+            'pygments.formatters',
+            'pygments.styles',
+            'pygments.filters',
+        ]
+
+try:
+    from distutils.command.build_py import build_py_2to3 as build_py
+except ImportError:
+    from distutils.command.build_py import build_py
+
+setup(
+    name = 'Pygments',
+    version = '1.1',
+    url = 'http://pygments.org/',
+    license = 'BSD License',
+    author = 'Georg Brandl',
+    author_email = 'georg@python.org',
+    description = 'Pygments is a syntax highlighting package written in Python.',
+    long_description = __doc__,
+    keywords = 'syntax highlighting',
+    packages = find_packages(),
+    scripts = ['pygmentize'],
+    platforms = 'any',
+    zip_safe = False,
+    include_package_data = True,
+    classifiers = [
+        'License :: OSI Approved :: BSD License',
+        'Intended Audience :: Developers',
+        'Intended Audience :: End Users/Desktop',
+        'Intended Audience :: System Administrators',
+        'Development Status :: 5 - Production/Stable',
+        'Programming Language :: Python',
+        'Operating System :: OS Independent',
+    ],
+    cmdclass = {'build_py': build_py},
+)
+# -*- coding: utf-8 -*-
+"""
+    Pygments unit tests
+    ~~~~~~~~~~~~~~~~~~
+
+    Usage::
+
+        python run.py [testfile ...]
+
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys, os
+import unittest
+
+from os.path import dirname, basename, join, abspath
+
+import pygments
+
+try:
+    import coverage
+except ImportError:
+    coverage = None
+
+testdir = abspath(dirname(__file__))
+
+failed = []
+total_test_count = 0
+error_test_count = 0
+
+
+def err(file, what, exc):
+    print >>sys.stderr, file, 'failed %s:' % what,
+    print >>sys.stderr, exc
+    failed.append(file[:-3])
+
+
+class QuietTestRunner(object):
+    """Customized test runner for relatively quiet output"""
+
+    def __init__(self, testname, stream=sys.stderr):
+        self.testname = testname
+        self.stream = unittest._WritelnDecorator(stream)
+
+    def run(self, test):
+        global total_test_count
+        global error_test_count
+        result = unittest._TextTestResult(self.stream, True, 1)
+        test(result)
+        if not result.wasSuccessful():
+            self.stream.write(' FAIL:')
+            result.printErrors()
+            failed.append(self.testname)
+        else:
+            self.stream.write(' ok\n')
+        total_test_count += result.testsRun
+        error_test_count += len(result.errors) + len(result.failures)
+        return result
+
+
+def run_tests(with_coverage=False):
+    # needed to avoid confusion involving atexit handlers
+    import logging
+
+    if sys.argv[1:]:
+        # test only files given on cmdline
+        files = [entry + '.py' for entry in sys.argv[1:] if entry.startswith('test_')]
+    else:
+        files = [entry for entry in os.listdir(testdir)
+                 if (entry.startswith('test_') and entry.endswith('.py'))]
+        files.sort()
+
+    WIDTH = 85
+
+    print >>sys.stderr, \
+        ('Pygments %s Test Suite running%s, stand by...' %
+         (pygments.__version__,
+          with_coverage and " with coverage analysis" or "")).center(WIDTH)
+    print >>sys.stderr, ('(using Python %s)' % sys.version.split()[0]).center(WIDTH)
+    print >>sys.stderr, '='*WIDTH
+
+    if with_coverage:
+        coverage.erase()
+        coverage.start()
+
+    for testfile in files:
+        globs = {'__file__': join(testdir, testfile)}
+        try:
+            execfile(join(testdir, testfile), globs)
+        except Exception, exc:
+            raise
+            err(testfile, 'execfile', exc)
+            continue
+        sys.stderr.write(testfile[:-3] + ': ')
+        try:
+            runner = QuietTestRunner(testfile[:-3])
+            # make a test suite of all TestCases in the file
+            tests = []
+            for name, thing in globs.iteritems():
+                if name.endswith('Test'):
+                    tests.append((name, unittest.makeSuite(thing)))
+            tests.sort()
+            suite = unittest.TestSuite()
+            suite.addTests([x[1] for x in tests])
+            runner.run(suite)
+        except Exception, exc:
+            err(testfile, 'running test', exc)
+
+    print >>sys.stderr, '='*WIDTH
+    if failed:
+        print >>sys.stderr, '%d of %d tests failed.' % \
+              (error_test_count, total_test_count)
+        print >>sys.stderr, 'Tests failed in:', ', '.join(failed)
+        ret = 1
+    else:
+        if total_test_count == 1:
+            print >>sys.stderr, '1 test happy.'
+        else:
+            print >>sys.stderr, 'All %d tests happy.' % total_test_count
+        ret = 0
+
+    if with_coverage:
+        coverage.stop()
+        modules = [mod for name, mod in sys.modules.iteritems()
+                   if name.startswith('pygments.') and mod]
+        coverage.report(modules)
+
+    return ret
+
+
+if __name__ == '__main__':
+    with_coverage = False
+    if sys.argv[1:2] == ['-C']:
+        with_coverage = bool(coverage)
+        del sys.argv[1]
+    sys.exit(run_tests(with_coverage))
+# -*- coding: utf-8 -*-
+"""
+    Pygments unit tests
+    ~~~~~~~~~~~~~~~~~~
+
+    Usage::
+
+        python run.py [testfile ...]
+
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys, os
+
+if sys.version_info >= (3,):
+    # copy test suite over to "build/lib" and convert it
+    print ('Copying and converting sources to build/lib/test...')
+    from distutils.util import copydir_run_2to3
+    testroot = os.path.dirname(__file__)
+    newroot = os.path.join(testroot, '..', 'build/lib/test')
+    copydir_run_2to3(testroot, newroot)
+    os.chdir(os.path.join(newroot, '..'))
+
+try:
+    import nose
+except ImportError:
+    print ("nose is required to run the test suites")
+    sys.exit(1)
+
+nose.main()
+# coding: utf-8
+"""
+Support for Pygments tests
+"""
+
+import os
+
+
+def location(mod_name):
+    """
+    Return the file and directory that the code for *mod_name* is in.
+    """
+    source = mod_name.endswith("pyc") and mod_name[:-1] or mod_name
+    source = os.path.abspath(source)
+    return source, os.path.dirname(source)
+# -*- coding: utf-8 -*-
+"""
+    Pygments basic API tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import unittest
+import random
+
+from pygments import lexers, formatters, filters, format
+from pygments.token import _TokenType, Text
+from pygments.lexer import RegexLexer
+from pygments.formatters.img import FontNotFound
+from pygments.util import BytesIO, StringIO, bytes, b
+
+import support
+
+TESTFILE, TESTDIR = support.location(__file__)
+
+test_content = [chr(i) for i in xrange(33, 128)] * 5
+random.shuffle(test_content)
+test_content = ''.join(test_content) + '\n'
+
+class LexersTest(unittest.TestCase):
+
+    def test_import_all(self):
+        # instantiate every lexer, to see if the token type defs are correct
+        for x in lexers.LEXERS.keys():
+            c = getattr(lexers, x)()
+
+    def test_lexer_classes(self):
+        a = self.assert_
+        ae = self.assertEquals
+        # test that every lexer class has the correct public API
+        for lexer in lexers._iter_lexerclasses():
+            a(type(lexer.name) is str)
+            for attr in 'aliases', 'filenames', 'alias_filenames', 'mimetypes':
+                a(hasattr(lexer, attr))
+                a(type(getattr(lexer, attr)) is list, "%s: %s attribute wrong" %
+                                                      (lexer, attr))
+            result = lexer.analyse_text("abc")
+            a(isinstance(result, float) and 0.0 <= result <= 1.0)
+
+            inst = lexer(opt1="val1", opt2="val2")
+            if issubclass(lexer, RegexLexer):
+                if not hasattr(lexer, '_tokens'):
+                    # if there's no "_tokens", the lexer has to be one with
+                    # multiple tokendef variants
+                    a(lexer.token_variants)
+                    for variant in lexer.tokens:
+                        a('root' in lexer.tokens[variant])
+                else:
+                    a('root' in lexer._tokens, '%s has no root state' % lexer)
+
+            tokens = list(inst.get_tokens(test_content))
+            txt = ""
+            for token in tokens:
+                a(isinstance(token, tuple))
+                a(isinstance(token[0], _TokenType))
+                if isinstance(token[1], str):
+                    print repr(token[1])
+                a(isinstance(token[1], unicode))
+                txt += token[1]
+            ae(txt, test_content, "%s lexer roundtrip failed: %r != %r" %
+                    (lexer.name, test_content, txt))
+
+    def test_get_lexers(self):
+        a = self.assert_
+        ae = self.assertEquals
+        # test that the lexers functions work
+
+        for func, args in [(lexers.get_lexer_by_name, ("python",)),
+                           (lexers.get_lexer_for_filename, ("test.py",)),
+                           (lexers.get_lexer_for_mimetype, ("text/x-python",)),
+                           (lexers.guess_lexer, ("#!/usr/bin/python -O\nprint",)),
+                           (lexers.guess_lexer_for_filename, ("a.py", "<%= @foo %>"))
+                           ]:
+            x = func(opt="val", *args)
+            a(isinstance(x, lexers.PythonLexer))
+            ae(x.options["opt"], "val")
+
+
+class FiltersTest(unittest.TestCase):
+
+    def test_basic(self):
+        filter_args = {
+            'whitespace': {'spaces': True, 'tabs': True, 'newlines': True},
+            'highlight': {'names': ['isinstance', 'lexers', 'x']},
+        }
+        for x in filters.FILTERS.keys():
+            lx = lexers.PythonLexer()
+            lx.add_filter(x, **filter_args.get(x, {}))
+            text = open(TESTFILE, 'rb').read().decode('utf-8')
+            tokens = list(lx.get_tokens(text))
+            roundtext = ''.join([t[1] for t in tokens])
+            if x not in ('whitespace', 'keywordcase'):
+                # these filters change the text
+                self.assertEquals(roundtext, text,
+                                  "lexer roundtrip with %s filter failed" % x)
+
+    def test_raiseonerror(self):
+        lx = lexers.PythonLexer()
+        lx.add_filter('raiseonerror', excclass=RuntimeError)
+        self.assertRaises(RuntimeError, list, lx.get_tokens('$'))
+
+    def test_whitespace(self):
+        lx = lexers.PythonLexer()
+        lx.add_filter('whitespace', spaces='%')
+        text = open(TESTFILE, 'rb').read().decode('utf-8')
+        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
+        self.failIf(' ' in lxtext)
+
+    def test_keywordcase(self):
+        lx = lexers.PythonLexer()
+        lx.add_filter('keywordcase', case='capitalize')
+        text = open(TESTFILE, 'rb').read().decode('utf-8')
+        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
+        self.assert_('Def' in lxtext and 'Class' in lxtext)
+
+    def test_codetag(self):
+        lx = lexers.PythonLexer()
+        lx.add_filter('codetagify')
+        text = u'# BUG: text'
+        tokens = list(lx.get_tokens(text))
+        self.assertEquals('# ', tokens[0][1])
+        self.assertEquals('BUG', tokens[1][1])
+
+    def test_codetag_boundary(self):
+        # http://dev.pocoo.org/projects/pygments/ticket/368
+        lx = lexers.PythonLexer()
+        lx.add_filter('codetagify')
+        text = u'# DEBUG: text'
+        tokens = list(lx.get_tokens(text))
+        self.assertEquals('# DEBUG: text', tokens[0][1])
+
+
+class FormattersTest(unittest.TestCase):
+
+    def test_public_api(self):
+        a = self.assert_
+        ae = self.assertEquals
+        ts = list(lexers.PythonLexer().get_tokens("def f(): pass"))
+        out = StringIO()
+        # test that every formatter class has the correct public API
+        for formatter, info in formatters.FORMATTERS.iteritems():
+            a(len(info) == 4)
+            a(info[0], "missing formatter name") # name
+            a(info[1], "missing formatter aliases") # aliases
+            a(info[3], "missing formatter docstring") # doc
+
+            if formatter.name == 'Raw tokens':
+                # will not work with Unicode output file
+                continue
+
+            try:
+                inst = formatter(opt1="val1")
+            except (ImportError, FontNotFound):
+                continue
+            try:
+                inst.get_style_defs()
+            except NotImplementedError:
+                # may be raised by formatters for which it doesn't make sense
+                pass
+            inst.format(ts, out)
+
+    def test_encodings(self):
+        from pygments.formatters import HtmlFormatter
+
+        # unicode output
+        fmt = HtmlFormatter()
+        tokens = [(Text, u"ä")]
+        out = format(tokens, fmt)
+        self.assert_(type(out) is unicode)
+        self.assert_(u"ä" in out)
+
+        # encoding option
+        fmt = HtmlFormatter(encoding="latin1")
+        tokens = [(Text, u"ä")]
+        self.assert_(u"ä".encode("latin1") in format(tokens, fmt))
+
+        # encoding and outencoding option
+        fmt = HtmlFormatter(encoding="latin1", outencoding="utf8")
+        tokens = [(Text, u"ä")]
+        self.assert_(u"ä".encode("utf8") in format(tokens, fmt))
+
+    def test_styles(self):
+        from pygments.formatters import HtmlFormatter
+        fmt = HtmlFormatter(style="pastie")
+
+    def test_unicode_handling(self):
+        # test that the formatter supports encoding and Unicode
+        tokens = list(lexers.PythonLexer(encoding='utf-8').
+                      get_tokens("def f(): 'ä'"))
+        for formatter, info in formatters.FORMATTERS.iteritems():
+            try:
+                inst = formatter(encoding=None)
+            except (ImportError, FontNotFound):
+                # some dependency or font not installed
+                continue
+
+            if formatter.name != 'Raw tokens':
+                out = format(tokens, inst)
+                if formatter.unicodeoutput:
+                    self.assert_(type(out) is unicode)
+
+                inst = formatter(encoding='utf-8')
+                out = format(tokens, inst)
+                self.assert_(type(out) is bytes, '%s: %r' % (formatter, out))
+                # Cannot test for encoding, since formatters may have to escape
+                # non-ASCII characters.
+            else:
+                inst = formatter()
+                out = format(tokens, inst)
+                self.assert_(type(out) is bytes, '%s: %r' % (formatter, out))
+
+    def test_get_formatters(self):
+        a = self.assert_
+        ae = self.assertEquals
+        # test that the formatters functions work
+        x = formatters.get_formatter_by_name("html", opt="val")
+        a(isinstance(x, formatters.HtmlFormatter))
+        ae(x.options["opt"], "val")
+
+        x = formatters.get_formatter_for_filename("a.html", opt="val")
+        a(isinstance(x, formatters.HtmlFormatter))
+        ae(x.options["opt"], "val")
+# -*- coding: utf-8 -*-
+"""
+    Basic CLexer Test
+    ~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+import os
+
+from pygments.token import Text, Number
+from pygments.lexers import CLexer
+
+
+class CLexerTest(unittest.TestCase):
+
+    def setUp(self):
+        self.lexer = CLexer()
+
+    def testNumbers(self):
+        code = '42 23.42 23. .42 023 0xdeadbeef 23e+42 42e-23'
+        wanted = []
+        for item in zip([Number.Integer, Number.Float, Number.Float,
+                         Number.Float, Number.Oct, Number.Hex,
+                         Number.Float, Number.Float], code.split()):
+            wanted.append(item)
+            wanted.append((Text, ' '))
+        wanted = [(Text, '')] + wanted[:-1] + [(Text, '\n')]
+        self.assertEqual(list(self.lexer.get_tokens(code)), wanted)
+
+
+if __name__ == '__main__':
+    unittest.main()
+# -*- coding: utf-8 -*-
+"""
+    Command line test
+    ~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+# Test the command line interface
+
+import sys, os
+import unittest
+import StringIO
+
+from pygments import highlight
+from pygments.cmdline import main as cmdline_main
+
+import support
+
+TESTFILE, TESTDIR = support.location(__file__)
+
+
+def run_cmdline(*args):
+    saved_stdout = sys.stdout
+    saved_stderr = sys.stderr
+    new_stdout = sys.stdout = StringIO.StringIO()
+    new_stderr = sys.stderr = StringIO.StringIO()
+    try:
+        ret = cmdline_main(["pygmentize"] + list(args))
+    finally:
+        sys.stdout = saved_stdout
+        sys.stderr = saved_stderr
+    return (ret, new_stdout.getvalue(), new_stderr.getvalue())
+
+
+class CmdLineTest(unittest.TestCase):
+
+    def test_L_opt(self):
+        c, o, e = run_cmdline("-L")
+        self.assertEquals(c, 0)
+        self.assert_("Lexers" in o and "Formatters" in o and
+                     "Filters" in o and "Styles" in o)
+        c, o, e = run_cmdline("-L", "lexer")
+        self.assertEquals(c, 0)
+        self.assert_("Lexers" in o and "Formatters" not in o)
+        c, o, e = run_cmdline("-L", "lexers")
+        self.assertEquals(c, 0)
+
+    def test_O_opt(self):
+        filename = TESTFILE
+        c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar",
+                              "-fhtml", filename)
+        self.assertEquals(c, 0)
+        self.assert_("foo, bar=baz=," in o)
+
+    def test_F_opt(self):
+        filename = TESTFILE
+        c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb,"
+                              "names=TESTFILE filename",
+                              "-fhtml", filename)
+        self.assertEquals(c, 0)
+        self.assert_('_filename '
+                                     'for overriding, thus no lexer found.'
+                                    % fn)
+            try:
+                name, rest = fn.split("_", 1)
+                lx = get_lexer_by_name(name)
+            except ClassNotFound:
+                raise AssertionError('no lexer found for file %r' % fn)
+        yield check_lexer, lx, absfn
+
+def check_lexer(lx, absfn):
+    text = open(absfn, 'rb').read()
+    text = text.strip(b('\n')) + b('\n')
+    try:
+        text = text.decode('utf-8')
+    except UnicodeError:
+        text = text.decode('latin1')
+    ntext = []
+    for type, val in lx.get_tokens(text):
+        ntext.append(val)
+        assert type != Error, 'lexer %s generated error token for %s' % \
+                (lx, absfn)
+    if u''.join(ntext) != text:
+        raise AssertionError('round trip failed for ' + absfn)
+# -*- coding: utf-8 -*-
+"""
+    Pygments HTML formatter tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import re
+import unittest
+import StringIO
+import tempfile
+from os.path import join, dirname, isfile, abspath
+
+from pygments.lexers import PythonLexer
+from pygments.formatters import HtmlFormatter, NullFormatter
+from pygments.formatters.html import escape_html
+
+import support
+
+TESTFILE, TESTDIR = support.location(__file__)
+
+tokensource = list(PythonLexer(encoding='utf-8').get_tokens(open(TESTFILE).read()))
+
+class HtmlFormatterTest(unittest.TestCase):
+    def test_correct_output(self):
+        hfmt = HtmlFormatter(nowrap=True)
+        houtfile = StringIO.StringIO()
+        hfmt.format(tokensource, houtfile)
+
+        nfmt = NullFormatter()
+        noutfile = StringIO.StringIO()
+        nfmt.format(tokensource, noutfile)
+
+        stripped_html = re.sub('<.*?>', '', houtfile.getvalue())
+        escaped_text = escape_html(noutfile.getvalue())
+        self.assertEquals(stripped_html, escaped_text)
+
+    def test_external_css(self):
+        # test correct behavior
+        # CSS should be in /tmp directory
+        fmt1 = HtmlFormatter(full=True, cssfile='fmt1.css', outencoding='utf-8')
+        # CSS should be in TESTDIR (TESTDIR is absolute)
+        fmt2 = HtmlFormatter(full=True, cssfile=join(TESTDIR, 'fmt2.css'),
+                             outencoding='utf-8')
+        tfile = tempfile.NamedTemporaryFile(suffix='.html')
+        fmt1.format(tokensource, tfile)
+        try:
+            fmt2.format(tokensource, tfile)
+            self.assert_(isfile(join(TESTDIR, 'fmt2.css')))
+        except IOError:
+            # test directory not writable
+            pass
+        tfile.close()
+
+        self.assert_(isfile(join(dirname(tfile.name), 'fmt1.css')))
+        os.unlink(join(dirname(tfile.name), 'fmt1.css'))
+        try:
+            os.unlink(join(TESTDIR, 'fmt2.css'))
+        except OSError:
+            pass
+
+    def test_all_options(self):
+        for optdict in [dict(nowrap=True),
+                        dict(linenos=True),
+                        dict(linenos=True, full=True),
+                        dict(linenos=True, full=True, noclasses=True)]:
+
+            outfile = StringIO.StringIO()
+            fmt = HtmlFormatter(**optdict)
+            fmt.format(tokensource, outfile)
+
+    def test_valid_output(self):
+        # test all available wrappers
+        fmt = HtmlFormatter(full=True, linenos=True, noclasses=True,
+                            outencoding='utf-8')
+
+        handle, pathname = tempfile.mkstemp('.html')
+        tfile = os.fdopen(handle, 'w+b')
+        fmt.format(tokensource, tfile)
+        tfile.close()
+        catname = os.path.join(TESTDIR, 'dtds', 'HTML4.soc')
+        try:
+            try:
+                import subprocess
+                ret = subprocess.Popen(['nsgmls', '-s', '-c', catname, pathname],
+                                       stdout=subprocess.PIPE).wait()
+            except ImportError:
+                # Python 2.3 - no subprocess module
+                ret = os.popen('nsgmls -s -c "%s" "%s"' % (catname, pathname)).close()
+                if ret == 32512: raise OSError  # not found
+        except OSError:
+            # nsgmls not available
+            pass
+        else:
+            self.failIf(ret, 'nsgmls run reported errors')
+
+        os.unlink(pathname)
+
+    def test_get_style_defs(self):
+        fmt = HtmlFormatter()
+        sd = fmt.get_style_defs()
+        self.assert_(sd.startswith('.'))
+
+        fmt = HtmlFormatter(cssclass='foo')
+        sd = fmt.get_style_defs()
+        self.assert_(sd.startswith('.foo'))
+        sd = fmt.get_style_defs('.bar')
+        self.assert_(sd.startswith('.bar'))
+        sd = fmt.get_style_defs(['.bar', '.baz'])
+        fl = sd.splitlines()[0]
+        self.assert_('.bar' in fl and '.baz' in fl)
+
+    def test_unicode_options(self):
+        fmt = HtmlFormatter(title=u'Föö',
+                            cssclass=u'bär',
+                            cssstyles=u'div:before { content: \'bäz\' }',
+                            encoding='utf-8')
+        handle, pathname = tempfile.mkstemp('.html')
+        tfile = os.fdopen(handle, 'w+b')
+        fmt.format(tokensource, tfile)
+        tfile.close()
+# -*- coding: utf-8 -*-
+"""
+    Pygments LaTeX formatter tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import unittest
+import tempfile
+
+from pygments.formatters import LatexFormatter
+from pygments.lexers import PythonLexer
+
+import support
+
+TESTFILE, TESTDIR = support.location(__file__)
+
+
+class LatexFormatterTest(unittest.TestCase):
+
+    def test_valid_output(self):
+        tokensource = list(PythonLexer().get_tokens(open(TESTFILE).read()))
+        fmt = LatexFormatter(full=True, encoding='latin1')
+
+        handle, pathname = tempfile.mkstemp('.tex')
+        # place all output files in /tmp too
+        old_wd = os.getcwd()
+        os.chdir(os.path.dirname(pathname))
+        tfile = os.fdopen(handle, 'wb')
+        fmt.format(tokensource, tfile)
+        tfile.close()
+        try:
+            try:
+                import subprocess
+                ret = subprocess.Popen(['latex', '-interaction=nonstopmode',
+                                        pathname],
+                                       stdout=subprocess.PIPE).wait()
+            except ImportError:
+                # Python 2.3 - no subprocess module
+                ret = os.popen('latex -interaction=nonstopmode "%s"'
+                               % pathname).close()
+                if ret == 32512: raise OSError  # not found
+        except OSError:
+            # latex not available
+            pass
+        else:
+            self.failIf(ret, 'latex run reported errors')
+
+        os.unlink(pathname)
+        os.chdir(old_wd)
+# -*- coding: utf-8 -*-
+"""
+    Pygments regex lexer tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments.token import Text
+from pygments.lexer import RegexLexer
+
+class TestLexer(RegexLexer):
+    """Test tuple state transitions including #pop."""
+    tokens = {
+        'root': [
+            ('a', Text.Root, 'rag'),
+            ('e', Text.Root),
+        ],
+        'beer': [
+            ('d', Text.Beer, ('#pop', '#pop')),
+        ],
+        'rag': [
+            ('b', Text.Rag, '#push'),
+            ('c', Text.Rag, ('#pop', 'beer')),
+        ],
+    }
+
+class TupleTransTest(unittest.TestCase):
+    def test(self):
+        lx = TestLexer()
+        toks = list(lx.get_tokens_unprocessed('abcde'))
+        self.assertEquals(toks,
+           [(0, Text.Root, 'a'), (1, Text.Rag, 'b'), (2, Text.Rag, 'c'),
+            (3, Text.Beer, 'd'), (4, Text.Root, 'e')])
+# -*- coding: utf-8 -*-
+"""
+    Test suite for the token module
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+import StringIO
+import sys
+
+from pygments import token
+
+
+class TokenTest(unittest.TestCase):
+
+    def test_tokentype(self):
+        e = self.assertEquals
+        r = self.assertRaises
+
+        t = token.String
+
+        e(t.split(), [token.Token, token.Literal, token.String])
+
+        e(t.__class__, token._TokenType)
+
+    def test_functions(self):
+        self.assert_(token.is_token_subtype(token.String, token.String))
+        self.assert_(token.is_token_subtype(token.String, token.Literal))
+        self.failIf(token.is_token_subtype(token.Literal, token.String))
+
+        self.assert_(token.string_to_tokentype(token.String) is token.String)
+        self.assert_(token.string_to_tokentype('') is token.Token)
+        self.assert_(token.string_to_tokentype('String') is token.String)
+
+    def test_sanity_check(self):
+        stp = token.STANDARD_TYPES.copy()
+        stp[token.Token] = '---' # Token and Text do conflict, that is okay
+        t = {}
+        for k, v in stp.iteritems():
+            t.setdefault(v, []).append(k)
+        if len(t) == len(stp):
+            return # Okay
+
+        for k, v in t.iteritems():
+            if len(v) > 1:
+                self.fail("%r has more than one key: %r" % (k, v))
+
+
+if __name__ == '__main__':
+    unittest.main()
+import unittest
+from pygments.lexer import using, bygroups, this, RegexLexer
+from pygments.token import String, Text, Keyword
+
+class TestLexer(RegexLexer):
+    tokens = {
+        'root': [
+            (r'#.*', using(this, state='invalid')),
+            (r'(")(.+?)(")', bygroups(String, using(this, state='string'), String)),
+            (r'[^"]+', Text),
+        ],
+        'string': [
+            (r'.+', Keyword),
+        ],
+    }
+
+class UsingStateTest(unittest.TestCase):
+    def test_basic(self):
+        expected = [(Text, 'a'), (String, '"'), (Keyword, 'bcd'),
+                    (String, '"'), (Text, 'e\n')]
+        t = list(TestLexer().get_tokens('a"bcd"e'))
+        self.assertEquals(t, expected)
+    def test_error(self):
+        def gen():
+            x = list(TestLexer().get_tokens('#a'))
+        #XXX: should probably raise a more specific exception if the state
+        #     doesn't exist.
+        self.assertRaises(Exception, gen)
+
+if __name__ == "__main__":
+    unittest.main()
+# -*- coding: utf-8 -*-
+"""
+    Test suite for the util module
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+import os
+
+from pygments import util
+
+
+class UtilTest(unittest.TestCase):
+
+    def test_getoptions(self):
+        raises = self.assertRaises
+        equals = self.assertEquals
+
+        equals(util.get_bool_opt({}, 'a', True), True)
+        equals(util.get_bool_opt({}, 'a', 1), True)
+        equals(util.get_bool_opt({}, 'a', 'true'), True)
+        equals(util.get_bool_opt({}, 'a', 'no'), False)
+        raises(util.OptionError, util.get_bool_opt, {}, 'a', [])
+        raises(util.OptionError, util.get_bool_opt, {}, 'a', 'foo')
+
+        equals(util.get_int_opt({}, 'a', 1), 1)
+        raises(util.OptionError, util.get_int_opt, {}, 'a', [])
+        raises(util.OptionError, util.get_int_opt, {}, 'a', 'bar')
+
+        equals(util.get_list_opt({}, 'a', [1]), [1])
+        equals(util.get_list_opt({}, 'a', '1 2'), ['1', '2'])
+        raises(util.OptionError, util.get_list_opt, {}, 'a', 1)
+
+
+    def test_docstring_headline(self):
+        def f1():
+            """
+            docstring headline
+
+            other text
+            """
+        def f2():
+            """
+            docstring
+            headline
+
+            other text
+            """
+
+        self.assertEquals(util.docstring_headline(f1), "docstring headline")
+        self.assertEquals(util.docstring_headline(f2), "docstring headline")
+
+    def test_analysator(self):
+        class X(object):
+            def analyse(text):
+                return 0.5
+            analyse = util.make_analysator(analyse)
+        self.assertEquals(X.analyse(''), 0.5)
+
+    def test_shebang_matches(self):
+        self.assert_(util.shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?'))
+        self.assert_(util.shebang_matches('#!/usr/bin/python2.4', r'python(2\.\d)?'))
+        self.assert_(util.shebang_matches('#!/usr/bin/startsomethingwith python',
+                                          r'python(2\.\d)?'))
+        self.assert_(util.shebang_matches('#!C:\\Python2.4\\Python.exe',
+                                          r'python(2\.\d)?'))
+
+        self.failIf(util.shebang_matches('#!/usr/bin/python-ruby', r'python(2\.\d)?'))
+        self.failIf(util.shebang_matches('#!/usr/bin/python/ruby', r'python(2\.\d)?'))
+        self.failIf(util.shebang_matches('#!', r'python'))
+
+    def test_doctype_matches(self):
+        self.assert_(util.doctype_matches(' ',
+                                          'html.*'))
+        self.failIf(util.doctype_matches('  ',
+                                         'html.*'))
+        self.assert_(util.html_doctype_matches(
+            ''))
+
+    def test_xml(self):
+        self.assert_(util.looks_like_xml(
+            ''))
+        self.assert_(util.looks_like_xml('abc'))
+        self.failIf(util.looks_like_xml(''))
+
+if __name__ == '__main__':
+    unittest.main()
-- 
cgit v1.2.1