#!/usr/bin/env python # -*- coding: utf-8 -*- """ Generate Pygments Documentation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Generates a bunch of html files containing the documentation. :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import os import sys from datetime import datetime from cgi import escape from docutils import nodes from docutils.parsers.rst import directives from docutils.core import publish_parts from docutils.writers import html4css1 from jinja import from_string from pygments import highlight from pygments.lexers import get_lexer_by_name from pygments.formatters import HtmlFormatter LEXERDOC = ''' `%s` %s :Short names: %s :Filename patterns: %s :Mimetypes: %s ''' def generate_lexer_docs(): from pygments.lexers import LEXERS out = [] modules = {} moduledocstrings = {} for classname, data in sorted(LEXERS.iteritems(), key=lambda x: x[0]): module = data[0] mod = __import__(module, None, None, [classname]) cls = getattr(mod, classname) if not cls.__doc__: print "Warning: %s does not have a docstring." % classname modules.setdefault(module, []).append(( classname, cls.__doc__, ', '.join(data[2]) or 'None', ', '.join(data[3]).replace('*', '\\*') or 'None', ', '.join(data[4]) or 'None')) if module not in moduledocstrings: moduledocstrings[module] = mod.__doc__ for module, lexers in sorted(modules.iteritems(), key=lambda x: x[0]): heading = moduledocstrings[module].splitlines()[4].strip().rstrip('.') out.append('\n' + heading + '\n' + '-'*len(heading) + '\n') for data in lexers: out.append(LEXERDOC % data) return ''.join(out) def generate_formatter_docs(): from pygments.formatters import FORMATTERS out = [] for cls, data in sorted(FORMATTERS.iteritems(), key=lambda x: x[0].__name__): heading = cls.__name__ out.append('`' + heading + '`\n' + '-'*(2+len(heading)) + '\n') out.append(cls.__doc__) out.append(''' :Short names: %s :Filename patterns: %s ''' % (', '.join(data[1]) or 'None', ', '.join(data[2]).replace('*', '\\*') or 'None')) return ''.join(out) def generate_filter_docs(): from pygments.filters import FILTERS out = [] for name, cls in FILTERS.iteritems(): out.append(''' `%s` %s :Name: %s ''' % (cls.__name__, cls.__doc__, name)) return ''.join(out) def generate_changelog(): fn = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'CHANGES')) f = file(fn) result = [] in_header = False header = True for line in f: if header: if not in_header and line.strip(): in_header = True elif in_header and not line.strip(): header = False else: result.append(line.rstrip()) f.close() return '\n'.join(result) def generate_authors(): fn = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'AUTHORS')) f = file(fn) r = f.read().rstrip() f.close() return r LEXERDOCS = generate_lexer_docs() FORMATTERDOCS = generate_formatter_docs() FILTERDOCS = generate_filter_docs() CHANGELOG = generate_changelog() AUTHORS = generate_authors() PYGMENTS_FORMATTER = HtmlFormatter(style='pastie', cssclass='syntax') USAGE = '''\ Usage: %s [ ...] Generate either python or html files out of the documentation. Mode can either be python or html.\ ''' % sys.argv[0] TEMPLATE = '''\ {{ title }} — Pygments

Pygments

{{ title }}

{% if file_id != "index" %} « Back To Index {% endif %} {% if toc %}

Contents

{% endif %} {{ body }}
\ ''' STYLESHEET = '''\ body { background-color: #f2f2f2; margin: 0; padding: 0; font-family: 'Georgia', serif; color: #111; } #content { background-color: white; padding: 20px; margin: 20px auto 20px auto; max-width: 800px; border: 4px solid #ddd; } h1 { font-weight: normal; font-size: 40px; color: #09839A; } h2 { font-weight: normal; font-size: 30px; color: #C73F00; } h1.heading { margin: 0 0 30px 0; } h2.subheading { margin: -30px 0 0 45px; } h3 { margin-top: 30px; } table.docutils { border-collapse: collapse; border: 2px solid #aaa; margin: 0.5em 1.5em 0.5em 1.5em; } table.docutils td { padding: 2px; border: 1px solid #ddd; } p, li, dd, dt, blockquote { font-size: 15px; color: #333; } p { line-height: 150%; margin-bottom: 0; margin-top: 10px; } hr { border-top: 1px solid #ccc; border-bottom: 0; border-right: 0; border-left: 0; margin-bottom: 10px; margin-top: 20px; } dl { margin-left: 10px; } li, dt { margin-top: 5px; } dt { font-weight: bold; } th { text-align: left; } a { color: #990000; } a:hover { color: #c73f00; } pre { background-color: #f9f9f9; border-top: 1px solid #ccc; border-bottom: 1px solid #ccc; padding: 5px; font-size: 13px; font-family: Bitstream Vera Sans Mono,monospace; } tt { font-size: 13px; font-family: Bitstream Vera Sans Mono,monospace; color: black; padding: 1px 2px 1px 2px; background-color: #f0f0f0; } cite { /* abusing , it's generated by ReST for `x` */ font-size: 13px; font-family: Bitstream Vera Sans Mono,monospace; font-weight: bold; font-style: normal; } #backlink { float: right; font-size: 11px; color: #888; } div.toc { margin: 0 0 10px 0; } div.toc h2 { font-size: 20px; } ''' #' def pygments_directive(name, arguments, options, content, lineno, content_offset, block_text, state, state_machine): try: lexer = get_lexer_by_name(arguments[0]) except ValueError: # no lexer found lexer = get_lexer_by_name('text') parsed = highlight(u'\n'.join(content), lexer, PYGMENTS_FORMATTER) return [nodes.raw('', parsed, format="html")] pygments_directive.arguments = (1, 0, 1) pygments_directive.content = 1 directives.register_directive('sourcecode', pygments_directive) def create_translator(link_style): class Translator(html4css1.HTMLTranslator): def visit_reference(self, node): refuri = node.get('refuri') if refuri is not None and '/' not in refuri and refuri.endswith('.txt'): node['refuri'] = link_style(refuri[:-4]) html4css1.HTMLTranslator.visit_reference(self, node) return Translator class DocumentationWriter(html4css1.Writer): def __init__(self, link_style): html4css1.Writer.__init__(self) self.translator_class = create_translator(link_style) def translate(self): html4css1.Writer.translate(self) # generate table of contents contents = self.build_contents(self.document) contents_doc = self.document.copy() contents_doc.children = contents contents_visitor = self.translator_class(contents_doc) contents_doc.walkabout(contents_visitor) self.parts['toc'] = self._generated_toc def build_contents(self, node, level=0): sections = [] i = len(node) - 1 while i >= 0 and isinstance(node[i], nodes.section): sections.append(node[i]) i -= 1 sections.reverse() toc = [] for section in sections: try: reference = nodes.reference('', '', refid=section['ids'][0], *section[0]) except IndexError: continue ref_id = reference['refid'] text = escape(reference.astext().encode('utf-8')) toc.append((ref_id, text)) self._generated_toc = [('#%s' % href, caption) for href, caption in toc] # no further processing return [] def generate_documentation(data, link_style): writer = DocumentationWriter(link_style) data = data.replace('[builtin_lexer_docs]', LEXERDOCS).\ replace('[builtin_formatter_docs]', FORMATTERDOCS).\ replace('[builtin_filter_docs]', FILTERDOCS).\ replace('[changelog]', CHANGELOG).\ replace('[authors]', AUTHORS) parts = publish_parts( data, writer=writer, settings_overrides={ 'initial_header_level': 3, 'field_name_limit': 50, } ) return { 'title': parts['title'].encode('utf-8'), 'body': parts['body'].encode('utf-8'), 'toc': parts['toc'] } def handle_python(filename, fp, dst): now = datetime.now() title = os.path.basename(filename)[:-4] content = fp.read() def urlize(href): # create links for the pygments webpage if href == 'index.txt': return '/docs/' else: return '/docs/%s/' % href parts = generate_documentation(content, urlize) result = file(os.path.join(dst, title + '.py'), 'w') result.write('# -*- coding: utf-8 -*-\n') result.write('"""\n Pygments Documentation - %s\n' % title) result.write(' %s\n\n' % ('~' * (24 + len(title)))) result.write(' Generated on: %s\n"""\n\n' % now) result.write('import datetime\n') result.write('DATE = %r\n' % now) result.write('TITLE = %r\n' % parts['title']) result.write('TOC = %r\n' % parts['toc']) result.write('BODY = %r\n' % parts['body']) result.close() def handle_html(filename, fp, dst): now = datetime.now() title = os.path.basename(filename)[:-4] content = fp.read() c = generate_documentation(content, (lambda x: './%s.html' % x)) result = file(os.path.join(dst, title + '.html'), 'w') c['style'] = STYLESHEET + PYGMENTS_FORMATTER.get_style_defs('.syntax') c['generation_date'] = now c['file_id'] = title t = from_string(TEMPLATE) result.write(t.render(c).encode('utf-8')) result.close() def run(handle_file, dst, sources=()): path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'src')) if not sources: sources = [os.path.join(path, fn) for fn in os.listdir(path)] for fn in sources: if not os.path.isfile(fn): continue print 'Processing %s' % fn f = open(fn) try: handle_file(fn, f, dst) finally: f.close() def main(mode, dst='build/', *sources): try: handler = { 'html': handle_html, 'python': handle_python }[mode] except KeyError: print 'Error: unknown mode "%s"' % mode sys.exit(1) run(handler, os.path.realpath(dst), sources) if __name__ == '__main__': if len(sys.argv) == 1: print USAGE else: main(*sys.argv[1:]) # -*- coding: utf-8 -*- """ The Pygments Markdown Preprocessor ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This fragment is a Markdown_ preprocessor that renders source code to HTML via Pygments. To use it, invoke Markdown like so:: from markdown import Markdown md = Markdown() md.textPreprocessors.insert(0, CodeBlockPreprocessor()) html = md.convert(someText) markdown is then a callable that can be passed to the context of a template and used in that template, for example. This uses CSS classes by default, so use ``pygmentize -S -f html > pygments.css`` to create a stylesheet to be added to the website. You can then highlight source code in your markdown markup:: [sourcecode:lexer] some code [/sourcecode] .. _Markdown: http://www.freewisdom.org/projects/python-markdown/ :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ # Options # ~~~~~~~ # Set to True if you want inline CSS styles instead of classes INLINESTYLES = False import re from markdown import TextPreprocessor from pygments import highlight from pygments.formatters import HtmlFormatter from pygments.lexers import get_lexer_by_name, TextLexer class CodeBlockPreprocessor(TextPreprocessor): pattern = re.compile( r'\[sourcecode:(.+?)\](.+?)\[/sourcecode\]', re.S) formatter = HtmlFormatter(noclasses=INLINESTYLES) def run(self, lines): def repl(m): try: lexer = get_lexer_by_name(m.group(1)) except ValueError: lexer = TextLexer() code = highlight(m.group(2), lexer, self.formatter) code = code.replace('\n\n', '\n \n').replace('\n', '
') return '\n\n
%s
\n\n' % code return self.pattern.sub( repl, lines)# -*- coding: utf-8 -*- """ The Pygments MoinMoin Parser ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This is a MoinMoin parser plugin that renders source code to HTML via Pygments; you need Pygments 0.7 or newer for this parser to work. To use it, set the options below to match your setup and put this file in the data/plugin/parser subdirectory of your Moin instance, and give it the name that the parser directive should have. For example, if you name the file ``code.py``, you can get a highlighted Python code sample with this Wiki markup:: {{{ #!code python [...] }}} Additionally, if you set ATTACHMENTS below to True, Pygments will also be called for all attachments for whose filenames there is no other parser registered. You are responsible for including CSS rules that will map the Pygments CSS classes to colors. You can output a stylesheet file with `pygmentize`, put it into the `htdocs` directory of your Moin instance and then include it in the `stylesheets` configuration option in the Moin config, e.g.:: stylesheets = [('screen', '/htdocs/pygments.css')] If you do not want to do that and are willing to accept larger HTML output, you can set the INLINESTYLES option below to True. :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ # Options # ~~~~~~~ # Set to True if you want to highlight attachments, in addition to # {{{ }}} blocks. ATTACHMENTS = True # Set to True if you want inline CSS styles instead of classes INLINESTYLES = False import sys from pygments import highlight from pygments.lexers import get_lexer_by_name, get_lexer_for_filename, TextLexer from pygments.formatters import HtmlFormatter from pygments.util import ClassNotFound # wrap lines in s so that the Moin-generated line numbers work class MoinHtmlFormatter(HtmlFormatter): def wrap(self, source, outfile): for line in source: yield 1, '' + line[1] + '' htmlformatter = MoinHtmlFormatter(noclasses=INLINESTYLES) textlexer = TextLexer() codeid = [0] class Parser: """ MoinMoin Pygments parser. """ if ATTACHMENTS: extensions = '*' else: extensions = [] Dependencies = [] def __init__(self, raw, request, **kw): self.raw = raw self.req = request if "format_args" in kw: # called from a {{{ }}} block try: self.lexer = get_lexer_by_name(kw['format_args'].strip()) except ClassNotFound: self.lexer = textlexer return if "filename" in kw: # called for an attachment filename = kw['filename'] else: # called for an attachment by an older moin # HACK: find out the filename by peeking into the execution # frame which might not always work try: frame = sys._getframe(1) filename = frame.f_locals['filename'] except: filename = 'x.txt' try: self.lexer = get_lexer_for_filename(filename) except ClassNotFound: self.lexer = textlexer def format(self, formatter): codeid[0] += 1 id = "pygments_%s" % codeid[0] w = self.req.write w(formatter.code_area(1, id, start=1, step=1)) w(formatter.rawHTML(highlight(self.raw, self.lexer, htmlformatter))) w(formatter.code_area(0, id)) # -*- coding: utf-8 -*- """ The Pygments reStructuredText directive ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This fragment is a Docutils_ 0.4 directive that renders source code (to HTML only, currently) via Pygments. To use it, adjust the options below and copy the code into a module that you import on initialization. The code then automatically registers a ``sourcecode`` directive that you can use instead of normal code blocks like this:: .. sourcecode:: python My code goes here. If you want to have different code styles, e.g. one with line numbers and one without, add formatters with their names in the VARIANTS dict below. You can invoke them instead of the DEFAULT one by using a directive option:: .. sourcecode:: python :linenos: My code goes here. Look at the `directive documentation`_ to get all the gory details. .. _Docutils: http://docutils.sf.net/ .. _directive documentation: http://docutils.sourceforge.net/docs/howto/rst-directives.html :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ # Options # ~~~~~~~ # Set to True if you want inline CSS styles instead of classes INLINESTYLES = False from pygments.formatters import HtmlFormatter # The default formatter DEFAULT = HtmlFormatter(noclasses=INLINESTYLES) # Add name -> formatter pairs for every variant you want to use VARIANTS = { # 'linenos': HtmlFormatter(noclasses=INLINESTYLES, linenos=True), } from docutils import nodes from docutils.parsers.rst import directives from pygments import highlight from pygments.lexers import get_lexer_by_name, TextLexer def pygments_directive(name, arguments, options, content, lineno, content_offset, block_text, state, state_machine): try: lexer = get_lexer_by_name(arguments[0]) except ValueError: # no lexer found - use the text one instead of an exception lexer = TextLexer() # take an arbitrary option if more than one is given formatter = options and VARIANTS[options.keys()[0]] or DEFAULT parsed = highlight(u'\n'.join(content), lexer, formatter) return [nodes.raw('', parsed, format='html')] pygments_directive.arguments = (1, 0, 1) pygments_directive.content = 1 pygments_directive.options = dict([(key, directives.flag) for key in VARIANTS]) directives.register_directive('sourcecode', pygments_directive) #!python """Bootstrap setuptools installation If you want to use setuptools in your package's setup.py, just include this file in the same directory with it, and add this to the top of your setup.py:: from ez_setup import use_setuptools use_setuptools() If you want to require a specific version of setuptools, set a download mirror, or use an alternate download directory, you can do so by supplying the appropriate options to ``use_setuptools()``. This file can also be run as a script to install or upgrade setuptools. """ import sys DEFAULT_VERSION = "0.6c9" DEFAULT_URL = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3] md5_data = { 'setuptools-0.6b1-py2.3.egg': '8822caf901250d848b996b7f25c6e6ca', 'setuptools-0.6b1-py2.4.egg': 'b79a8a403e4502fbb85ee3f1941735cb', 'setuptools-0.6b2-py2.3.egg': '5657759d8a6d8fc44070a9d07272d99b', 'setuptools-0.6b2-py2.4.egg': '4996a8d169d2be661fa32a6e52e4f82a', 'setuptools-0.6b3-py2.3.egg': 'bb31c0fc7399a63579975cad9f5a0618', 'setuptools-0.6b3-py2.4.egg': '38a8c6b3d6ecd22247f179f7da669fac', 'setuptools-0.6b4-py2.3.egg': '62045a24ed4e1ebc77fe039aa4e6f7e5', 'setuptools-0.6b4-py2.4.egg': '4cb2a185d228dacffb2d17f103b3b1c4', 'setuptools-0.6c1-py2.3.egg': 'b3f2b5539d65cb7f74ad79127f1a908c', 'setuptools-0.6c1-py2.4.egg': 'b45adeda0667d2d2ffe14009364f2a4b', 'setuptools-0.6c2-py2.3.egg': 'f0064bf6aa2b7d0f3ba0b43f20817c27', 'setuptools-0.6c2-py2.4.egg': '616192eec35f47e8ea16cd6a122b7277', 'setuptools-0.6c3-py2.3.egg': 'f181fa125dfe85a259c9cd6f1d7b78fa', 'setuptools-0.6c3-py2.4.egg': 'e0ed74682c998bfb73bf803a50e7b71e', 'setuptools-0.6c3-py2.5.egg': 'abef16fdd61955514841c7c6bd98965e', 'setuptools-0.6c4-py2.3.egg': 'b0b9131acab32022bfac7f44c5d7971f', 'setuptools-0.6c4-py2.4.egg': '2a1f9656d4fbf3c97bf946c0a124e6e2', 'setuptools-0.6c4-py2.5.egg': '8f5a052e32cdb9c72bcf4b5526f28afc', 'setuptools-0.6c5-py2.3.egg': 'ee9fd80965da04f2f3e6b3576e9d8167', 'setuptools-0.6c5-py2.4.egg': 'afe2adf1c01701ee841761f5bcd8aa64', 'setuptools-0.6c5-py2.5.egg': 'a8d3f61494ccaa8714dfed37bccd3d5d', 'setuptools-0.6c6-py2.3.egg': '35686b78116a668847237b69d549ec20', 'setuptools-0.6c6-py2.4.egg': '3c56af57be3225019260a644430065ab', 'setuptools-0.6c6-py2.5.egg': 'b2f8a7520709a5b34f80946de5f02f53', 'setuptools-0.6c7-py2.3.egg': '209fdf9adc3a615e5115b725658e13e2', 'setuptools-0.6c7-py2.4.egg': '5a8f954807d46a0fb67cf1f26c55a82e', 'setuptools-0.6c7-py2.5.egg': '45d2ad28f9750e7434111fde831e8372', 'setuptools-0.6c8-py2.3.egg': '50759d29b349db8cfd807ba8303f1902', 'setuptools-0.6c8-py2.4.egg': 'cba38d74f7d483c06e9daa6070cce6de', 'setuptools-0.6c8-py2.5.egg': '1721747ee329dc150590a58b3e1ac95b', 'setuptools-0.6c9-py2.3.egg': 'a83c4020414807b496e4cfbe08507c03', 'setuptools-0.6c9-py2.4.egg': '260a2be2e5388d66bdaee06abec6342a', 'setuptools-0.6c9-py2.5.egg': 'fe67c3e5a17b12c0e7c541b7ea43a8e6', 'setuptools-0.6c9-py2.6.egg': 'ca37b1ff16fa2ede6e19383e7b59245a', } import sys, os try: from hashlib import md5 except ImportError: from md5 import md5 def _validate_md5(egg_name, data): if egg_name in md5_data: digest = md5(data).hexdigest() if digest != md5_data[egg_name]: print >>sys.stderr, ( "md5 validation of %s failed! (Possible download problem?)" % egg_name ) sys.exit(2) return data def use_setuptools( version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, download_delay=15 ): """Automatically find/download setuptools and make it available on sys.path `version` should be a valid setuptools version number that is available as an egg for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where setuptools will be downloaded, if it is not already available. If `download_delay` is specified, it should be the number of seconds that will be paused before initiating a download, should one be required. If an older version of setuptools is installed, this routine will print a message to ``sys.stderr`` and raise SystemExit in an attempt to abort the calling script. """ was_imported = 'pkg_resources' in sys.modules or 'setuptools' in sys.modules def do_download(): egg = download_setuptools(version, download_base, to_dir, download_delay) sys.path.insert(0, egg) import setuptools; setuptools.bootstrap_install_from = egg try: import pkg_resources except ImportError: return do_download() try: pkg_resources.require("setuptools>="+version); return except pkg_resources.VersionConflict, e: if was_imported: print >>sys.stderr, ( "The required version of setuptools (>=%s) is not available, and\n" "can't be installed while this script is running. Please install\n" " a more recent version first, using 'easy_install -U setuptools'." "\n\n(Currently using %r)" ) % (version, e.args[0]) sys.exit(2) else: del pkg_resources, sys.modules['pkg_resources'] # reload ok return do_download() except pkg_resources.DistributionNotFound: return do_download() def download_setuptools( version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, delay = 15 ): """Download setuptools from a specified location and return its filename `version` should be a valid setuptools version number that is available as an egg for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. """ import urllib2, shutil egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3]) url = download_base + egg_name saveto = os.path.join(to_dir, egg_name) src = dst = None if not os.path.exists(saveto): # Avoid repeated downloads try: from distutils import log if delay: log.warn(""" --------------------------------------------------------------------------- This script requires setuptools version %s to run (even to display help). I will attempt to download it for you (from %s), but you may need to enable firewall access for this script first. I will start the download in %d seconds. (Note: if this machine does not have network access, please obtain the file %s and place it in this directory before rerunning this script.) ---------------------------------------------------------------------------""", version, download_base, delay, url ); from time import sleep; sleep(delay) log.warn("Downloading %s", url) src = urllib2.urlopen(url) # Read/write all in one block, so we don't create a corrupt file # if the download is interrupted. data = _validate_md5(egg_name, src.read()) dst = open(saveto,"wb"); dst.write(data) finally: if src: src.close() if dst: dst.close() return os.path.realpath(saveto) def main(argv, version=DEFAULT_VERSION): """Install or upgrade setuptools and EasyInstall""" try: import setuptools except ImportError: egg = None try: egg = download_setuptools(version, delay=0) sys.path.insert(0,egg) from setuptools.command.easy_install import main return main(list(argv)+[egg]) # we're done here finally: if egg and os.path.exists(egg): os.unlink(egg) else: if setuptools.__version__ == '0.0.1': print >>sys.stderr, ( "You have an obsolete version of setuptools installed. Please\n" "remove it from your system entirely before rerunning this script." ) sys.exit(2) req = "setuptools>="+version import pkg_resources try: pkg_resources.require(req) except pkg_resources.VersionConflict: try: from setuptools.command.easy_install import main except ImportError: from easy_install import main main(list(argv)+[download_setuptools(delay=0)]) sys.exit(0) # try to force an exit else: if argv: from setuptools.command.easy_install import main main(argv) else: print "Setuptools version",version,"or greater has been installed." print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)' def update_md5(filenames): """Update our built-in md5 registry""" import re for name in filenames: base = os.path.basename(name) f = open(name,'rb') md5_data[base] = md5(f.read()).hexdigest() f.close() data = [" %r: %r,\n" % it for it in md5_data.items()] data.sort() repl = "".join(data) import inspect srcfile = inspect.getsourcefile(sys.modules[__name__]) f = open(srcfile, 'rb'); src = f.read(); f.close() match = re.search("\nmd5_data = {\n([^}]+)}", src) if not match: print >>sys.stderr, "Internal error!" sys.exit(2) src = src[:match.start(1)] + repl + src[match.end(1):] f = open(srcfile,'w') f.write(src) f.close() if __name__=='__main__': if len(sys.argv)>2 and sys.argv[1]=='--md5update': update_md5(sys.argv[2:]) else: main(sys.argv[1:]) # -*- coding: utf-8 -*- """ Pygments ~~~~~~~~ Pygments is a syntax highlighting package written in Python. It is a generic syntax highlighter for general use in all kinds of software such as forum systems, wikis or other applications that need to prettify source code. Highlights are: * a wide range of common languages and markup formats is supported * special attention is paid to details, increasing quality by a fair amount * support for new languages and formats are added easily * a number of output formats, presently HTML, LaTeX, RTF, SVG and ANSI sequences * it is usable as a command-line tool and as a library * ... and it highlights even Brainfuck! The `Pygments tip`_ is installable with ``easy_install Pygments==dev``. .. _Pygments tip: http://dev.pocoo.org/hg/pygments-main/archive/tip.tar.gz#egg=Pygments-dev :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ __version__ = '1.1' __docformat__ = 'restructuredtext' __all__ = ['lex', 'format', 'highlight'] import sys, os from pygments.util import StringIO, BytesIO def lex(code, lexer): """ Lex ``code`` with ``lexer`` and return an iterable of tokens. """ try: return lexer.get_tokens(code) except TypeError, err: if isinstance(err.args[0], str) and \ 'unbound method get_tokens' in err.args[0]: raise TypeError('lex() argument must be a lexer instance, ' 'not a class') raise def format(tokens, formatter, outfile=None): """ Format a tokenlist ``tokens`` with the formatter ``formatter``. If ``outfile`` is given and a valid file object (an object with a ``write`` method), the result will be written to it, otherwise it is returned as a string. """ try: if not outfile: #print formatter, 'using', formatter.encoding realoutfile = formatter.encoding and BytesIO() or StringIO() formatter.format(tokens, realoutfile) return realoutfile.getvalue() else: formatter.format(tokens, outfile) except TypeError, err: if isinstance(err.args[0], str) and \ 'unbound method format' in err.args[0]: raise TypeError('format() argument must be a formatter instance, ' 'not a class') raise def highlight(code, lexer, formatter, outfile=None): """ Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``. If ``outfile`` is given and a valid file object (an object with a ``write`` method), the result will be written to it, otherwise it is returned as a string. """ return format(lex(code, lexer), formatter, outfile) if __name__ == '__main__': from pygments.cmdline import main sys.exit(main(sys.argv)) # -*- coding: utf-8 -*- """ pygments.cmdline ~~~~~~~~~~~~~~~~ Command line interface. :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import sys import getopt from textwrap import dedent from pygments import __version__, highlight from pygments.util import ClassNotFound, OptionError, docstring_headline from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \ find_lexer_class, guess_lexer, TextLexer from pygments.formatters import get_all_formatters, get_formatter_by_name, \ get_formatter_for_filename, find_formatter_class, \ TerminalFormatter # pylint:disable-msg=E0611 from pygments.filters import get_all_filters, find_filter_class from pygments.styles import get_all_styles, get_style_by_name USAGE = """\ Usage: %s [-l | -g] [-F [:]] [-f ] [-O ] [-P ] [-o ] [] %s -S

%(title)s

''' DOC_HEADER_EXTERNALCSS = '''\ %(title)s

%(title)s

''' DOC_FOOTER = '''\ ''' class HtmlFormatter(Formatter): r""" Format tokens as HTML 4 ```` tags within a ``
`` tag, wrapped
    in a ``
`` tag. The ``
``'s CSS class can be set by the `cssclass`
    option.

    If the `linenos` option is set to ``"table"``, the ``
`` is
    additionally wrapped inside a ```` which has one row and two
    cells: one containing the line numbers and one containing the code.
    Example:

    .. sourcecode:: html

        

        

          
            
1
            2

          		
          
            
def foo(bar):
              pass
            

          		
        


    (whitespace added to improve clarity).

    Wrapping can be disabled using the `nowrap` option.

    A list of lines can be specified using the `hl_lines` option to make these
    lines highlighted (as of Pygments 0.11).

    With the `full` option, a complete HTML 4 document is output, including
    the style definitions inside a ``



Code tags report for %s


%s



LineTagWhoDescription



'''

    TABLE = '\nFile: %s\n'

    TR = ('%%(lno)d'
          '%%(tag)s'
          '%%(who)s%%(what)s')

    f = file(output, 'w')
    table = '\n'.join(TABLE % fname +
                      '\n'.join(TR % (no % 2,) % entry
                                for no, entry in enumerate(store[fname]))
                      for fname in sorted(store))
    f.write(HTML % (', '.join(map(abspath, args)), table))
    f.close()

    print "Report written to %s." % output
    return 0

if __name__ == '__main__':
    sys.exit(main())
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
    Lexing error finder
    ~~~~~~~~~~~~~~~~~~~

    For the source files given on the command line, display
    the text where Error tokens are being generated, along
    with some context.

    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import sys, os

try:
    import pygments
except ImportError:
    # try parent path
    sys.path.append(os.path.join(os.path.dirname(__file__), ".."))

from pygments import highlight
from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
from pygments.token import Error

def main(fn):
    try:
        lx = get_lexer_for_filename(fn)
    except ValueError:
        try:
            name, rest = fn.split("_", 1)
            lx = get_lexer_by_name(name)
        except ValueError:
            raise AssertionError('no lexer found for file %r' % fn)
    text = file(fn, 'U').read()
    text = text.strip('\n') + '\n'
    text = text.decode('latin1')
    ntext = []
    for type, val in lx.get_tokens(text):
        if type == Error:
            print "Error parsing", fn
            print "\n".join(['   ' + repr(x) for x in ntext[-num:]])
            print `val` + "<<<"
            return 1
        ntext.append((type,val))
    return 0


num = 10

if __name__ == "__main__":
    if sys.argv[1][:2] == '-n':
        num = int(sys.argv[1][2:])
        del sys.argv[1]
    ret = 0
    for f in sys.argv[1:]:
        ret += main(f)
    sys.exit(bool(ret))
import re
from pprint import pprint

r_line = re.compile(r"^(syn keyword vimCommand contained|syn keyword vimOption "
                    r"contained|syn keyword vimAutoEvent contained)\s+(.*)")
r_item = re.compile(r"(\w+)(?:\[(\w+)\])?")

def getkw(input, output):
    out = file(output, 'w')

    output_info = {'command': [], 'option': [], 'auto': []}
    for line in file(input):
        m = r_line.match(line)
        if m:
            # Decide which output gets mapped to d
            if 'vimCommand' in m.group(1):
                d = output_info['command']
            elif 'AutoEvent' in m.group(1):
                d = output_info['auto']
            else:
                d = output_info['option']

            # Extract all the shortened versions
            for i in r_item.finditer(m.group(2)):
                d.append((i.group(1), "%s%s" % (i.group(1), i.group(2) or '')))
            d.sort()

    for a, b in output_info.items():
        print >>out, '%s=%r' % (a, b)

def is_keyword(w, keywords):
    for i in range(len(w), 0, -1):
        if w[:i] in keywords:
            return signals[w[:i]][:len(w)] == w
    return False

if __name__ == "__main__":
    getkw("/usr/share/vim/vim70/syntax/vim.vim", "temp.py")
#! /usr/bin/env python

# Released to the public domain, by Tim Peters, 03 October 2000.
# -B option added by Georg Brandl, 2006.

"""reindent [-d][-r][-v] [ path ... ]

-d (--dryrun)  Dry run.  Analyze, but don't make any changes to files.
-r (--recurse) Recurse.  Search for all .py files in subdirectories too.
-B (--no-backup)         Don't write .bak backup files.
-v (--verbose) Verbose.  Print informative msgs; else only names of changed files.
-h (--help)    Help.     Print this usage information and exit.

Change Python (.py) files to use 4-space indents and no hard tab characters.
Also trim excess spaces and tabs from ends of lines, and remove empty lines
at the end of files.  Also ensure the last line ends with a newline.

If no paths are given on the command line, reindent operates as a filter,
reading a single source file from standard input and writing the transformed
source to standard output.  In this case, the -d, -r and -v flags are
ignored.

You can pass one or more file and/or directory paths.  When a directory
path, all .py files within the directory will be examined, and, if the -r
option is given, likewise recursively for subdirectories.

If output is not to standard output, reindent overwrites files in place,
renaming the originals with a .bak extension.  If it finds nothing to
change, the file is left alone.  If reindent does change a file, the changed
file is a fixed-point for future runs (i.e., running reindent on the
resulting .py file won't change it again).

The hard part of reindenting is figuring out what to do with comment
lines.  So long as the input files get a clean bill of health from
tabnanny.py, reindent should do a good job.
"""

__version__ = "1"

import tokenize
import os
import sys

verbose = 0
recurse = 0
dryrun  = 0
no_backup = 0

def usage(msg=None):
    if msg is not None:
        print >> sys.stderr, msg
    print >> sys.stderr, __doc__

def errprint(*args):
    sep = ""
    for arg in args:
        sys.stderr.write(sep + str(arg))
        sep = " "
    sys.stderr.write("\n")

def main():
    import getopt
    global verbose, recurse, dryrun, no_backup

    try:
        opts, args = getopt.getopt(sys.argv[1:], "drvhB",
                                   ["dryrun", "recurse", "verbose", "help",
                                    "no-backup"])
    except getopt.error, msg:
        usage(msg)
        return
    for o, a in opts:
        if o in ('-d', '--dryrun'):
            dryrun += 1
        elif o in ('-r', '--recurse'):
            recurse += 1
        elif o in ('-v', '--verbose'):
            verbose += 1
        elif o in ('-B', '--no-backup'):
            no_backup += 1
        elif o in ('-h', '--help'):
            usage()
            return
    if not args:
        r = Reindenter(sys.stdin)
        r.run()
        r.write(sys.stdout)
        return
    for arg in args:
        check(arg)

def check(file):
    if os.path.isdir(file) and not os.path.islink(file):
        if verbose:
            print "listing directory", file
        names = os.listdir(file)
        for name in names:
            fullname = os.path.join(file, name)
            if ((recurse and os.path.isdir(fullname) and
                 not os.path.islink(fullname))
                or name.lower().endswith(".py")):
                check(fullname)
        return

    if verbose:
        print "checking", file, "...",
    try:
        f = open(file)
    except IOError, msg:
        errprint("%s: I/O Error: %s" % (file, str(msg)))
        return

    r = Reindenter(f)
    f.close()
    if r.run():
        if verbose:
            print "changed."
            if dryrun:
                print "But this is a dry run, so leaving it alone."
        else:
            print "reindented", file, (dryrun and "(dry run => not really)" or "")
        if not dryrun:
            if not no_backup:
                bak = file + ".bak"
                if os.path.exists(bak):
                    os.remove(bak)
                os.rename(file, bak)
                if verbose:
                    print "renamed", file, "to", bak
            f = open(file, "w")
            r.write(f)
            f.close()
            if verbose:
                print "wrote new", file
    else:
        if verbose:
            print "unchanged."


class Reindenter:

    def __init__(self, f):
        self.find_stmt = 1  # next token begins a fresh stmt?
        self.level = 0      # current indent level

        # Raw file lines.
        self.raw = f.readlines()

        # File lines, rstripped & tab-expanded.  Dummy at start is so
        # that we can use tokenize's 1-based line numbering easily.
        # Note that a line is all-blank iff it's "\n".
        self.lines = [line.rstrip('\n \t').expandtabs() + "\n"
                      for line in self.raw]
        self.lines.insert(0, None)
        self.index = 1  # index into self.lines of next line

        # List of (lineno, indentlevel) pairs, one for each stmt and
        # comment line.  indentlevel is -1 for comment lines, as a
        # signal that tokenize doesn't know what to do about them;
        # indeed, they're our headache!
        self.stats = []

    def run(self):
        tokenize.tokenize(self.getline, self.tokeneater)
        # Remove trailing empty lines.
        lines = self.lines
        while lines and lines[-1] == "\n":
            lines.pop()
        # Sentinel.
        stats = self.stats
        stats.append((len(lines), 0))
        # Map count of leading spaces to # we want.
        have2want = {}
        # Program after transformation.
        after = self.after = []
        # Copy over initial empty lines -- there's nothing to do until
        # we see a line with *something* on it.
        i = stats[0][0]
        after.extend(lines[1:i])
        for i in range(len(stats)-1):
            thisstmt, thislevel = stats[i]
            nextstmt = stats[i+1][0]
            have = getlspace(lines[thisstmt])
            want = thislevel * 4
            if want < 0:
                # A comment line.
                if have:
                    # An indented comment line.  If we saw the same
                    # indentation before, reuse what it most recently
                    # mapped to.
                    want = have2want.get(have, -1)
                    if want < 0:
                        # Then it probably belongs to the next real stmt.
                        for j in xrange(i+1, len(stats)-1):
                            jline, jlevel = stats[j]
                            if jlevel >= 0:
                                if have == getlspace(lines[jline]):
                                    want = jlevel * 4
                                break
                    if want < 0:           # Maybe it's a hanging
                                           # comment like this one,
                        # in which case we should shift it like its base
                        # line got shifted.
                        for j in xrange(i-1, -1, -1):
                            jline, jlevel = stats[j]
                            if jlevel >= 0:
                                want = have + getlspace(after[jline-1]) - \
                                       getlspace(lines[jline])
                                break
                    if want < 0:
                        # Still no luck -- leave it alone.
                        want = have
                else:
                    want = 0
            assert want >= 0
            have2want[have] = want
            diff = want - have
            if diff == 0 or have == 0:
                after.extend(lines[thisstmt:nextstmt])
            else:
                for line in lines[thisstmt:nextstmt]:
                    if diff > 0:
                        if line == "\n":
                            after.append(line)
                        else:
                            after.append(" " * diff + line)
                    else:
                        remove = min(getlspace(line), -diff)
                        after.append(line[remove:])
        return self.raw != self.after

    def write(self, f):
        f.writelines(self.after)

    # Line-getter for tokenize.
    def getline(self):
        if self.index >= len(self.lines):
            line = ""
        else:
            line = self.lines[self.index]
            self.index += 1
        return line

    # Line-eater for tokenize.
    def tokeneater(self, type, token, (sline, scol), end, line,
                   INDENT=tokenize.INDENT,
                   DEDENT=tokenize.DEDENT,
                   NEWLINE=tokenize.NEWLINE,
                   COMMENT=tokenize.COMMENT,
                   NL=tokenize.NL):

        if type == NEWLINE:
            # A program statement, or ENDMARKER, will eventually follow,
            # after some (possibly empty) run of tokens of the form
            #     (NL | COMMENT)* (INDENT | DEDENT+)?
            self.find_stmt = 1

        elif type == INDENT:
            self.find_stmt = 1
            self.level += 1

        elif type == DEDENT:
            self.find_stmt = 1
            self.level -= 1

        elif type == COMMENT:
            if self.find_stmt:
                self.stats.append((sline, -1))
                # but we're still looking for a new stmt, so leave
                # find_stmt alone

        elif type == NL:
            pass

        elif self.find_stmt:
            # This is the first "real token" following a NEWLINE, so it
            # must be the first token of the next program statement, or an
            # ENDMARKER.
            self.find_stmt = 0
            if line:   # not endmarker
                self.stats.append((sline, self.level))

# Count number of leading blanks.
def getlspace(line):
    i, n = 0, len(line)
    while i < n and line[i] == " ":
        i += 1
    return i

if __name__ == '__main__':
    main()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    Vim Colorscheme Converter
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    This script converts vim colorscheme files to valid pygments
    style classes meant for putting into modules.

    :copyright 2006 by Armin Ronacher.
    :license: BSD, see LICENSE for details.
"""

import sys
import re
from os import path
from cStringIO import StringIO

split_re = re.compile(r'(? 2 and \
           len(parts[0]) >= 2 and \
           'highlight'.startswith(parts[0]):
            token = parts[1].lower()
            if token not in TOKENS:
                continue
            for item in parts[2:]:
                p = item.split('=', 1)
                if not len(p) == 2:
                    continue
                key, value = p
                if key in ('ctermfg', 'guifg'):
                    color = get_vim_color(value)
                    if color:
                        set('color', color)
                elif key in ('ctermbg', 'guibg'):
                    color = get_vim_color(value)
                    if color:
                        set('bgcolor', color)
                elif key in ('term', 'cterm', 'gui'):
                    items = value.split(',')
                    for item in items:
                        item = item.lower()
                        if item == 'none':
                            set('noinherit', True)
                        elif item == 'bold':
                            set('bold', True)
                        elif item == 'underline':
                            set('underline', True)
                        elif item == 'italic':
                            set('italic', True)

    if bg_color is not None and not colors['Normal'].get('bgcolor'):
        colors['Normal']['bgcolor'] = bg_color

    color_map = {}
    for token, styles in colors.iteritems():
        if token in TOKENS:
            tmp = []
            if styles.get('noinherit'):
                tmp.append('noinherit')
            if 'color' in styles:
                tmp.append(styles['color'])
            if 'bgcolor' in styles:
                tmp.append('bg:' + styles['bgcolor'])
            if styles.get('bold'):
                tmp.append('bold')
            if styles.get('italic'):
                tmp.append('italic')
            if styles.get('underline'):
                tmp.append('underline')
            tokens = TOKENS[token]
            if not isinstance(tokens, tuple):
                tokens = (tokens,)
            for token in tokens:
                color_map[token] = ' '.join(tmp)

    default_token = color_map.pop('')
    return default_token, color_map


class StyleWriter(object):

    def __init__(self, code, name):
        self.code = code
        self.name = name.lower()

    def write_header(self, out):
        out.write('# -*- coding: utf-8 -*-\n"""\n')
        out.write('    %s Colorscheme\n' % self.name.title())
        out.write('    %s\n\n' % ('~' * (len(self.name) + 12)))
        out.write('    Converted by %s\n' % SCRIPT_NAME)
        out.write('"""\nfrom pygments.style import Style\n')
        out.write('from pygments.token import Token, %s\n\n' % ', '.join(TOKEN_TYPES))
        out.write('class %sStyle(Style):\n\n' % self.name.title())

    def write(self, out):
        self.write_header(out)
        default_token, tokens = find_colors(self.code)
        tokens = tokens.items()
        tokens.sort(lambda a, b: cmp(len(a[0]), len(a[1])))
        bg_color = [x[3:] for x in default_token.split() if x.startswith('bg:')]
        if bg_color:
            out.write('    background_color = %r\n' % bg_color[0])
        out.write('    styles = {\n')
        out.write('        %-20s%r\n' % ('Token:', default_token))
        for token, definition in tokens:
            if definition:
                out.write('        %-20s%r\n' % (token + ':', definition))
        out.write('    }')

    def __repr__(self):
        out = StringIO()
        self.write_style(out)
        return out.getvalue()


def convert(filename, stream=None):
    name = path.basename(filename)
    if name.endswith('.vim'):
        name = name[:-4]
    f = file(filename)
    code = f.read()
    f.close()
    writer = StyleWriter(code, name)
    if stream is not None:
        out = stream
    else:
        out = StringIO()
    writer.write(out)
    if stream is None:
        return out.getvalue()


def main():
    if len(sys.argv) != 2 or sys.argv[1] in ('-h', '--help'):
        print 'Usage: %s ' % sys.argv[0]
        return 2
    if sys.argv[1] in ('-v', '--version'):
        print '%s %s' % (SCRIPT_NAME, SCRIPT_VERSION)
        return
    filename = sys.argv[1]
    if not (path.exists(filename) and path.isfile(filename)):
        print 'Error: %s not found' % filename
        return 1
    convert(filename, sys.stdout)
    sys.stdout.write('\n')


if __name__ == '__main__':
    sys.exit(main() or 0)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    Pygments
    ~~~~~~~~

    Pygments is a syntax highlighting package written in Python.

    It is a generic syntax highlighter for general use in all kinds of software
    such as forum systems, wikis or other applications that need to prettify
    source code. Highlights are:

    * a wide range of common languages and markup formats is supported
    * special attention is paid to details, increasing quality by a fair amount
    * support for new languages and formats are added easily
    * a number of output formats, presently HTML, LaTeX, RTF, SVG and ANSI sequences
    * it is usable as a command-line tool and as a library
    * ... and it highlights even Brainfuck!

    The `Pygments tip`_ is installable with ``easy_install Pygments==dev``.

    .. _Pygments tip: http://dev.pocoo.org/hg/pygments-main/archive/tip.tar.gz#egg=Pygments-dev

    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

try:
    from setuptools import setup, find_packages
except ImportError:
    from distutils.core import setup
    def find_packages():
        return [
            'pygments',
            'pygments.lexers',
            'pygments.formatters',
            'pygments.styles',
            'pygments.filters',
        ]

try:
    from distutils.command.build_py import build_py_2to3 as build_py
except ImportError:
    from distutils.command.build_py import build_py

setup(
    name = 'Pygments',
    version = '1.1',
    url = 'http://pygments.org/',
    license = 'BSD License',
    author = 'Georg Brandl',
    author_email = 'georg@python.org',
    description = 'Pygments is a syntax highlighting package written in Python.',
    long_description = __doc__,
    keywords = 'syntax highlighting',
    packages = find_packages(),
    scripts = ['pygmentize'],
    platforms = 'any',
    zip_safe = False,
    include_package_data = True,
    classifiers = [
        'License :: OSI Approved :: BSD License',
        'Intended Audience :: Developers',
        'Intended Audience :: End Users/Desktop',
        'Intended Audience :: System Administrators',
        'Development Status :: 5 - Production/Stable',
        'Programming Language :: Python',
        'Operating System :: OS Independent',
    ],
    cmdclass = {'build_py': build_py},
)
# -*- coding: utf-8 -*-
"""
    Pygments unit tests
    ~~~~~~~~~~~~~~~~~~

    Usage::

        python run.py [testfile ...]


    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import sys, os
import unittest

from os.path import dirname, basename, join, abspath

import pygments

try:
    import coverage
except ImportError:
    coverage = None

testdir = abspath(dirname(__file__))

failed = []
total_test_count = 0
error_test_count = 0


def err(file, what, exc):
    print >>sys.stderr, file, 'failed %s:' % what,
    print >>sys.stderr, exc
    failed.append(file[:-3])


class QuietTestRunner(object):
    """Customized test runner for relatively quiet output"""

    def __init__(self, testname, stream=sys.stderr):
        self.testname = testname
        self.stream = unittest._WritelnDecorator(stream)

    def run(self, test):
        global total_test_count
        global error_test_count
        result = unittest._TextTestResult(self.stream, True, 1)
        test(result)
        if not result.wasSuccessful():
            self.stream.write(' FAIL:')
            result.printErrors()
            failed.append(self.testname)
        else:
            self.stream.write(' ok\n')
        total_test_count += result.testsRun
        error_test_count += len(result.errors) + len(result.failures)
        return result


def run_tests(with_coverage=False):
    # needed to avoid confusion involving atexit handlers
    import logging

    if sys.argv[1:]:
        # test only files given on cmdline
        files = [entry + '.py' for entry in sys.argv[1:] if entry.startswith('test_')]
    else:
        files = [entry for entry in os.listdir(testdir)
                 if (entry.startswith('test_') and entry.endswith('.py'))]
        files.sort()

    WIDTH = 85

    print >>sys.stderr, \
        ('Pygments %s Test Suite running%s, stand by...' %
         (pygments.__version__,
          with_coverage and " with coverage analysis" or "")).center(WIDTH)
    print >>sys.stderr, ('(using Python %s)' % sys.version.split()[0]).center(WIDTH)
    print >>sys.stderr, '='*WIDTH

    if with_coverage:
        coverage.erase()
        coverage.start()

    for testfile in files:
        globs = {'__file__': join(testdir, testfile)}
        try:
            execfile(join(testdir, testfile), globs)
        except Exception, exc:
            raise
            err(testfile, 'execfile', exc)
            continue
        sys.stderr.write(testfile[:-3] + ': ')
        try:
            runner = QuietTestRunner(testfile[:-3])
            # make a test suite of all TestCases in the file
            tests = []
            for name, thing in globs.iteritems():
                if name.endswith('Test'):
                    tests.append((name, unittest.makeSuite(thing)))
            tests.sort()
            suite = unittest.TestSuite()
            suite.addTests([x[1] for x in tests])
            runner.run(suite)
        except Exception, exc:
            err(testfile, 'running test', exc)

    print >>sys.stderr, '='*WIDTH
    if failed:
        print >>sys.stderr, '%d of %d tests failed.' % \
              (error_test_count, total_test_count)
        print >>sys.stderr, 'Tests failed in:', ', '.join(failed)
        ret = 1
    else:
        if total_test_count == 1:
            print >>sys.stderr, '1 test happy.'
        else:
            print >>sys.stderr, 'All %d tests happy.' % total_test_count
        ret = 0

    if with_coverage:
        coverage.stop()
        modules = [mod for name, mod in sys.modules.iteritems()
                   if name.startswith('pygments.') and mod]
        coverage.report(modules)

    return ret


if __name__ == '__main__':
    with_coverage = False
    if sys.argv[1:2] == ['-C']:
        with_coverage = bool(coverage)
        del sys.argv[1]
    sys.exit(run_tests(with_coverage))
# -*- coding: utf-8 -*-
"""
    Pygments unit tests
    ~~~~~~~~~~~~~~~~~~

    Usage::

        python run.py [testfile ...]


    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import sys, os

if sys.version_info >= (3,):
    # copy test suite over to "build/lib" and convert it
    print ('Copying and converting sources to build/lib/test...')
    from distutils.util import copydir_run_2to3
    testroot = os.path.dirname(__file__)
    newroot = os.path.join(testroot, '..', 'build/lib/test')
    copydir_run_2to3(testroot, newroot)
    os.chdir(os.path.join(newroot, '..'))

try:
    import nose
except ImportError:
    print ("nose is required to run the test suites")
    sys.exit(1)

nose.main()
# coding: utf-8
"""
Support for Pygments tests
"""

import os


def location(mod_name):
    """
    Return the file and directory that the code for *mod_name* is in.
    """
    source = mod_name.endswith("pyc") and mod_name[:-1] or mod_name
    source = os.path.abspath(source)
    return source, os.path.dirname(source)
# -*- coding: utf-8 -*-
"""
    Pygments basic API tests
    ~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import os
import unittest
import random

from pygments import lexers, formatters, filters, format
from pygments.token import _TokenType, Text
from pygments.lexer import RegexLexer
from pygments.formatters.img import FontNotFound
from pygments.util import BytesIO, StringIO, bytes, b

import support

TESTFILE, TESTDIR = support.location(__file__)

test_content = [chr(i) for i in xrange(33, 128)] * 5
random.shuffle(test_content)
test_content = ''.join(test_content) + '\n'

class LexersTest(unittest.TestCase):

    def test_import_all(self):
        # instantiate every lexer, to see if the token type defs are correct
        for x in lexers.LEXERS.keys():
            c = getattr(lexers, x)()

    def test_lexer_classes(self):
        a = self.assert_
        ae = self.assertEquals
        # test that every lexer class has the correct public API
        for lexer in lexers._iter_lexerclasses():
            a(type(lexer.name) is str)
            for attr in 'aliases', 'filenames', 'alias_filenames', 'mimetypes':
                a(hasattr(lexer, attr))
                a(type(getattr(lexer, attr)) is list, "%s: %s attribute wrong" %
                                                      (lexer, attr))
            result = lexer.analyse_text("abc")
            a(isinstance(result, float) and 0.0 <= result <= 1.0)

            inst = lexer(opt1="val1", opt2="val2")
            if issubclass(lexer, RegexLexer):
                if not hasattr(lexer, '_tokens'):
                    # if there's no "_tokens", the lexer has to be one with
                    # multiple tokendef variants
                    a(lexer.token_variants)
                    for variant in lexer.tokens:
                        a('root' in lexer.tokens[variant])
                else:
                    a('root' in lexer._tokens, '%s has no root state' % lexer)

            tokens = list(inst.get_tokens(test_content))
            txt = ""
            for token in tokens:
                a(isinstance(token, tuple))
                a(isinstance(token[0], _TokenType))
                if isinstance(token[1], str):
                    print repr(token[1])
                a(isinstance(token[1], unicode))
                txt += token[1]
            ae(txt, test_content, "%s lexer roundtrip failed: %r != %r" %
                    (lexer.name, test_content, txt))

    def test_get_lexers(self):
        a = self.assert_
        ae = self.assertEquals
        # test that the lexers functions work

        for func, args in [(lexers.get_lexer_by_name, ("python",)),
                           (lexers.get_lexer_for_filename, ("test.py",)),
                           (lexers.get_lexer_for_mimetype, ("text/x-python",)),
                           (lexers.guess_lexer, ("#!/usr/bin/python -O\nprint",)),
                           (lexers.guess_lexer_for_filename, ("a.py", "<%= @foo %>"))
                           ]:
            x = func(opt="val", *args)
            a(isinstance(x, lexers.PythonLexer))
            ae(x.options["opt"], "val")


class FiltersTest(unittest.TestCase):

    def test_basic(self):
        filter_args = {
            'whitespace': {'spaces': True, 'tabs': True, 'newlines': True},
            'highlight': {'names': ['isinstance', 'lexers', 'x']},
        }
        for x in filters.FILTERS.keys():
            lx = lexers.PythonLexer()
            lx.add_filter(x, **filter_args.get(x, {}))
            text = open(TESTFILE, 'rb').read().decode('utf-8')
            tokens = list(lx.get_tokens(text))
            roundtext = ''.join([t[1] for t in tokens])
            if x not in ('whitespace', 'keywordcase'):
                # these filters change the text
                self.assertEquals(roundtext, text,
                                  "lexer roundtrip with %s filter failed" % x)

    def test_raiseonerror(self):
        lx = lexers.PythonLexer()
        lx.add_filter('raiseonerror', excclass=RuntimeError)
        self.assertRaises(RuntimeError, list, lx.get_tokens('$'))

    def test_whitespace(self):
        lx = lexers.PythonLexer()
        lx.add_filter('whitespace', spaces='%')
        text = open(TESTFILE, 'rb').read().decode('utf-8')
        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
        self.failIf(' ' in lxtext)

    def test_keywordcase(self):
        lx = lexers.PythonLexer()
        lx.add_filter('keywordcase', case='capitalize')
        text = open(TESTFILE, 'rb').read().decode('utf-8')
        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
        self.assert_('Def' in lxtext and 'Class' in lxtext)

    def test_codetag(self):
        lx = lexers.PythonLexer()
        lx.add_filter('codetagify')
        text = u'# BUG: text'
        tokens = list(lx.get_tokens(text))
        self.assertEquals('# ', tokens[0][1])
        self.assertEquals('BUG', tokens[1][1])

    def test_codetag_boundary(self):
        # http://dev.pocoo.org/projects/pygments/ticket/368
        lx = lexers.PythonLexer()
        lx.add_filter('codetagify')
        text = u'# DEBUG: text'
        tokens = list(lx.get_tokens(text))
        self.assertEquals('# DEBUG: text', tokens[0][1])


class FormattersTest(unittest.TestCase):

    def test_public_api(self):
        a = self.assert_
        ae = self.assertEquals
        ts = list(lexers.PythonLexer().get_tokens("def f(): pass"))
        out = StringIO()
        # test that every formatter class has the correct public API
        for formatter, info in formatters.FORMATTERS.iteritems():
            a(len(info) == 4)
            a(info[0], "missing formatter name") # name
            a(info[1], "missing formatter aliases") # aliases
            a(info[3], "missing formatter docstring") # doc

            if formatter.name == 'Raw tokens':
                # will not work with Unicode output file
                continue

            try:
                inst = formatter(opt1="val1")
            except (ImportError, FontNotFound):
                continue
            try:
                inst.get_style_defs()
            except NotImplementedError:
                # may be raised by formatters for which it doesn't make sense
                pass
            inst.format(ts, out)

    def test_encodings(self):
        from pygments.formatters import HtmlFormatter

        # unicode output
        fmt = HtmlFormatter()
        tokens = [(Text, u"ä")]
        out = format(tokens, fmt)
        self.assert_(type(out) is unicode)
        self.assert_(u"ä" in out)

        # encoding option
        fmt = HtmlFormatter(encoding="latin1")
        tokens = [(Text, u"ä")]
        self.assert_(u"ä".encode("latin1") in format(tokens, fmt))

        # encoding and outencoding option
        fmt = HtmlFormatter(encoding="latin1", outencoding="utf8")
        tokens = [(Text, u"ä")]
        self.assert_(u"ä".encode("utf8") in format(tokens, fmt))

    def test_styles(self):
        from pygments.formatters import HtmlFormatter
        fmt = HtmlFormatter(style="pastie")

    def test_unicode_handling(self):
        # test that the formatter supports encoding and Unicode
        tokens = list(lexers.PythonLexer(encoding='utf-8').
                      get_tokens("def f(): 'ä'"))
        for formatter, info in formatters.FORMATTERS.iteritems():
            try:
                inst = formatter(encoding=None)
            except (ImportError, FontNotFound):
                # some dependency or font not installed
                continue

            if formatter.name != 'Raw tokens':
                out = format(tokens, inst)
                if formatter.unicodeoutput:
                    self.assert_(type(out) is unicode)

                inst = formatter(encoding='utf-8')
                out = format(tokens, inst)
                self.assert_(type(out) is bytes, '%s: %r' % (formatter, out))
                # Cannot test for encoding, since formatters may have to escape
                # non-ASCII characters.
            else:
                inst = formatter()
                out = format(tokens, inst)
                self.assert_(type(out) is bytes, '%s: %r' % (formatter, out))

    def test_get_formatters(self):
        a = self.assert_
        ae = self.assertEquals
        # test that the formatters functions work
        x = formatters.get_formatter_by_name("html", opt="val")
        a(isinstance(x, formatters.HtmlFormatter))
        ae(x.options["opt"], "val")

        x = formatters.get_formatter_for_filename("a.html", opt="val")
        a(isinstance(x, formatters.HtmlFormatter))
        ae(x.options["opt"], "val")
# -*- coding: utf-8 -*-
"""
    Basic CLexer Test
    ~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import unittest
import os

from pygments.token import Text, Number
from pygments.lexers import CLexer


class CLexerTest(unittest.TestCase):

    def setUp(self):
        self.lexer = CLexer()

    def testNumbers(self):
        code = '42 23.42 23. .42 023 0xdeadbeef 23e+42 42e-23'
        wanted = []
        for item in zip([Number.Integer, Number.Float, Number.Float,
                         Number.Float, Number.Oct, Number.Hex,
                         Number.Float, Number.Float], code.split()):
            wanted.append(item)
            wanted.append((Text, ' '))
        wanted = [(Text, '')] + wanted[:-1] + [(Text, '\n')]
        self.assertEqual(list(self.lexer.get_tokens(code)), wanted)


if __name__ == '__main__':
    unittest.main()
# -*- coding: utf-8 -*-
"""
    Command line test
    ~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

# Test the command line interface

import sys, os
import unittest
import StringIO

from pygments import highlight
from pygments.cmdline import main as cmdline_main

import support

TESTFILE, TESTDIR = support.location(__file__)


def run_cmdline(*args):
    saved_stdout = sys.stdout
    saved_stderr = sys.stderr
    new_stdout = sys.stdout = StringIO.StringIO()
    new_stderr = sys.stderr = StringIO.StringIO()
    try:
        ret = cmdline_main(["pygmentize"] + list(args))
    finally:
        sys.stdout = saved_stdout
        sys.stderr = saved_stderr
    return (ret, new_stdout.getvalue(), new_stderr.getvalue())


class CmdLineTest(unittest.TestCase):

    def test_L_opt(self):
        c, o, e = run_cmdline("-L")
        self.assertEquals(c, 0)
        self.assert_("Lexers" in o and "Formatters" in o and
                     "Filters" in o and "Styles" in o)
        c, o, e = run_cmdline("-L", "lexer")
        self.assertEquals(c, 0)
        self.assert_("Lexers" in o and "Formatters" not in o)
        c, o, e = run_cmdline("-L", "lexers")
        self.assertEquals(c, 0)

    def test_O_opt(self):
        filename = TESTFILE
        c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar",
                              "-fhtml", filename)
        self.assertEquals(c, 0)
        self.assert_("foo, bar=baz=," in o)

    def test_F_opt(self):
        filename = TESTFILE
        c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb,"
                              "names=TESTFILE filename",
                              "-fhtml", filename)
        self.assertEquals(c, 0)
        self.assert_('_filename '
                                     'for overriding, thus no lexer found.'
                                    % fn)
            try:
                name, rest = fn.split("_", 1)
                lx = get_lexer_by_name(name)
            except ClassNotFound:
                raise AssertionError('no lexer found for file %r' % fn)
        yield check_lexer, lx, absfn

def check_lexer(lx, absfn):
    text = open(absfn, 'rb').read()
    text = text.strip(b('\n')) + b('\n')
    try:
        text = text.decode('utf-8')
    except UnicodeError:
        text = text.decode('latin1')
    ntext = []
    for type, val in lx.get_tokens(text):
        ntext.append(val)
        assert type != Error, 'lexer %s generated error token for %s' % \
                (lx, absfn)
    if u''.join(ntext) != text:
        raise AssertionError('round trip failed for ' + absfn)
# -*- coding: utf-8 -*-
"""
    Pygments HTML formatter tests
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import os
import re
import unittest
import StringIO
import tempfile
from os.path import join, dirname, isfile, abspath

from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter, NullFormatter
from pygments.formatters.html import escape_html

import support

TESTFILE, TESTDIR = support.location(__file__)

tokensource = list(PythonLexer(encoding='utf-8').get_tokens(open(TESTFILE).read()))

class HtmlFormatterTest(unittest.TestCase):
    def test_correct_output(self):
        hfmt = HtmlFormatter(nowrap=True)
        houtfile = StringIO.StringIO()
        hfmt.format(tokensource, houtfile)

        nfmt = NullFormatter()
        noutfile = StringIO.StringIO()
        nfmt.format(tokensource, noutfile)

        stripped_html = re.sub('<.*?>', '', houtfile.getvalue())
        escaped_text = escape_html(noutfile.getvalue())
        self.assertEquals(stripped_html, escaped_text)

    def test_external_css(self):
        # test correct behavior
        # CSS should be in /tmp directory
        fmt1 = HtmlFormatter(full=True, cssfile='fmt1.css', outencoding='utf-8')
        # CSS should be in TESTDIR (TESTDIR is absolute)
        fmt2 = HtmlFormatter(full=True, cssfile=join(TESTDIR, 'fmt2.css'),
                             outencoding='utf-8')
        tfile = tempfile.NamedTemporaryFile(suffix='.html')
        fmt1.format(tokensource, tfile)
        try:
            fmt2.format(tokensource, tfile)
            self.assert_(isfile(join(TESTDIR, 'fmt2.css')))
        except IOError:
            # test directory not writable
            pass
        tfile.close()

        self.assert_(isfile(join(dirname(tfile.name), 'fmt1.css')))
        os.unlink(join(dirname(tfile.name), 'fmt1.css'))
        try:
            os.unlink(join(TESTDIR, 'fmt2.css'))
        except OSError:
            pass

    def test_all_options(self):
        for optdict in [dict(nowrap=True),
                        dict(linenos=True),
                        dict(linenos=True, full=True),
                        dict(linenos=True, full=True, noclasses=True)]:

            outfile = StringIO.StringIO()
            fmt = HtmlFormatter(**optdict)
            fmt.format(tokensource, outfile)

    def test_valid_output(self):
        # test all available wrappers
        fmt = HtmlFormatter(full=True, linenos=True, noclasses=True,
                            outencoding='utf-8')

        handle, pathname = tempfile.mkstemp('.html')
        tfile = os.fdopen(handle, 'w+b')
        fmt.format(tokensource, tfile)
        tfile.close()
        catname = os.path.join(TESTDIR, 'dtds', 'HTML4.soc')
        try:
            try:
                import subprocess
                ret = subprocess.Popen(['nsgmls', '-s', '-c', catname, pathname],
                                       stdout=subprocess.PIPE).wait()
            except ImportError:
                # Python 2.3 - no subprocess module
                ret = os.popen('nsgmls -s -c "%s" "%s"' % (catname, pathname)).close()
                if ret == 32512: raise OSError  # not found
        except OSError:
            # nsgmls not available
            pass
        else:
            self.failIf(ret, 'nsgmls run reported errors')

        os.unlink(pathname)

    def test_get_style_defs(self):
        fmt = HtmlFormatter()
        sd = fmt.get_style_defs()
        self.assert_(sd.startswith('.'))

        fmt = HtmlFormatter(cssclass='foo')
        sd = fmt.get_style_defs()
        self.assert_(sd.startswith('.foo'))
        sd = fmt.get_style_defs('.bar')
        self.assert_(sd.startswith('.bar'))
        sd = fmt.get_style_defs(['.bar', '.baz'])
        fl = sd.splitlines()[0]
        self.assert_('.bar' in fl and '.baz' in fl)

    def test_unicode_options(self):
        fmt = HtmlFormatter(title=u'Föö',
                            cssclass=u'bär',
                            cssstyles=u'div:before { content: \'bäz\' }',
                            encoding='utf-8')
        handle, pathname = tempfile.mkstemp('.html')
        tfile = os.fdopen(handle, 'w+b')
        fmt.format(tokensource, tfile)
        tfile.close()
# -*- coding: utf-8 -*-
"""
    Pygments LaTeX formatter tests
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import os
import unittest
import tempfile

from pygments.formatters import LatexFormatter
from pygments.lexers import PythonLexer

import support

TESTFILE, TESTDIR = support.location(__file__)


class LatexFormatterTest(unittest.TestCase):

    def test_valid_output(self):
        tokensource = list(PythonLexer().get_tokens(open(TESTFILE).read()))
        fmt = LatexFormatter(full=True, encoding='latin1')

        handle, pathname = tempfile.mkstemp('.tex')
        # place all output files in /tmp too
        old_wd = os.getcwd()
        os.chdir(os.path.dirname(pathname))
        tfile = os.fdopen(handle, 'wb')
        fmt.format(tokensource, tfile)
        tfile.close()
        try:
            try:
                import subprocess
                ret = subprocess.Popen(['latex', '-interaction=nonstopmode',
                                        pathname],
                                       stdout=subprocess.PIPE).wait()
            except ImportError:
                # Python 2.3 - no subprocess module
                ret = os.popen('latex -interaction=nonstopmode "%s"'
                               % pathname).close()
                if ret == 32512: raise OSError  # not found
        except OSError:
            # latex not available
            pass
        else:
            self.failIf(ret, 'latex run reported errors')

        os.unlink(pathname)
        os.chdir(old_wd)
# -*- coding: utf-8 -*-
"""
    Pygments regex lexer tests
    ~~~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import unittest

from pygments.token import Text
from pygments.lexer import RegexLexer

class TestLexer(RegexLexer):
    """Test tuple state transitions including #pop."""
    tokens = {
        'root': [
            ('a', Text.Root, 'rag'),
            ('e', Text.Root),
        ],
        'beer': [
            ('d', Text.Beer, ('#pop', '#pop')),
        ],
        'rag': [
            ('b', Text.Rag, '#push'),
            ('c', Text.Rag, ('#pop', 'beer')),
        ],
    }

class TupleTransTest(unittest.TestCase):
    def test(self):
        lx = TestLexer()
        toks = list(lx.get_tokens_unprocessed('abcde'))
        self.assertEquals(toks,
           [(0, Text.Root, 'a'), (1, Text.Rag, 'b'), (2, Text.Rag, 'c'),
            (3, Text.Beer, 'd'), (4, Text.Root, 'e')])
# -*- coding: utf-8 -*-
"""
    Test suite for the token module
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import unittest
import StringIO
import sys

from pygments import token


class TokenTest(unittest.TestCase):

    def test_tokentype(self):
        e = self.assertEquals
        r = self.assertRaises

        t = token.String

        e(t.split(), [token.Token, token.Literal, token.String])

        e(t.__class__, token._TokenType)

    def test_functions(self):
        self.assert_(token.is_token_subtype(token.String, token.String))
        self.assert_(token.is_token_subtype(token.String, token.Literal))
        self.failIf(token.is_token_subtype(token.Literal, token.String))

        self.assert_(token.string_to_tokentype(token.String) is token.String)
        self.assert_(token.string_to_tokentype('') is token.Token)
        self.assert_(token.string_to_tokentype('String') is token.String)

    def test_sanity_check(self):
        stp = token.STANDARD_TYPES.copy()
        stp[token.Token] = '---' # Token and Text do conflict, that is okay
        t = {}
        for k, v in stp.iteritems():
            t.setdefault(v, []).append(k)
        if len(t) == len(stp):
            return # Okay

        for k, v in t.iteritems():
            if len(v) > 1:
                self.fail("%r has more than one key: %r" % (k, v))


if __name__ == '__main__':
    unittest.main()
import unittest
from pygments.lexer import using, bygroups, this, RegexLexer
from pygments.token import String, Text, Keyword

class TestLexer(RegexLexer):
    tokens = {
        'root': [
            (r'#.*', using(this, state='invalid')),
            (r'(")(.+?)(")', bygroups(String, using(this, state='string'), String)),
            (r'[^"]+', Text),
        ],
        'string': [
            (r'.+', Keyword),
        ],
    }

class UsingStateTest(unittest.TestCase):
    def test_basic(self):
        expected = [(Text, 'a'), (String, '"'), (Keyword, 'bcd'),
                    (String, '"'), (Text, 'e\n')]
        t = list(TestLexer().get_tokens('a"bcd"e'))
        self.assertEquals(t, expected)
    def test_error(self):
        def gen():
            x = list(TestLexer().get_tokens('#a'))
        #XXX: should probably raise a more specific exception if the state
        #     doesn't exist.
        self.assertRaises(Exception, gen)

if __name__ == "__main__":
    unittest.main()
# -*- coding: utf-8 -*-
"""
    Test suite for the util module
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import unittest
import os

from pygments import util


class UtilTest(unittest.TestCase):

    def test_getoptions(self):
        raises = self.assertRaises
        equals = self.assertEquals

        equals(util.get_bool_opt({}, 'a', True), True)
        equals(util.get_bool_opt({}, 'a', 1), True)
        equals(util.get_bool_opt({}, 'a', 'true'), True)
        equals(util.get_bool_opt({}, 'a', 'no'), False)
        raises(util.OptionError, util.get_bool_opt, {}, 'a', [])
        raises(util.OptionError, util.get_bool_opt, {}, 'a', 'foo')

        equals(util.get_int_opt({}, 'a', 1), 1)
        raises(util.OptionError, util.get_int_opt, {}, 'a', [])
        raises(util.OptionError, util.get_int_opt, {}, 'a', 'bar')

        equals(util.get_list_opt({}, 'a', [1]), [1])
        equals(util.get_list_opt({}, 'a', '1 2'), ['1', '2'])
        raises(util.OptionError, util.get_list_opt, {}, 'a', 1)


    def test_docstring_headline(self):
        def f1():
            """
            docstring headline

            other text
            """
        def f2():
            """
            docstring
            headline

            other text
            """

        self.assertEquals(util.docstring_headline(f1), "docstring headline")
        self.assertEquals(util.docstring_headline(f2), "docstring headline")

    def test_analysator(self):
        class X(object):
            def analyse(text):
                return 0.5
            analyse = util.make_analysator(analyse)
        self.assertEquals(X.analyse(''), 0.5)

    def test_shebang_matches(self):
        self.assert_(util.shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?'))
        self.assert_(util.shebang_matches('#!/usr/bin/python2.4', r'python(2\.\d)?'))
        self.assert_(util.shebang_matches('#!/usr/bin/startsomethingwith python',
                                          r'python(2\.\d)?'))
        self.assert_(util.shebang_matches('#!C:\\Python2.4\\Python.exe',
                                          r'python(2\.\d)?'))

        self.failIf(util.shebang_matches('#!/usr/bin/python-ruby', r'python(2\.\d)?'))
        self.failIf(util.shebang_matches('#!/usr/bin/python/ruby', r'python(2\.\d)?'))
        self.failIf(util.shebang_matches('#!', r'python'))

    def test_doctype_matches(self):
        self.assert_(util.doctype_matches(' ',
                                          'html.*'))
        self.failIf(util.doctype_matches('  ',
                                         'html.*'))
        self.assert_(util.html_doctype_matches(
            ''))

    def test_xml(self):
        self.assert_(util.looks_like_xml(
            ''))
        self.assert_(util.looks_like_xml('abc'))
        self.failIf(util.looks_like_xml(''))

if __name__ == '__main__':
    unittest.main()