summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTanner Prynn <tanner.prynn@nccgroup.trust>2016-02-24 17:46:32 -0600
committerTanner Prynn <tanner.prynn@nccgroup.trust>2016-02-24 17:46:32 -0600
commit4a49415561c00e2d235d266ee81cb026982f8e20 (patch)
tree6bb3883e3689bdeec35178a9504f8e9ab3f99daf
parent604256ba5bdd0e7d707201d119db7035f478234d (diff)
downloadpygments-git-4a49415561c00e2d235d266ee81cb026982f8e20.tar.gz
Update pull request per comments by birkenfeld.
Add optional function parameter for the class name to instantiate, and update cli to support this. Move error handling to within the loading functions; they now only raise ClassNotFound. Modify doc with these updates and the version number. Test case clean up and additions.
-rw-r--r--doc/docs/api.rst32
-rw-r--r--doc/docs/cmdline.rst10
-rw-r--r--doc/docs/lexerdevelopment.rst14
-rw-r--r--pygments/cmdline.py59
-rw-r--r--pygments/formatters/__init__.py34
-rw-r--r--pygments/lexers/__init__.py33
-rw-r--r--tests/support/html_formatter.py5
-rw-r--r--tests/support/python_lexer.py4
-rw-r--r--tests/test_cmdline.py35
9 files changed, 140 insertions, 86 deletions
diff --git a/doc/docs/api.rst b/doc/docs/api.rst
index ea158d65..a6b242dd 100644
--- a/doc/docs/api.rst
+++ b/doc/docs/api.rst
@@ -62,21 +62,17 @@ Functions from :mod:`pygments.lexers`:
Will raise :exc:`pygments.util.ClassNotFound` if not lexer for that mimetype
is found.
-.. function:: load_lexer_from_file(filename, **options)
+.. function:: load_lexer_from_file(filename, lexername="CustomLexer", **options)
Return a `Lexer` subclass instance loaded from the provided file, relative
- to the current directory. The file is expected to contain a CustomLexer class
- which matches Pygments' lexer definitions. Users should be very careful with
+ to the current directory. The file is expected to contain a Lexer class
+ named `lexername` (by default, CustomLexer). Users should be very careful with
the input, because this method is equivalent to running eval on the input file.
The lexer is given the `options` at its instantiation.
- :exc:`IOError` is raised if the file is not found or unreadable
+ :exc:`ClassNotFound` is raised if there are any errors loading the Lexer
- :exc:`ImportError` is raised if the file doesn't have a CustomLexer class
-
- Additionally, arbitrary exceptions could occur when evaluating the file
-
- .. versionadded:: ?
+ .. versionadded:: 2.2
.. function:: guess_lexer(text, **options)
@@ -141,21 +137,17 @@ Functions from :mod:`pygments.formatters`:
Will raise :exc:`pygments.util.ClassNotFound` if no formatter for that filename
is found.
-.. function:: load_formatter_from_file(filename, **options)
+.. function:: load_formatter_from_file(filename, formattername="CustomFormatter", **options)
Return a `Formatter` subclass instance loaded from the provided file, relative
- to the current directory. The file is expected to contain a CustomFormatter class
- which matches Pygments' formatter definitions. Users should be very careful with
- the input, because this method is equivalent to running eval on the input file.
- The formatter is given the `options` at its instantiation.
+ to the current directory. The file is expected to contain a Formatter class
+ named ``formattername`` (by default, CustomFormatter). Users should be very
+ careful with the input, because this method is equivalent to running eval
+ on the input file. The formatter is given the `options` at its instantiation.
- :exc:`IOError` is raised if the file is not found or unreadable
+ :exc:`ClassNotFound` is raised if there are any errors loading the Formatter
- :exc:`ImportError` is raised if the file doesn't have a CustomFormatter class
-
- Additionally, arbitrary exceptions could occur when evaluating the file
-
- .. versionadded:: ?
+ .. versionadded:: 2.2
.. module:: pygments.styles
diff --git a/doc/docs/cmdline.rst b/doc/docs/cmdline.rst
index 8fcd3c8c..e4f94ea5 100644
--- a/doc/docs/cmdline.rst
+++ b/doc/docs/cmdline.rst
@@ -102,13 +102,17 @@ lexer is known for that filename, ``text`` is printed.
Custom Lexers and Formatters
----------------------------
-.. versionadded:: ?
+.. versionadded:: 2.2
-The ``--load-from-file`` flag enables custom lexers and formatters to be loaded
+The ``-x`` flag enables custom lexers and formatters to be loaded
from files relative to the current directory. Create a file with a class named
CustomLexer or CustomFormatter, then specify it on the command line::
- $ pygmentize -l your_lexer.py -f your_formatter.py --load-from-file
+ $ pygmentize -l your_lexer.py -f your_formatter.py -x
+
+You can also specify the name of your class with a colon::
+
+ $ pygmentize -l your_lexer.py:SomeLexer -x
For more information, see :doc:`the Pygments documentation on Lexer development
<lexerdevelopment>`.
diff --git a/doc/docs/lexerdevelopment.rst b/doc/docs/lexerdevelopment.rst
index 8c9b7baa..bba81d24 100644
--- a/doc/docs/lexerdevelopment.rst
+++ b/doc/docs/lexerdevelopment.rst
@@ -102,18 +102,28 @@ First, change the name of your lexer class to CustomLexer:
"""All your lexer code goes here!"""
Then you can load the lexer from the command line with the additional
-flag ``--load-from-file``:
+flag ``-x``:
.. code-block:: console
- $ pygmentize -l your_lexer_file.py --load-from-file
+ $ pygmentize -l your_lexer_file.py -x
+
+To specify a class name other than CustomLexer, append it with a colon:
+
+.. code-block:: console
+
+ $ pygmentize -l your_lexer.py:SomeLexer -x
Or, using the Python API:
.. code-block:: python
+ # For a lexer named CustomLexer
your_lexer = load_lexer_from_file(filename, **options)
+ # For a lexer named MyNewLexer
+ your_named_lexer = load_lexer_from_file(filename, "MyNewLexer", **options)
+
When loading custom lexers and formatters, be extremely careful to use only
trusted files; Pygments will perform the equivalent of ``eval`` on them.
diff --git a/pygments/cmdline.py b/pygments/cmdline.py
index f5fb2d0d..69604481 100644
--- a/pygments/cmdline.py
+++ b/pygments/cmdline.py
@@ -31,7 +31,7 @@ from pygments.styles import get_all_styles, get_style_by_name
USAGE = """\
Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
- [-O <options>] [-P <option=value>] [-s] [-v] [-o <outfile>] [<infile>]
+ [-O <options>] [-P <option=value>] [-s] [-v] [-x] [-o <outfile>] [<infile>]
%s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
%s -L [<which> ...]
@@ -57,12 +57,13 @@ Likewise, <formatter> is a formatter name, and will be guessed from
the extension of the output file name. If no output file is given,
the terminal formatter will be used by default.
-The additional option --load-from-file allows custom lexers and formatters
-to be loaded from a .py file relative to the current working directory.
-For example, ``-l ./customlexer.py --load-from-file``. The file should
-contain a CustomLexer or CustomFormatter class which matches the Pygments
-lexer and formatter definitions. Users should be very careful not to use
-this option with untrusted files, because it will eval() them.
+The additional option -x allows custom lexers and formatters to be
+loaded from a .py file relative to the current working directory. For
+example, ``-l ./customlexer.py -x``. By default, this option expects a
+file with a class named CustomLexer or CustomFormatter; you can also
+specify your own class name with a colon (``-l ./lexer.py:MyLexer``).
+Users should be very careful not to use this option with untrusted files,
+because it will import and run them.
With the -O option, you can give the lexer and formatter a comma-
separated list of options, e.g. ``-O bg=light,python=cool``.
@@ -322,8 +323,8 @@ def main_inner(popts, args, usage):
opts.pop('-F', None)
allow_custom_lexer_formatter = False
- # --load-from-file: allow custom lexers and formatters
- if opts.pop('--load-from-file', None) is not None:
+ # -x: allow custom (eXternal) lexers and formatters
+ if opts.pop('-x', None) is not None:
allow_custom_lexer_formatter = True
# select lexer
@@ -333,18 +334,15 @@ def main_inner(popts, args, usage):
lexername = opts.pop('-l', None)
if lexername:
# custom lexer, located relative to user's cwd
- if allow_custom_lexer_formatter and lexername[-3:] == '.py':
+ if allow_custom_lexer_formatter and '.py' in lexername:
try:
- lexer = load_lexer_from_file(lexername, **parsed_opts)
- except IOError as err:
- print('Error: cannot read %s:' % lexername, err,
- file=sys.stderr)
- return 1
- except ImportError as err:
- print('Error: no CustomLexer class found in %s' % lexername,
- file=sys.stderr)
- return 1
- except Exception as err:
+ if ':' in lexername:
+ filename, name = lexername.rsplit(':', 1)
+ lexer = load_lexer_from_file(filename, name,
+ **parsed_opts)
+ else:
+ lexer = load_lexer_from_file(lexername, **parsed_opts)
+ except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
else:
@@ -430,17 +428,15 @@ def main_inner(popts, args, usage):
fmter = opts.pop('-f', None)
if fmter:
# custom formatter, located relative to user's cwd
- if allow_custom_lexer_formatter and fmter[-3:] == '.py':
+ if allow_custom_lexer_formatter and '.py' in fmter:
try:
- fmter = load_formatter_from_file(fmter, **parsed_opts)
- except IOError as err:
- print('Error: cannot read %s:' % fmter, err, file=sys.stderr)
- return 1
- except ImportError as err:
- print('Error: no CustomFormatter class found in %s' % fmter,
- file=sys.stderr)
- return 1
- except Exception as err:
+ if ':' in fmter:
+ file, fmtername = fmter.rsplit(':', 1)
+ fmter = load_formatter_from_file(file, fmtername,
+ **parsed_opts)
+ else:
+ fmter = load_formatter_from_file(fmter, **parsed_opts)
+ except ClassNotFound as err:
print('Error:', err, file=sys.stderr)
return 1
else:
@@ -538,8 +534,7 @@ def main(args=sys.argv):
usage = USAGE % ((args[0],) * 6)
try:
- popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgs",
- ["load-from-file"])
+ popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgsx")
except getopt.GetoptError:
print(usage, file=sys.stderr)
return 2
diff --git a/pygments/formatters/__init__.py b/pygments/formatters/__init__.py
index 369fb891..9034e435 100644
--- a/pygments/formatters/__init__.py
+++ b/pygments/formatters/__init__.py
@@ -80,24 +80,38 @@ def get_formatter_by_name(_alias, **options):
return cls(**options)
-def load_formatter_from_file(filename, **options):
+def load_formatter_from_file(filename, formattername="CustomFormatter",
+ **options):
"""Load a formatter from a file.
This method expects a file located relative to the current working
- directory, which contains a class named CustomFormatter.
+ directory, which contains a class named CustomFormatter. By default,
+ it expects the Formatter to be named CustomFormatter; you can specify
+ your own class name as the second argument to this function.
Users should be very careful with the input, because this method
is equivalent to running eval on the input file.
- Raises IOError if the file is not found/unreadable
- Raises ImportError if the file doesn't have a CustomFormatter class
- Raises whatever errors could happen when we eval(file)
+ Raises ClassNotFound if there are any problems importing the Formatter
"""
- # Load file as if calling import _ as customformatter
- load_source('customformatter', filename)
- # Instantiate the CustomFormatter from that file
- from customformatter import CustomFormatter
- return CustomFormatter(**options)
+ try:
+ # Load file as if calling import
+ customformatter = load_source('pygments.formatters.Custom_%s' %
+ formattername, filename)
+
+ if not hasattr(customformatter, formattername):
+ raise ClassNotFound('no valid %s class found in %s' %
+ (formattername, filename))
+
+ # Instantiate the CustomLexer from that file
+ formatter_class = getattr(customformatter, formattername)
+ return formatter_class(**options)
+ except IOError as err:
+ raise ClassNotFound('cannot read %s' % filename)
+ except ClassNotFound as err:
+ raise err
+ except Exception as err:
+ raise ClassNotFound('error when loading custom formatter: %s' % err)
def get_formatter_for_filename(fn, **options):
diff --git a/pygments/lexers/__init__.py b/pygments/lexers/__init__.py
index ff4bfb34..19a787b9 100644
--- a/pygments/lexers/__init__.py
+++ b/pygments/lexers/__init__.py
@@ -116,24 +116,37 @@ def get_lexer_by_name(_alias, **options):
raise ClassNotFound('no lexer for alias %r found' % _alias)
-def load_lexer_from_file(filename, **options):
+def load_lexer_from_file(filename, lexername="CustomLexer", **options):
"""Load a lexer from a file.
This method expects a file located relative to the current working
- directory, which contains a class named CustomLexer.
+ directory, which contains a Lexer class. By default, it expects the
+ Lexer to be name CustomLexer; you can specify your own class name
+ as the second argument to this function.
Users should be very careful with the input, because this method
is equivalent to running eval on the input file.
- Raises IOError if the file is not found/unreadable
- Raises ImportError if the file doesn't have a CustomLexer class
- Raises whatever errors could happen when we eval(file)
+ Raises ClassNotFound if there are any problems importing the Lexer
"""
- # Load file as if calling import _ as customlexer
- load_source('customlexer', filename)
- # Instantiate the CustomLexer from that file
- from customlexer import CustomLexer
- return CustomLexer(**options)
+ try:
+ # Load file as if calling import
+ customlexer = load_source('pygments.lexers.Custom_%s' % lexername,
+ filename)
+
+ if not hasattr(customlexer, lexername):
+ raise ClassNotFound('no valid %s class found in %s' %
+ (lexername, filename))
+
+ # Instantiate the CustomLexer from that file
+ lexer_class = getattr(customlexer, lexername)
+ return lexer_class(**options)
+ except IOError as err:
+ raise ClassNotFound('cannot read %s' % filename)
+ except ClassNotFound as err:
+ raise err
+ except Exception as err:
+ raise ClassNotFound('error when loading custom lexer: %s' % err)
def find_lexer_class_for_filename(_fn, code=None):
diff --git a/tests/support/html_formatter.py b/tests/support/html_formatter.py
new file mode 100644
index 00000000..7f5581ed
--- /dev/null
+++ b/tests/support/html_formatter.py
@@ -0,0 +1,5 @@
+
+from pygments.formatters import HtmlFormatter
+
+class HtmlFormatterWrapper(HtmlFormatter):
+ name = 'HtmlWrapper'
diff --git a/tests/support/python_lexer.py b/tests/support/python_lexer.py
index b1367715..f3085748 100644
--- a/tests/support/python_lexer.py
+++ b/tests/support/python_lexer.py
@@ -17,7 +17,6 @@ from pygments import unistring as uni
line_re = re.compile('.*?\n')
-
class CustomLexer(RegexLexer):
"""
For `Python <http://www.python.org>`_ source code.
@@ -224,3 +223,6 @@ class CustomLexer(RegexLexer):
return shebang_matches(text, r'pythonw?(2(\.\d)?)?') or \
'import ' in text[:1000]
+
+class LexerWrapper(CustomLexer):
+ name="PythonLexerWrapper" \ No newline at end of file
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
index 80055a43..6e2c917a 100644
--- a/tests/test_cmdline.py
+++ b/tests/test_cmdline.py
@@ -111,8 +111,27 @@ class CmdLineTest(unittest.TestCase):
os.unlink(name)
def test_load_from_file(self):
- o = self.check_success('-l', TESTDIR + '/support/python_lexer.py',
- '-f', 'html', '--load-from-file', stdin=TESTCODE)
+ lexer_file = os.path.join(TESTDIR, 'support', 'python_lexer.py')
+ formatter_file = os.path.join(TESTDIR, 'support', 'html_formatter.py')
+
+ # By default, use CustomLexer
+ o = self.check_success('-l', lexer_file, '-f', 'html',
+ '-x', stdin=TESTCODE)
+ o = re.sub('<[^>]*>', '', o)
+ # rstrip is necessary since HTML inserts a \n after the last </div>
+ self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
+ # If user specifies a name, use it
+ o = self.check_success('-f', 'html', '-x', '-l',
+ lexer_file + ':LexerWrapper', stdin=TESTCODE)
+ o = re.sub('<[^>]*>', '', o)
+ # rstrip is necessary since HTML inserts a \n after the last </div>
+ self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
+ # Should also work for formatters
+ o = self.check_success('-lpython', '-f',
+ formatter_file + ':HtmlFormatterWrapper',
+ '-x', stdin=TESTCODE)
o = re.sub('<[^>]*>', '', o)
# rstrip is necessary since HTML inserts a \n after the last </div>
self.assertEqual(o.rstrip(), TESTCODE.rstrip())
@@ -224,13 +243,13 @@ class CmdLineTest(unittest.TestCase):
# lexer file is missing/unreadable
e = self.check_failure('-l', 'nonexistent.py',
- '--load-from-file', TESTFILE)
+ '-x', TESTFILE)
self.assertTrue('Error: cannot read' in e)
# lexer file is malformed
e = self.check_failure('-l', 'support/empty.py',
- '--load-from-file', TESTFILE)
- self.assertTrue('Error: no CustomLexer class found' in e)
+ '-x', TESTFILE)
+ self.assertTrue('Error: no valid CustomLexer class found' in e)
# formatter not found
e = self.check_failure('-lpython', '-ffoo', TESTFILE)
@@ -246,13 +265,13 @@ class CmdLineTest(unittest.TestCase):
# formatter file is missing/unreadable
e = self.check_failure('-f', 'nonexistent.py',
- '--load-from-file', TESTFILE)
+ '-x', TESTFILE)
self.assertTrue('Error: cannot read' in e)
# formatter file is malformed
e = self.check_failure('-f', 'support/empty.py',
- '--load-from-file', TESTFILE)
- self.assertTrue('Error: no CustomFormatter class found' in e)
+ '-x', TESTFILE)
+ self.assertTrue('Error: no valid CustomFormatter class found' in e)
# output file not writable
e = self.check_failure('-o', os.path.join('nonexistent', 'dir', 'out.html'),