Merged in thomas_beale/pygments-main (pull request #483)

Add lexer and test files for the Archetype Definition Language (ADL) and sub-sytaxes.

17 files changed, 173 insertions, 89 deletions

diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py
index 89e2da5f..2e855570 100644
--- a/pygments/lexers/_mapping.py
+++ b/pygments/lexers/_mapping.py
@@ -128,7 +128,8 @@ LEXERS = {
     'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)),
     'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)),
     'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)),
-    'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f', '*.f90', '*.F', '*.F90'), ('text/x-fortran',)),
+    'FortranFixedLexer': ('pygments.lexers.fortran', 'FortranFixed', ('fortranfixed',), ('*.f', '*.F'), ()),
+    'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)),
     'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()),
     'GAPLexer': ('pygments.lexers.algebra', 'GAP', ('gap',), ('*.g', '*.gd', '*.gi', '*.gap'), ()),
     'GLShaderLexer': ('pygments.lexers.graphics', 'GLSL', ('glsl',), ('*.vert', '*.frag', '*.geo'), ('text/x-glslsrc',)),
@@ -144,7 +145,7 @@ LEXERS = {
     'GosuLexer': ('pygments.lexers.jvm', 'Gosu', ('gosu',), ('*.gs', '*.gsx', '*.gsp', '*.vark'), ('text/x-gosu',)),
     'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)),
     'GroffLexer': ('pygments.lexers.markup', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')),
-    'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy',), ('text/x-groovy',)),
+    'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy', '*.gradle'), ('text/x-groovy',)),
     'HamlLexer': ('pygments.lexers.html', 'Haml', ('haml',), ('*.haml',), ('text/x-haml',)),
     'HandlebarsHtmlLexer': ('pygments.lexers.templates', 'HTML+Handlebars', ('html+handlebars',), ('*.handlebars', '*.hbs'), ('text/html+handlebars', 'text/x-handlebars-template')),
     'HandlebarsLexer': ('pygments.lexers.templates', 'Handlebars', ('handlebars',), (), ()),
@@ -177,7 +178,7 @@ LEXERS = {
     'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')),
     'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')),
     'JavascriptGenshiLexer': ('pygments.lexers.templates', 'JavaScript+Genshi Text', ('js+genshitext', 'js+genshi', 'javascript+genshitext', 'javascript+genshi'), (), ('application/x-javascript+genshi', 'text/x-javascript+genshi', 'text/javascript+genshi')),
-    'JavascriptLexer': ('pygments.lexers.javascript', 'JavaScript', ('js', 'javascript'), ('*.js',), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')),
+    'JavascriptLexer': ('pygments.lexers.javascript', 'JavaScript', ('js', 'javascript'), ('*.js', '*.jsm'), ('application/javascript', 'application/x-javascript', 'text/x-javascript', 'text/javascript')),
     'JavascriptPhpLexer': ('pygments.lexers.templates', 'JavaScript+PHP', ('js+php', 'javascript+php'), (), ('application/x-javascript+php', 'text/x-javascript+php', 'text/javascript+php')),
     'JavascriptSmartyLexer': ('pygments.lexers.templates', 'JavaScript+Smarty', ('js+smarty', 'javascript+smarty'), (), ('application/x-javascript+smarty', 'text/x-javascript+smarty', 'text/javascript+smarty')),
     'JsonLdLexer': ('pygments.lexers.data', 'JSON-LD', ('jsonld', 'json-ld'), ('*.jsonld',), ('application/ld+json',)),
@@ -315,7 +316,7 @@ LEXERS = {
     'RstLexer': ('pygments.lexers.markup', 'reStructuredText', ('rst', 'rest', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst', 'text/prs.fallenstein.rst')),
     'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)),
     'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby'), ('text/x-ruby', 'application/x-ruby')),
-    'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs',), ('text/x-rustsrc',)),
+    'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs',), ('text/rust',)),
     'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')),
     'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')),
     'SassLexer': ('pygments.lexers.css', 'Sass', ('sass',), ('*.sass',), ('text/x-sass',)),
diff --git a/pygments/lexers/_stan_builtins.py b/pygments/lexers/_stan_builtins.py
index 0a225eba..6bf44574 100644
--- a/pygments/lexers/_stan_builtins.py
+++ b/pygments/lexers/_stan_builtins.py
@@ -4,7 +4,7 @@
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     This file contains the names of functions for Stan used by
-    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.5.0.
+    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.7.0
 
     :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
@@ -35,6 +35,7 @@ TYPES = (
     'positive_ordered',
     'real',
     'row_vector',
+    'row_vectormatrix',
     'simplex',
     'unit_vector',
     'vector',
@@ -200,7 +201,6 @@ FUNCTIONS = (
     'lkj_corr_cholesky_rng',
     'lkj_corr_log',
     'lkj_corr_rng',
-    'lkj_cov_log',
     'lmgamma',
     'log',
     'log10',
@@ -214,6 +214,7 @@ FUNCTIONS = (
     'log_diff_exp',
     'log_falling_factorial',
     'log_inv_logit',
+    'log_mix',
     'log_rising_factorial',
     'log_softmax',
     'log_sum_exp',
@@ -236,6 +237,7 @@ FUNCTIONS = (
     'min',
     'modified_bessel_first_kind',
     'modified_bessel_second_kind',
+    'multi_gp_cholesky_log',
     'multi_gp_log',
     'multi_normal_cholesky_log',
     'multi_normal_cholesky_rng',
@@ -248,6 +250,9 @@ FUNCTIONS = (
     'multinomial_rng',
     'multiply_log',
     'multiply_lower_tri_self_transpose',
+    'neg_binomial_2_ccdf_log',
+    'neg_binomial_2_cdf',
+    'neg_binomial_2_cdf_log',
     'neg_binomial_2_log',
     'neg_binomial_2_log_log',
     'neg_binomial_2_log_rng',
@@ -284,6 +289,7 @@ FUNCTIONS = (
     'poisson_cdf_log',
     'poisson_log',
     'poisson_log_log',
+    'poisson_log_rng',
     'poisson_rng',
     'positive_infinity',
     'pow',
@@ -371,6 +377,7 @@ FUNCTIONS = (
     'weibull_cdf_log',
     'weibull_log',
     'weibull_rng',
+    'wiener_log',
     'wishart_log',
     'wishart_rng'
 )
@@ -400,10 +407,10 @@ DISTRIBUTIONS = (
     'inv_wishart',
     'lkj_corr',
     'lkj_corr_cholesky',
-    'lkj_cov',
     'logistic',
     'lognormal',
     'multi_gp',
+    'multi_gp_cholesky',
     'multi_normal',
     'multi_normal_cholesky',
     'multi_normal_prec',
@@ -425,6 +432,7 @@ DISTRIBUTIONS = (
     'uniform',
     'von_mises',
     'weibull',
+    'wiener',
     'wishart'
 )
 
diff --git a/pygments/lexers/c_cpp.py b/pygments/lexers/c_cpp.py
index bbac406e..35ea517f 100644
--- a/pygments/lexers/c_cpp.py
+++ b/pygments/lexers/c_cpp.py
@@ -28,8 +28,10 @@ class CFamilyLexer(RegexLexer):
 
     #: optional Comment or Whitespace
     _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+'
+
+    # The trailing ?, rather than *, avoids a geometric performance drop here.
     #: only one /* */ style comment
-    _ws1 = r'\s*(?:/[*].*?[*]/\s*)*'
+    _ws1 = r'\s*(?:/[*].*?[*]/\s*)?'
 
     tokens = {
         'whitespace': [
diff --git a/pygments/lexers/chapel.py b/pygments/lexers/chapel.py
index 520be37b..6fb6920c 100644
--- a/pygments/lexers/chapel.py
+++ b/pygments/lexers/chapel.py
@@ -46,9 +46,10 @@ class ChapelLexer(RegexLexer):
                 'continue', 'delete', 'dmapped', 'do', 'domain', 'else', 'enum',
                 'export', 'extern', 'for', 'forall', 'if', 'index', 'inline',
                 'iter', 'label', 'lambda', 'let', 'local', 'new', 'noinit', 'on',
-                'otherwise', 'pragma', 'reduce', 'return', 'scan', 'select',
-                'serial', 'single', 'sparse', 'subdomain', 'sync', 'then', 'use',
-                'when', 'where', 'while', 'with', 'yield', 'zip'), suffix=r'\b'),
+                'otherwise', 'pragma', 'private', 'public', 'reduce', 'return',
+                'scan', 'select', 'serial', 'single', 'sparse', 'subdomain',
+                'sync', 'then', 'use', 'when', 'where', 'while', 'with', 'yield',
+                'zip'), suffix=r'\b'),
              Keyword),
             (r'(proc)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'procname'),
             (r'(class|module|record|union)(\s+)', bygroups(Keyword, Text),
diff --git a/pygments/lexers/css.py b/pygments/lexers/css.py
index c11e7ec0..6f27d63c 100644
--- a/pygments/lexers/css.py
+++ b/pygments/lexers/css.py
@@ -484,6 +484,7 @@ class ScssLexer(RegexLexer):
             (r'[^\s:="\[]+', Name.Attribute),
             (r'#\{', String.Interpol, 'interpolation'),
             (r'[ \t]*:', Operator, 'value'),
+            default('#pop'),
         ],
 
         'inline-comment': [
diff --git a/pygments/lexers/fortran.py b/pygments/lexers/fortran.py
index 8ba54aff..df3fed4f 100644
--- a/pygments/lexers/fortran.py
+++ b/pygments/lexers/fortran.py
@@ -11,11 +11,11 @@
 
 import re
 
-from pygments.lexer import RegexLexer, include, words
+from pygments.lexer import RegexLexer, bygroups, include, words, using
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
-    Number, Punctuation
+    Number, Punctuation, Generic
 
-__all__ = ['FortranLexer']
+__all__ = ['FortranLexer', 'FortranFixedLexer']
 
 
 class FortranLexer(RegexLexer):
@@ -26,7 +26,7 @@ class FortranLexer(RegexLexer):
     """
     name = 'Fortran'
     aliases = ['fortran']
-    filenames = ['*.f', '*.f90', '*.F', '*.F90']
+    filenames = ['*.f03', '*.f90', '*.F03', '*.F90']
     mimetypes = ['text/x-fortran']
     flags = re.IGNORECASE | re.MULTILINE
 
@@ -159,3 +159,45 @@ class FortranLexer(RegexLexer):
             (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float),
         ],
     }
+
+
+class FortranFixedLexer(RegexLexer):
+    """
+    Lexer for fixed format Fortran.
+    """
+    name = 'FortranFixed'
+    aliases = ['fortranfixed']
+    filenames = ['*.f', '*.F']
+
+    flags  = re.IGNORECASE
+
+    def _lex_fortran(self, match, ctx=None):
+        """Lex a line just as free form fortran without line break."""
+        lexer = FortranLexer()
+        text = match.group(0) + "\n"
+        for index, token, value in lexer.get_tokens_unprocessed(text):
+            value = value.replace('\n', '')
+            if value != '':
+                yield index, token, value
+
+    tokens = {
+        'root': [
+            (r'[C*].*\n', Comment),
+            (r'#.*\n', Comment.Preproc),
+            (r' {0,4}!.*\n', Comment),
+            (r'(.{5})', Name.Label, 'cont-char'),
+            (r'.*\n', using(FortranLexer)),
+        ],
+
+        'cont-char': [
+            (' ', Text, 'code'),
+            ('0', Comment, 'code'),
+            ('.', Generic.Strong, 'code')
+        ],
+
+        'code': [
+            (r'(.{66})(.*)(\n)',
+             bygroups(_lex_fortran, Comment, Text), 'root'),
+            (r'(.*)(\n)', bygroups(_lex_fortran, Text), 'root'),
+            (r'', Text, 'root')]
+    }
diff --git a/pygments/lexers/javascript.py b/pygments/lexers/javascript.py
index aed8438e..7dcfbb4b 100644
--- a/pygments/lexers/javascript.py
+++ b/pygments/lexers/javascript.py
@@ -36,7 +36,7 @@ class JavascriptLexer(RegexLexer):
 
     name = 'JavaScript'
     aliases = ['js', 'javascript']
-    filenames = ['*.js', ]
+    filenames = ['*.js', '*.jsm', ]
     mimetypes = ['application/javascript', 'application/x-javascript',
                  'text/x-javascript', 'text/javascript', ]
 
diff --git a/pygments/lexers/julia.py b/pygments/lexers/julia.py
index 1b7d543a..1304b395 100644
--- a/pygments/lexers/julia.py
+++ b/pygments/lexers/julia.py
@@ -30,6 +30,8 @@ class JuliaLexer(RegexLexer):
     filenames = ['*.jl']
     mimetypes = ['text/x-julia', 'application/x-julia']
 
+    flags = re.MULTILINE | re.UNICODE
+
     builtins = [
         'exit', 'whos', 'edit', 'load', 'is', 'isa', 'isequal', 'typeof', 'tuple',
         'ntuple', 'uid', 'hash', 'finalizer', 'convert', 'promote', 'subtype',
@@ -89,7 +91,7 @@ class JuliaLexer(RegexLexer):
 
             # names
             (r'@[\w.]+', Name.Decorator),
-            (r'[a-zA-Z_]\w*', Name),
+            (u'[a-zA-Z_\u00A1-\U0010FFFF][a-zA-Z_0-9\u00A1-\U0010FFFF]*!*', Name),
 
             # numbers
             (r'(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?', Number.Float),
diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py
index 6b302c7e..4d3c9159 100644
--- a/pygments/lexers/jvm.py
+++ b/pygments/lexers/jvm.py
@@ -457,7 +457,7 @@ class GroovyLexer(RegexLexer):
 
     name = 'Groovy'
     aliases = ['groovy']
-    filenames = ['*.groovy']
+    filenames = ['*.groovy','*.gradle']
     mimetypes = ['text/x-groovy']
 
     flags = re.MULTILINE | re.DOTALL
diff --git a/pygments/lexers/modeling.py b/pygments/lexers/modeling.py
index 43194436..ec99543f 100644
--- a/pygments/lexers/modeling.py
+++ b/pygments/lexers/modeling.py
@@ -284,8 +284,8 @@ class StanLexer(RegexLexer):
     """Pygments Lexer for Stan models.
 
     The Stan modeling language is specified in the *Stan Modeling Language
-    User's Guide and Reference Manual, v2.5.0*,
-    `pdf <https://github.com/stan-dev/stan/releases/download/v2.5.0/stan-reference-2.5.0.pdf>`__.
+    User's Guide and Reference Manual, v2.7.0*,
+    `pdf <https://github.com/stan-dev/stan/releases/download/v2.7.0/stan-reference-2.7.0.pdf>`__.
 
     .. versionadded:: 1.6
     """
diff --git a/pygments/lexers/python.py b/pygments/lexers/python.py
index 49c15b6d..ea97b855 100644
--- a/pygments/lexers/python.py
+++ b/pygments/lexers/python.py
@@ -215,10 +215,10 @@ class Python3Lexer(RegexLexer):
     tokens = PythonLexer.tokens.copy()
     tokens['keywords'] = [
         (words((
-            'assert', 'break', 'continue', 'del', 'elif', 'else', 'except',
-            'finally', 'for', 'global', 'if', 'lambda', 'pass', 'raise',
-            'nonlocal', 'return', 'try', 'while', 'yield', 'yield from', 'as',
-            'with'), suffix=r'\b'),
+            'assert', 'async', 'await', 'break', 'continue', 'del', 'elif',
+            'else', 'except', 'finally', 'for', 'global', 'if', 'lambda', 'pass',
+            'raise', 'nonlocal', 'return', 'try', 'while', 'yield', 'yield from',
+            'as', 'with'), suffix=r'\b'),
          Keyword),
         (words((
             'True', 'False', 'None'), suffix=r'\b'),
@@ -272,6 +272,7 @@ class Python3Lexer(RegexLexer):
     tokens['backtick'] = []
     tokens['name'] = [
         (r'@\w+', Name.Decorator),
+        (r'@', Operator),  # new matrix multiplication operator
         (uni_name, Name),
     ]
     tokens['funcname'] = [
diff --git a/pygments/lexers/robotframework.py b/pygments/lexers/robotframework.py
index 56996fa8..eab06efe 100644
--- a/pygments/lexers/robotframework.py
+++ b/pygments/lexers/robotframework.py
@@ -87,7 +87,7 @@ class RobotFrameworkLexer(Lexer):
 class VariableTokenizer(object):
 
     def tokenize(self, string, token):
-        var = VariableSplitter(string, identifiers='$@%')
+        var = VariableSplitter(string, identifiers='$@%&')
         if var.start < 0 or token in (COMMENT, ERROR):
             yield string, token
             return
@@ -205,7 +205,7 @@ class Tokenizer(object):
     def _is_assign(self, value):
         if value.endswith('='):
             value = value[:-1].strip()
-        var = VariableSplitter(value, identifiers='$@')
+        var = VariableSplitter(value, identifiers='$@&')
         return var.start == 0 and var.end == len(value)
 
 
@@ -262,7 +262,7 @@ class TestCaseSetting(Setting):
 
 class KeywordSetting(TestCaseSetting):
     _keyword_settings = ('teardown',)
-    _other_settings = ('documentation', 'arguments', 'return', 'timeout')
+    _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags')
 
 
 class Variable(Tokenizer):
@@ -465,13 +465,13 @@ class VariableSplitter:
         self.identifier = self._variable_chars[0]
         self.base = ''.join(self._variable_chars[2:-1])
         self.end = self.start + len(self._variable_chars)
-        if self._has_list_variable_index():
-            self.index = ''.join(self._list_variable_index_chars[1:-1])
-            self.end += len(self._list_variable_index_chars)
+        if self._has_list_or_dict_variable_index():
+            self.index = ''.join(self._list_and_dict_variable_index_chars[1:-1])
+            self.end += len(self._list_and_dict_variable_index_chars)
 
-    def _has_list_variable_index(self):
-        return self._list_variable_index_chars\
-        and self._list_variable_index_chars[-1] == ']'
+    def _has_list_or_dict_variable_index(self):
+        return self._list_and_dict_variable_index_chars\
+        and self._list_and_dict_variable_index_chars[-1] == ']'
 
     def _split(self, string):
         start_index, max_index = self._find_variable(string)
@@ -479,7 +479,7 @@ class VariableSplitter:
         self._open_curly = 1
         self._state = self._variable_state
         self._variable_chars = [string[start_index], '{']
-        self._list_variable_index_chars = []
+        self._list_and_dict_variable_index_chars = []
         self._string = string
         start_index += 2
         for index, char in enumerate(string[start_index:]):
@@ -530,14 +530,14 @@ class VariableSplitter:
         if char == '}' and not self._is_escaped(self._string, index):
             self._open_curly -= 1
             if self._open_curly == 0:
-                if not self._is_list_variable():
+                if not self._is_list_or_dict_variable():
                     raise StopIteration
                 self._state = self._waiting_list_variable_index_state
         elif char in self._identifiers:
             self._state = self._internal_variable_start_state
 
-    def _is_list_variable(self):
-        return self._variable_chars[0] == '@'
+    def _is_list_or_dict_variable(self):
+        return self._variable_chars[0] in ('@','&')
 
     def _internal_variable_start_state(self, char, index):
         self._state = self._variable_state
@@ -551,10 +551,10 @@ class VariableSplitter:
     def _waiting_list_variable_index_state(self, char, index):
         if char != '[':
             raise StopIteration
-        self._list_variable_index_chars.append(char)
+        self._list_and_dict_variable_index_chars.append(char)
         self._state = self._list_variable_index_state
 
     def _list_variable_index_state(self, char, index):
-        self._list_variable_index_chars.append(char)
+        self._list_and_dict_variable_index_chars.append(char)
         if char == ']':
             raise StopIteration
diff --git a/pygments/lexers/rust.py b/pygments/lexers/rust.py
index 2ca860d6..d8939678 100644
--- a/pygments/lexers/rust.py
+++ b/pygments/lexers/rust.py
@@ -10,7 +10,7 @@
 """
 
 from pygments.lexer import RegexLexer, include, bygroups, words, default
-from pygments.token import Comment, Operator, Keyword, Name, String, \
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation, Whitespace
 
 __all__ = ['RustLexer']
@@ -18,33 +18,40 @@ __all__ = ['RustLexer']
 
 class RustLexer(RegexLexer):
     """
-    Lexer for the Rust programming language (version 0.9).
+    Lexer for the Rust programming language (version 1.0).
 
     .. versionadded:: 1.6
     """
     name = 'Rust'
     filenames = ['*.rs']
     aliases = ['rust']
-    mimetypes = ['text/x-rustsrc']
+    mimetypes = ['text/rust']
 
     tokens = {
         'root': [
+            # rust allows a file to start with a shebang, but if the first line
+            # starts with #![ then it’s not a shebang but a crate attribute.
+            (r'#![^[\r\n].*$', Comment.Preproc),
+            default('base'),
+        ],
+        'base': [
             # Whitespace and Comments
             (r'\n', Whitespace),
             (r'\s+', Whitespace),
-            (r'//[/!](.*?)\n', Comment.Doc),
+            (r'//!.*?\n', String.Doc),
+            (r'///(\n|[^/].*?\n)', String.Doc),
             (r'//(.*?)\n', Comment.Single),
+            (r'/\*\*(\n|[^/*])', String.Doc, 'doccomment'),
+            (r'/\*!', String.Doc, 'doccomment'),
             (r'/\*', Comment.Multiline, 'comment'),
 
-            # Lifetime
-            (r"""'[a-zA-Z_]\w*""", Name.Label),
             # Macro parameters
             (r"""\$([a-zA-Z_]\w*|\(,?|\),?|,?)""", Comment.Preproc),
             # Keywords
             (words((
-                'as', 'box', 'break', 'continue', 'do', 'else', 'enum', 'extern',
+                'as', 'box', 'crate', 'do', 'else', 'enum', 'extern',  # break and continue are in labels
                 'fn', 'for', 'if', 'impl', 'in', 'loop', 'match', 'mut', 'priv',
-                'proc', 'pub', 'ref', 'return', 'static', '\'static', 'struct',
+                'proc', 'pub', 'ref', 'return', 'static', 'struct',
                 'trait', 'true', 'type', 'unsafe', 'while'), suffix=r'\b'),
              Keyword),
             (words(('alignof', 'be', 'const', 'offsetof', 'pure', 'sizeof',
@@ -53,44 +60,45 @@ class RustLexer(RegexLexer):
             (r'(mod|use)\b', Keyword.Namespace),
             (r'(true|false)\b', Keyword.Constant),
             (r'let\b', Keyword.Declaration),
-            (words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64', 'uint',
-                    'int', 'f32', 'f64', 'str', 'bool'), suffix=r'\b'),
+            (words(('u8', 'u16', 'u32', 'u64', 'i8', 'i16', 'i32', 'i64', 'usize',
+                    'isize', 'f32', 'f64', 'str', 'bool'), suffix=r'\b'),
              Keyword.Type),
             (r'self\b', Name.Builtin.Pseudo),
-            # Prelude
+            # Prelude (taken from Rust’s src/libstd/prelude.rs)
             (words((
-                'Freeze', 'Pod', 'Send', 'Sized', 'Add', 'Sub', 'Mul', 'Div', 'Rem', 'Neg', 'Not', 'BitAnd',
-                'BitOr', 'BitXor', 'Drop', 'Shl', 'Shr', 'Index', 'Option', 'Some', 'None', 'Result',
-                'Ok', 'Err', 'from_str', 'range', 'print', 'println', 'Any', 'AnyOwnExt', 'AnyRefExt',
-                'AnyMutRefExt', 'Ascii', 'AsciiCast', 'OnwedAsciiCast', 'AsciiStr',
-                'IntoBytes', 'Bool', 'ToCStr', 'Char', 'Clone', 'DeepClone', 'Eq', 'ApproxEq',
-                'Ord', 'TotalEq', 'Ordering', 'Less', 'Equal', 'Greater', 'Equiv', 'Container',
-                'Mutable', 'Map', 'MutableMap', 'Set', 'MutableSet', 'Default', 'FromStr',
-                'Hash', 'FromIterator', 'Extendable', 'Iterator', 'DoubleEndedIterator',
-                'RandomAccessIterator', 'CloneableIterator', 'OrdIterator',
-                'MutableDoubleEndedIterator', 'ExactSize', 'Times', 'Algebraic',
-                'Trigonometric', 'Exponential', 'Hyperbolic', 'Bitwise', 'BitCount',
-                'Bounded', 'Integer', 'Fractional', 'Real', 'RealExt', 'Num', 'NumCast',
-                'CheckedAdd', 'CheckedSub', 'CheckedMul', 'Orderable', 'Signed',
-                'Unsigned', 'Round', 'Primitive', 'Int', 'Float', 'ToStrRadix',
-                'ToPrimitive', 'FromPrimitive', 'GenericPath', 'Path', 'PosixPath',
-                'WindowsPath', 'RawPtr', 'Buffer', 'Writer', 'Reader', 'Seek',
-                'SendStr', 'SendStrOwned', 'SendStrStatic', 'IntoSendStr', 'Str',
-                'StrVector', 'StrSlice', 'OwnedStr', 'IterBytes', 'ToStr', 'IntoStr',
-                'CopyableTuple', 'ImmutableTuple', 'ImmutableEqVector', 'ImmutableTotalOrdVector',
-                'ImmutableCopyableVector', 'OwnedVector', 'OwnedCopyableVector',
-                'OwnedEqVector', 'MutableVector', 'MutableTotalOrdVector',
-                'Vector', 'VectorVector', 'CopyableVector', 'ImmutableVector',
-                'Port', 'Chan', 'SharedChan', 'spawn', 'drop'), suffix=r'\b'),
+                # Reexported core operators
+                'Copy', 'Send', 'Sized', 'Sync',
+                'Drop', 'Fn', 'FnMut', 'FnOnce',
+
+                # Reexported functions
+                'drop',
+
+                # Reexported types and traits
+                'Box',
+                'ToOwned',
+                'Clone',
+                'PartialEq', 'PartialOrd', 'Eq', 'Ord',
+                'AsRef', 'AsMut', 'Into', 'From',
+                'Default',
+                'Iterator', 'Extend', 'IntoIterator',
+                'DoubleEndedIterator', 'ExactSizeIterator',
+                'Option',
+                'Some', 'None',
+                'Result',
+                'Ok', 'Err',
+                'SliceConcatExt',
+                'String', 'ToString',
+                'Vec',
+                ), suffix=r'\b'),
              Name.Builtin),
-            (r'(ImmutableTuple\d+|Tuple\d+)\b', Name.Builtin),
-            # Borrowed pointer
-            (r'(&)(\'[A-Za-z_]\w*)?', bygroups(Operator, Name)),
             # Labels
-            (r'\'[A-Za-z_]\w*:', Name.Label),
+            (r'(break|continue)(\s*)(\'[A-Za-z_]\w*)?', bygroups(Keyword, Text.Whitespace, Name.Label)),
             # Character Literal
-            (r"""'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}"""
-             r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|.)'""",
+            (r"""'(\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""",
+             String.Char),
+            (r"""b'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}|.)'""",
              String.Char),
             # Binary Literal
             (r'0b[01_]+', Number.Bin, 'number_lit'),
@@ -100,11 +108,16 @@ class RustLexer(RegexLexer):
             (r'0[xX][0-9a-fA-F_]+', Number.Hex, 'number_lit'),
             # Decimal Literal
             (r'[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|'
-             r'\.[0-9_]*|[eE][+\-]?[0-9_]+)', Number.Float, 'number_lit'),
+             r'\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)', Number.Float, 'number_lit'),
             (r'[0-9][0-9_]*', Number.Integer, 'number_lit'),
             # String Literal
+            (r'b"', String, 'bytestring'),
             (r'"', String, 'string'),
-            (r'r(#*)".*?"\1', String.Raw),
+            (r'b?r(#*)".*?"\1', String),
+
+            # Lifetime
+            (r"""'static""", Name.Builtin),
+            (r"""'[a-zA-Z_]\w*""", Name.Attribute),
 
             # Operators and Punctuation
             (r'[{}()\[\],.;]', Punctuation),
@@ -129,18 +142,28 @@ class RustLexer(RegexLexer):
             (r'\*/', Comment.Multiline, '#pop'),
             (r'[*/]', Comment.Multiline),
         ],
+        'doccomment': [
+            (r'[^*/]+', String.Doc),
+            (r'/\*', String.Doc, '#push'),
+            (r'\*/', String.Doc, '#pop'),
+            (r'[*/]', String.Doc),
+        ],
         'number_lit': [
-            (r'[ui](8|16|32|64)', Keyword, '#pop'),
+            (r'[ui](8|16|32|64|size)', Keyword, '#pop'),
             (r'f(32|64)', Keyword, '#pop'),
             default('#pop'),
         ],
         'string': [
             (r'"', String, '#pop'),
-            (r"""\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}"""
-             r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}""", String.Escape),
+            (r"""\\['"\\nrt]|\\x[0-7][0-9a-fA-F]|\\0"""
+             r"""|\\u\{[0-9a-fA-F]{1,6}\}""", String.Escape),
             (r'[^\\"]+', String),
             (r'\\', String),
         ],
+        'bytestring': [
+            (r"""\\x[89a-fA-F][0-9a-fA-F]""", String.Escape),
+            include('string'),
+        ],
         'macro{': [
             (r'\{', Operator, '#push'),
             (r'\}', Operator, '#pop'),
diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py
index a9c1e6b9..1e3640bf 100644
--- a/pygments/lexers/shell.py
+++ b/pygments/lexers/shell.py
@@ -47,7 +47,7 @@ class BashLexer(RegexLexer):
             (r'\$\(\(', Keyword, 'math'),
             (r'\$\(', Keyword, 'paren'),
             (r'\$\{#?', String.Interpol, 'curly'),
-            (r'\$#?(\w+|.)', Name.Variable),
+            (r'\$(\w+|.)', Name.Variable),
         ],
         'basic': [
             (r'\b(if|fi|else|while|do|done|for|then|return|function|case|'
diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py
index 3cb73059..bfca0d38 100644
--- a/pygments/lexers/templates.py
+++ b/pygments/lexers/templates.py
@@ -369,7 +369,7 @@ class DjangoLexer(RegexLexer):
              r'with(?:(?:out)?\s*context)?|scoped|ignore\s+missing)\b',
              Keyword),
             (r'(loop|block|super|forloop)\b', Name.Builtin),
-            (r'[a-zA-Z][\w-]*', Name.Variable),
+            (r'[a-zA-Z_][\w-]*', Name.Variable),
             (r'\.\w+', Name.Variable),
             (r':?"(\\\\|\\"|[^"])*"', String.Double),
             (r":?'(\\\\|\\'|[^'])*'", String.Single),
diff --git a/pygments/lexers/testing.py b/pygments/lexers/testing.py
index 55f4c054..4a91c5b1 100644
--- a/pygments/lexers/testing.py
+++ b/pygments/lexers/testing.py
@@ -27,9 +27,9 @@ class GherkinLexer(RegexLexer):
     mimetypes = ['text/x-gherkin']
 
     feature_keywords         = u'^(기능|機能|功能|フィーチャ|خاصية|תכונה|Функціонал|Функционалност|Функционал|Фича|Особина|Могућност|Özellik|Właściwość|Tính năng|Trajto|Savybė|Požiadavka|Požadavek|Osobina|Ominaisuus|Omadus|OH HAI|Mogućnost|Mogucnost|Jellemző|Fīča|Funzionalità|Funktionalität|Funkcionalnost|Funkcionalitāte|Funcționalitate|Functionaliteit|Functionalitate|Funcionalitat|Funcionalidade|Fonctionnalité|Fitur|Feature|Egenskap|Egenskab|Crikey|Característica|Arwedd)(:)(.*)$'
-    feature_element_keywords = u'^(\\s*)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Тарих|Сценарій|Сценарио|Сценарий структураси|Сценарий|Структура сценарію|Структура сценарија|Структура сценария|Скица|Рамка на сценарий|Пример|Предыстория|Предистория|Позадина|Передумова|Основа|Концепт|Контекст|Założenia|Wharrimean is|Tình huống|The thing of it is|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|Situācija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenaro|Scenariusz|Scenariul de şablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus šablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|Pozadí|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Konturo de la scenaro|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Geçmiş|Forgatókönyv vázlat|Forgatókönyv|Fono|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dis is what went down|Dasar|Contexto|Contexte|Contesto|Condiţii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$'
+    feature_element_keywords = u'^(\\s*)(시나리오 개요|시나리오|배경|背景|場景大綱|場景|场景大纲|场景|劇本大綱|劇本|剧本大纲|剧本|テンプレ|シナリオテンプレート|シナリオテンプレ|シナリオアウトライン|シナリオ|سيناريو مخطط|سيناريو|الخلفية|תרחיש|תבנית תרחיש|רקע|Тарих|Сценарій|Сценарио|Сценарий структураси|Сценарий|Структура сценарію|Структура сценарија|Структура сценария|Скица|Рамка на сценарий|Пример|Предыстория|Предистория|Позадина|Передумова|Основа|Концепт|Контекст|Założenia|Wharrimean is|Tình huống|The thing of it is|Tausta|Taust|Tapausaihio|Tapaus|Szenariogrundriss|Szenario|Szablon scenariusza|Stsenaarium|Struktura scenarija|Skica|Skenario konsep|Skenario|Situācija|Senaryo taslağı|Senaryo|Scénář|Scénario|Schema dello scenario|Scenārijs pēc parauga|Scenārijs|Scenár|Scenaro|Scenariusz|Scenariul de şablon|Scenariul de sablon|Scenariu|Scenario Outline|Scenario Amlinellol|Scenario|Scenarijus|Scenarijaus šablonas|Scenarij|Scenarie|Rerefons|Raamstsenaarium|Primer|Pozadí|Pozadina|Pozadie|Plan du scénario|Plan du Scénario|Osnova scénáře|Osnova|Náčrt Scénáře|Náčrt Scenáru|Mate|MISHUN SRSLY|MISHUN|Kịch bản|Konturo de la scenaro|Kontext|Konteksts|Kontekstas|Kontekst|Koncept|Khung tình huống|Khung kịch bản|Háttér|Grundlage|Geçmiş|Forgatókönyv vázlat|Forgatókönyv|Fono|Esquema do Cenário|Esquema do Cenario|Esquema del escenario|Esquema de l\'escenari|Escenario|Escenari|Dis is what went down|Dasar|Contexto|Contexte|Contesto|Condiţii|Conditii|Cenário|Cenario|Cefndir|Bối cảnh|Blokes|Bakgrunn|Bakgrund|Baggrund|Background|B4|Antecedents|Antecedentes|All y\'all|Achtergrond|Abstrakt Scenario|Abstract Scenario)(:)(.*)$'
     examples_keywords        = u'^(\\s*)(예|例子|例|サンプル|امثلة|דוגמאות|Сценарији|Примери|Приклади|Мисоллар|Значения|Örnekler|Voorbeelden|Variantai|Tapaukset|Scenarios|Scenariji|Scenarijai|Příklady|Példák|Príklady|Przykłady|Primjeri|Primeri|Piemēri|Pavyzdžiai|Paraugs|Juhtumid|Exemplos|Exemples|Exemplele|Exempel|Examples|Esempi|Enghreifftiau|Ekzemploj|Eksempler|Ejemplos|EXAMPLZ|Dữ liệu|Contoh|Cobber|Beispiele)(:)(.*)$'
-    step_keywords            = u'^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假如|但是|但し|並且|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\* )'
+    step_keywords            = u'^(\\s*)(하지만|조건|먼저|만일|만약|단|그리고|그러면|那麼|那么|而且|當|当|前提|假設|假设|假如|假定|但是|但し|並且|并且|同時|同时|もし|ならば|ただし|しかし|かつ|و |متى |لكن |عندما |ثم |بفرض |اذاً |כאשר |וגם |בהינתן |אזי |אז |אבל |Якщо |Унда |То |Припустимо, що |Припустимо |Онда |Но |Нехай |Лекин |Когато |Када |Кад |К тому же |И |Задато |Задати |Задате |Если |Допустим |Дадено |Ва |Бирок |Аммо |Али |Але |Агар |А |І |Și |És |Zatati |Zakładając |Zadato |Zadate |Zadano |Zadani |Zadan |Youse know when youse got |Youse know like when |Yna |Ya know how |Ya gotta |Y |Wun |Wtedy |When y\'all |When |Wenn |WEN |Và |Ve |Und |Un |Thì |Then y\'all |Then |Tapi |Tak |Tada |Tad |Så |Stel |Soit |Siis |Si |Sed |Se |Quando |Quand |Quan |Pryd |Pokud |Pokiaľ |Però |Pero |Pak |Oraz |Onda |Ond |Oletetaan |Og |Och |O zaman |Når |När |Niin |Nhưng |N |Mutta |Men |Mas |Maka |Majd |Mais |Maar |Ma |Lorsque |Lorsqu\'|Kun |Kuid |Kui |Khi |Keď |Ketika |Když |Kaj |Kai |Kada |Kad |Jeżeli |Ja |Ir |I CAN HAZ |I |Ha |Givun |Givet |Given y\'all |Given |Gitt |Gegeven |Gegeben sei |Fakat |Eğer ki |Etant donné |Et |Então |Entonces |Entao |En |Eeldades |E |Duota |Dun |Donitaĵo |Donat |Donada |Do |Diyelim ki |Dengan |Den youse gotta |De |Dato |Dar |Dann |Dan |Dado |Dacă |Daca |DEN |Când |Cuando |Cho |Cept |Cand |Cal |But y\'all |But |Buh |Biết |Bet |BUT |Atès |Atunci |Atesa |Anrhegedig a |Angenommen |And y\'all |And |An |Ama |Als |Alors |Allora |Ali |Aleshores |Ale |Akkor |Aber |AN |A také |A |\* )'
 
     tokens = {
         'comments': [
diff --git a/pygments/modeline.py b/pygments/modeline.py
index 54df90c4..2200f1cf 100644
--- a/pygments/modeline.py
+++ b/pygments/modeline.py
@@ -13,16 +13,19 @@ import re
 
 __all__ = ['get_filetype_from_buffer']
 
+
 modeline_re = re.compile(r'''
     (?: vi | vim | ex ) (?: [<=>]? \d* )? :
     .* (?: ft | filetype | syn | syntax ) = ( [^:\s]+ )
 ''', re.VERBOSE)
 
+
 def get_filetype_from_line(l):
     m = modeline_re.search(l)
     if m:
         return m.group(1)
 
+
 def get_filetype_from_buffer(buf, max_lines=5):
     """
     Scan the buffer for modelines and return filetype if one is found.
@@ -32,7 +35,7 @@ def get_filetype_from_buffer(buf, max_lines=5):
         ret = get_filetype_from_line(l)
         if ret:
             return ret
-    for l in lines[max_lines:0:-1]:
+    for l in lines[max_lines:-1:-1]:
         ret = get_filetype_from_line(l)
         if ret:
             return ret