Merged birkenfeld/pygments-main into default

3 files changed, 179 insertions, 15 deletions

diff --git a/pygments/lexers/praat.py b/pygments/lexers/praat.py
index 9255216d..6679d3db 100644
--- a/pygments/lexers/praat.py
+++ b/pygments/lexers/praat.py
@@ -168,8 +168,8 @@ class PraatLexer(RegexLexer):
         ],
         'function_call': [
             (words(functions_string, suffix=r'\$(?=\s*[:(])'), Name.Function, 'function'),
-            (words(functions_array, suffix=r'#(?=\s*[:(])'), Name.Function, 'function'),
-            (words(functions_numeric, suffix=r'(?=\s*[:(])'), Name.Function, 'function'),
+            (words(functions_array, suffix=r'#(?=\s*[:(])'),   Name.Function, 'function'),
+            (words(functions_numeric, suffix=r'(?=\s*[:(])'),  Name.Function, 'function'),
         ],
         'function': [
             (r'\s+',   Text),
@@ -190,6 +190,7 @@ class PraatLexer(RegexLexer):
             include('operator'),
             include('number'),
 
+            (r'[()]', Text),
             (r',', Punctuation),
         ],
         'old_arguments': [
@@ -203,6 +204,7 @@ class PraatLexer(RegexLexer):
             (r'[^\n]', Text),
         ],
         'number': [
+            (r'\n', Text, '#pop'),
             (r'\b\d+(\.\d*)?([eE][-+]?\d+)?%?', Number),
         ],
         'object_attributes': [
@@ -235,8 +237,8 @@ class PraatLexer(RegexLexer):
             (r"'(?=.*')", String.Interpol, 'string_interpolated'),
         ],
         'operator': [
-            (r'([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)', Operator),
-            (r'\b(and|or|not|div|mod)\b',          Operator.Word),
+            (r'([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)',       Operator),
+            (r'(?<![\w.])(and|or|not|div|mod)(?![\w.])', Operator.Word),
         ],
         'string_interpolated': [
             (r'\.?[_a-z][\w.]*[$#]?(?:\[[a-zA-Z0-9,]+\])?(:[0-9]+)?',
@@ -267,9 +269,6 @@ class PraatLexer(RegexLexer):
              bygroups(Keyword, Text), 'number'),
 
             (r'(option|button)([ \t]+)',
-             bygroups(Keyword, Text), 'number'),
-
-            (r'(option|button)([ \t]+)',
              bygroups(Keyword, Text), 'string_unquoted'),
 
             (r'(sentence|text)([ \t]+\S+)',
diff --git a/tests/examplefiles/example.praat b/tests/examplefiles/example.praat
index bf2d005f..85573919 100644
--- a/tests/examplefiles/example.praat
+++ b/tests/examplefiles/example.praat
@@ -2,17 +2,27 @@ form Highlighter test
   sentence Blank
   sentence My_sentence This should all be a string
   text My_text This should also all be a string
-  word My_word Only the first word is a string, the rest is invalid
+  word My_word Only the first word is a string, the rest is discarded
   boolean Binary 1
   boolean Text no
   boolean Quoted "yes"
   comment This should be a string
+  optionmenu Choice: 1
+    option Foo
+    option Bar
+    option 100
   real left_Range -123.6
   positive right_Range_max 3.3
   integer Int 4
   natural Nat 4
 endform
 
+# Periods do not establish boundaries for keywords
+form.var = 10
+# Or operators
+not.an.operator$ = "Bad variable name"
+bad.or.not = 1
+
 # External scripts
 include /path/to/file
 runScript: "/path/to/file"
@@ -51,12 +61,16 @@ endif
 
 string$ = "Strings can be 'interpolated'"
 string$ = "But don't interpolate everything!"
+string$(10)
+
+repeat
+  string$ = string$ - right$(string$)
+until !length(string$)
 
 Text... 1 Right 0.2 Half many----hyphens
 Text... 1 Right -0.4 Bottom aحبيبa
 Text... 1 Right -0.6 Bottom 日本
 Draw circle (mm)... 0.5 0.5 i
-x=1
 
 rows   = Object_'table'.nrow
 value$ = Table_'table'$[25, "f0"]
@@ -83,17 +97,19 @@ var = if macintosh = 1 then 0 else 1 fi ; This is an inline comment
 n = numberOfSelected("Sound")
 for i from newStyle.local to n
   name = selected$(extractWord$(selected$(), " "))
-  sound'i' = selected("Sound", i)
+  sound'i' = selected("Sound", i+(a*b))
   sound[i] = sound'i'
 endfor
 
-for i from 1 to n
+i = 1
+while i < n
+  i++
   # Different styles of object selection
   select sound'i'
   sound = selected()
   sound$ = selected$("Sound")
   select Sound 'sound$'
-  selectObject(sound[i])
+  selectObject( sound[i])
   selectObject: sound
 
   # Pause commands
@@ -124,14 +140,16 @@ for i from 1 to n
   # Multi-line command with modifier
   pitch = noprogress To Pitch (ac): 0, 75, 15, "no",
     ...0.03, 0.45, 0.01, 0.35, 0.14, 600
+  # Formulas are strings
+  Formula: "if col = 1 then row * Object_'pitch'.dx + 'first' else self fi"
 
   # do-style command with assignment
   minimum = do("Get minimum...", 0, 0, "Hertz", "Parabolic")
 
   # New-style multi-line command call with broken strings
   table = Create Table with column names: "table", 0,
-    ..."file subject speaker 
-    ...f0 f1 f2 f3 " +
+    ..."file subject speaker
+    ... f0 f1 f2 f" + string$(3) + " " +
     ..."duration response"
 
   # Function call with trailing space
@@ -156,7 +174,7 @@ for i from 1 to n
   demoWaitForInput ( )
   demo Erase all
   demo Text: 50, "centre", 50, "half", "Finished"
-endfor
+endwhile
 
 switch$ = if switch == 1 then "a" else
   ...     if switch == 2 then "b" else
@@ -207,6 +225,11 @@ assert  a !=  b  &&  c
 assert  a <>  b  ||  c
 assert  a <   b  |   c
 assert  a >   b
+
+assert (a)or (b)
+assert (a) or(b)
+assert (a)and(b)
+
 assert "hello" =  "he" + "llo"
 assert "hello" == "hello world" - " world"
 
@@ -243,3 +266,15 @@ endproc
 asserterror Unknown symbol:'newline$'« _
 assert '_new_style.local'
 
+@proc: a, selected("string"), b
+# Comment
+
+for i to saveSelection.n
+  selectObject: saveSelection.id[i]
+  appendInfoLine: selected$()
+endfor
+
+@ok(if selected$("Sound") = "tone" then 1 else 0 fi,
+  ... "selected sound is tone")
+
+@ok_formula("selected$(""Sound"") = ""tone""", "selected sound is tone")
diff --git a/tests/test_praat.py b/tests/test_praat.py
new file mode 100644
index 00000000..471d5e2c
--- /dev/null
+++ b/tests/test_praat.py
@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+"""
+    Praat lexer tests
+    ~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments.token import Token
+from pygments.lexers import PraatLexer
+
+class PraatTest(unittest.TestCase):
+
+    def setUp(self):
+        self.lexer = PraatLexer()
+        self.maxDiff = None
+
+    def testNumericAssignment(self):
+        fragment = u'var = -15e4\n'
+        tokens = [
+            (Token.Text, u'var'),
+            (Token.Text, u' '),
+            (Token.Operator, u'='),
+            (Token.Text, u' '),
+            (Token.Operator, u'-'),
+            (Token.Literal.Number, u'15e4'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testStringAssignment(self):
+        fragment = u'var$ = "foo"\n'
+        tokens = [
+            (Token.Text, u'var$'),
+            (Token.Text, u' '),
+            (Token.Operator, u'='),
+            (Token.Text, u' '),
+            (Token.Literal.String, u'"'),
+            (Token.Literal.String, u'foo'),
+            (Token.Literal.String, u'"'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testStringEscapedQuotes(self):
+        fragment = u'"it said ""foo"""\n'
+        tokens = [
+            (Token.Literal.String, u'"'),
+            (Token.Literal.String, u'it said '),
+            (Token.Literal.String, u'"'),
+            (Token.Literal.String, u'"'),
+            (Token.Literal.String, u'foo'),
+            (Token.Literal.String, u'"'),
+            (Token.Literal.String, u'"'),
+            (Token.Literal.String, u'"'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testFunctionCall(self):
+        fragment = u'selected("Sound", i+(a*b))\n'
+        tokens = [
+            (Token.Name.Function, u'selected'),
+            (Token.Punctuation, u'('),
+            (Token.Literal.String, u'"'),
+            (Token.Literal.String, u'Sound'),
+            (Token.Literal.String, u'"'),
+            (Token.Punctuation, u','),
+            (Token.Text, u' '),
+            (Token.Text, u'i'),
+            (Token.Operator, u'+'),
+            (Token.Text, u'('),
+            (Token.Text, u'a'),
+            (Token.Operator, u'*'),
+            (Token.Text, u'b'),
+            (Token.Text, u')'),
+            (Token.Punctuation, u')'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testBrokenUnquotedString(self):
+        fragment = u'printline string\n... \'interpolated\' string\n'
+        tokens = [
+            (Token.Keyword, u'printline'),
+            (Token.Text, u' '),
+            (Token.Literal.String, u'string'),
+            (Token.Text, u'\n'),
+            (Token.Punctuation, u'...'),
+            (Token.Text, u' '),
+            (Token.Literal.String.Interpol, u"'"),
+            (Token.Literal.String.Interpol, u'interpolated'),
+            (Token.Literal.String.Interpol, u"'"),
+            (Token.Text, u' '),
+            (Token.Literal.String, u'string'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testInlinIf(self):
+        fragment = u'var = if true == 1 then -1 else 0 fi'
+        tokens = [
+            (Token.Text, u'var'),
+            (Token.Text, u' '),
+            (Token.Operator, u'='),
+            (Token.Text, u' '),
+            (Token.Keyword, u'if'),
+            (Token.Text, u' '),
+            (Token.Text, u'true'),
+            (Token.Text, u' '),
+            (Token.Operator, u'=='),
+            (Token.Text, u' '),
+            (Token.Literal.Number, u'1'),
+            (Token.Text, u' '),
+            (Token.Keyword, u'then'),
+            (Token.Text, u' '),
+            (Token.Operator, u'-'),
+            (Token.Literal.Number, u'1'),
+            (Token.Text, u' '),
+            (Token.Keyword, u'else'),
+            (Token.Text, u' '),
+            (Token.Literal.Number, u'0'),
+            (Token.Text, u' '),
+            (Token.Keyword, u'fi'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))