SystemVerilog keyword/operator improvements (#1464)

* Move SystemVerilog type keywords

Put them next to the generic keywords list.

* Change a couple SystemVerilog keywords to operators

The 'inside' and 'dist' keywords are described as operators in the
SystemVerilog standard, below unary increment/decrement, and above
concatenation in precedence.
See 1800-2017 tables 11-1 and 11-2 for a list of operators.

This matches the description of pygemnts Operator.Word token:
"For any operator that is a word (e.g. not)."

* Add a SystemVerilog operators unit test

Copy/paste the contents of 1800-2017 Table 11-2,
and see what the SV lexer chops it up into.

I made lots of comments for potential improvements.
Some operators, such as '[' and '.' are being labeled as punctuation.
Also, multi-character operators such as '<<<=' are being split up
into multiple, single-character tokens, eg '<' '<' '<' '='.

1 files changed, 249 insertions, 0 deletions

diff --git a/tests/test_hdl.py b/tests/test_hdl.py
index 86cb6c7b..939efe35 100644
--- a/tests/test_hdl.py
+++ b/tests/test_hdl.py
@@ -343,3 +343,252 @@ def test_systemverilog_numbers(lexer):
     """Test most types of numbers"""
     tokens = list(lexer.get_tokens(SYSTEMVERILOG_NUMBERS_TEXT))
     assert tokens == SYSTEMVERILOG_NUMBERS_TOKENS
+
+
+# See 1800-2017 Table 11-2: Operator Precedence and Associativity
+# Note that the duplicates (unary/binary) have been removed,
+# ie '+', '-', '&', '|', '^', '~^', '^~'
+SYSTEMVERILOG_OPERATORS_TEXT = """
+() [] :: .
++ - ! ~ & ~& | ~| ^ ~^ ^~ ++ --
+**
+* / %
+<< >> <<< >>>
+< <= > >= inside dist
+== != === !== ==? !=?
+&&
+||
+?:
+-> <->
+= += -= *= /= %= &= ^= |= <<= >>= <<<= >>>= := :/ <=
+{} {{}}
+"""
+
+# Note: This is a inconsistent mix of operator and punctuation
+SYSTEMVERILOG_OPERATORS_TOKENS = [
+    (Punctuation,   '('),
+    (Punctuation,   ')'),
+    (Text,          ' '),
+    (Punctuation,   '['),
+    (Punctuation,   ']'),
+    (Text,          ' '),
+    # Note: This should be '::'
+    (Operator,      ':'),
+    (Operator,      ':'),
+    (Text,          ' '),
+    (Punctuation,   '.'),
+    (Text,          '\n'),
+    (Operator,      '+'),
+    (Text,          ' '),
+    (Operator,      '-'),
+    (Text,          ' '),
+    (Operator,      '!'),
+    (Text,          ' '),
+    (Operator,      '~'),
+    (Text,          ' '),
+    (Operator,      '&'),
+    (Text,          ' '),
+    # Note: This should be '~&'
+    (Operator,      '~'),
+    (Operator,      '&'),
+    (Text,          ' '),
+    (Operator,      '|'),
+    (Text,          ' '),
+    # Note: This should be '~|'
+    (Operator,      '~'),
+    (Operator,      '|'),
+    (Text,          ' '),
+    (Operator,      '^'),
+    (Text,          ' '),
+    # Note: This should be '~^'
+    (Operator,      '~'),
+    (Operator,      '^'),
+    (Text,          ' '),
+    # Note: This should be '^~'
+    (Operator,      '^'),
+    (Operator,      '~'),
+    (Text,          ' '),
+    # Note: This should be '++'
+    (Operator,      '+'),
+    (Operator,      '+'),
+    (Text,          ' '),
+    # Note: This should be '--'
+    (Operator,      '-'),
+    (Operator,      '-'),
+    (Text,          '\n'),
+    # Note: This should be '**'
+    (Operator,      '*'),
+    (Operator,      '*'),
+    (Text,          '\n'),
+    (Operator,      '*'),
+    (Text,          ' '),
+    (Operator,      '/'),
+    (Text,          ' '),
+    (Operator,      '%'),
+    (Text,          '\n'),
+    # Note: This should be '<<'
+    (Operator,      '<'),
+    (Operator,      '<'),
+    (Text,          ' '),
+    # Note: This should be '>>'
+    (Operator,      '>'),
+    (Operator,      '>'),
+    (Text,          ' '),
+    # Note: This should be '<<<'
+    (Operator,      '<'),
+    (Operator,      '<'),
+    (Operator,      '<'),
+    (Text,          ' '),
+    # Note: This should be '>>>'
+    (Operator,      '>'),
+    (Operator,      '>'),
+    (Operator,      '>'),
+    (Text,          '\n'),
+    (Operator,      '<'),
+    (Text,          ' '),
+    # Note: This should be '<='
+    (Operator,      '<'),
+    (Operator,      '='),
+    (Text,          ' '),
+    (Operator,      '>'),
+    (Text,          ' '),
+    # Note: This should be '>='
+    (Operator,      '>'),
+    (Operator,      '='),
+    (Text,          ' '),
+    (Operator.Word, 'inside'),
+    (Text,          ' '),
+    (Operator.Word, 'dist'),
+    (Text,          '\n'),
+    # Note: This should be '=='
+    (Operator,      '='),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '!='
+    (Operator,      '!'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '==='
+    (Operator,      '='),
+    (Operator,      '='),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '!=='
+    (Operator,      '!'),
+    (Operator,      '='),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '==?'
+    (Operator,      '='),
+    (Operator,      '='),
+    (Operator,      '?'),
+    (Text,          ' '),
+    # Note: This should be '!=?'
+    (Operator,      '!'),
+    (Operator,      '='),
+    (Operator,      '?'),
+    (Text,          '\n'),
+    # Note: This should be '&&'
+    (Operator,      '&'),
+    (Operator,      '&'),
+    (Text,          '\n'),
+    # Note: This should be '||'
+    (Operator,      '|'),
+    (Operator,      '|'),
+    (Text,          '\n'),
+    (Operator,      '?'),
+    (Operator,      ':'),
+    (Text,          '\n'),
+    # Note: This should be '->'
+    (Operator,      '-'),
+    (Operator,      '>'),
+    (Text,          ' '),
+    # Note: This should be '<->'
+    (Operator,      '<'),
+    (Operator,      '-'),
+    (Operator,      '>'),
+    (Text,          '\n'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '+='
+    (Operator,      '+'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '-='
+    (Operator,      '-'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '*='
+    (Operator,      '*'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '/='
+    (Operator,      '/'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '%='
+    (Operator,      '%'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '&='
+    (Operator,      '&'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '^='
+    (Operator,      '^'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '|='
+    (Operator,      '|'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '<<='
+    (Operator,      '<'),
+    (Operator,      '<'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '>>='
+    (Operator,      '>'),
+    (Operator,      '>'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '<<<='
+    (Operator,      '<'),
+    (Operator,      '<'),
+    (Operator,      '<'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be '>>>='
+    (Operator,      '>'),
+    (Operator,      '>'),
+    (Operator,      '>'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be ':='
+    (Operator,      ':'),
+    (Operator,      '='),
+    (Text,          ' '),
+    # Note: This should be ':/'
+    (Operator,      ':'),
+    (Operator,      '/'),
+    (Text,          ' '),
+    # Note: This should be '<='
+    (Operator,      '<'),
+    (Operator,      '='),
+    (Text,          '\n'),
+    (Punctuation,   '{'),
+    (Punctuation,   '}'),
+    (Text,          ' '),
+    # Note: This should be '{{'
+    (Punctuation,   '{'),
+    (Punctuation,   '{'),
+    # Note: This should be '}}'
+    (Punctuation,   '}'),
+    (Punctuation,   '}'),
+    (Text,          '\n'),
+]
+
+def test_systemverilog_operators(lexer):
+    """Test various operators"""
+    tokens = list(lexer.get_tokens(SYSTEMVERILOG_OPERATORS_TEXT))
+    assert tokens == SYSTEMVERILOG_OPERATORS_TOKENS