Move test_examplefiles to new tests/lexers scheme.examplefiles

1 files changed, 885 insertions, 0 deletions

diff --git a/tests/lexers/dg/example.txt b/tests/lexers/dg/example.txt
new file mode 100644
index 00000000..8856e4dc
--- /dev/null
+++ b/tests/lexers/dg/example.txt
@@ -0,0 +1,885 @@
+---input---
+import '/re'
+import '/sys'
+
+
+# IPv6address    =  hexpart [ ":" IPv4address ]
+# IPv4address    =  1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT
+# hexpart        =  [ hexseq ] [ "::" [ hexseq ] ]
+# hexseq         =  hex4 *( ":" hex4)
+# hex4           =  1*4HEXDIG
+hexpart = r'({0}|)(?:::({0}|)|)'.format r'(?:[\da-f]{1,4})(?::[\da-f]{1,4})*'
+addrv4  = r'(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})'
+addrv6  = re.compile $ r'(?i)(?:{})(?::{})?$'.format hexpart addrv4
+
+
+# Parse a base-N number given a list of its digits.
+#
+# :param q: the number of digits in that numeral system
+#
+# :param digits: an iterable of integers in range [0..q]
+#
+# :return: a decimal integer
+#
+base_n = q digits -> foldl (x y -> x * q + y) 0 digits
+
+
+# Parse a sequence of hexadecimal numbers
+#
+# :param q: a string of colon-separated base-16 integers
+#
+# :return: an iterable of Python ints
+#
+unhex = q -> q and map (p -> int p 16) (q.split ':')
+
+
+# Parse an IPv6 address as specified in RFC 4291.
+#
+# :param address: a string, obviously.
+#
+# :return: an integer which, written in binary form, points to the same node.
+#
+inet_pton6 = address ->
+  not (match = addrv6.match address) => raise $ ValueError 'not a valid IPv6 address'
+  start, end, *ipv4 = match.groups!
+
+  is_ipv4 = not $ None in ipv4
+  shift   = (7 - start.count ':' - 2 * is_ipv4) * 16
+
+  (end is None and shift) or shift < 0 => raise $ ValueError 'not a valid IPv6 address'
+  hexaddr = (base_n 0x10000 (unhex start) << shift) + base_n 0x10000 (unhex $ end or '')
+  if (is_ipv4 => (hexaddr << 32) + base_n 0x100 (map int ipv4)) (otherwise => hexaddr)
+
+
+inet6_type = q -> if
+  q == 0 => 'unspecified'
+  q == 1 => 'loopback'
+  (q >>  32)              == 0x000000000000ffff => 'IPv4-mapped'
+  (q >>  64)              == 0xfe80000000000000 => 'link-local'
+  (q >> 120)              != 0x00000000000000ff => 'general unicast'
+  (q >> 112) % (1 << 4)   == 0x0000000000000000 => 'multicast w/ reserved scope value'
+  (q >> 112) % (1 << 4)   == 0x000000000000000f => 'multicast w/ reserved scope value'
+  (q >> 112) % (1 << 4)   == 0x0000000000000001 => 'interface-local multicast'
+  (q >> 112) % (1 << 4)   == 0x0000000000000004 => 'admin-local multicast'
+  (q >> 112) % (1 << 4)   == 0x0000000000000005 => 'site-local multicast'
+  (q >> 112) % (1 << 4)   == 0x0000000000000008 => 'organization-local multicast'
+  (q >> 112) % (1 << 4)   == 0x000000000000000e => 'global multicast'
+  (q >> 112) % (1 << 4)   != 0x0000000000000002 => 'multicast w/ unknown scope value'
+  (q >>  24) % (1 << 112) == 0x00000000000001ff => 'solicited-node multicast'
+  otherwise => 'link-local multicast'
+
+
+print $ (x -> inet6_type x, hex x) $ inet_pton6 $ sys.stdin.read!.strip!
+
+---tokens---
+'import'      Keyword.Reserved
+' '           Text
+"'"           Literal.String
+'/re'         Literal.String
+"'"           Literal.String
+'\n'          Text
+
+'import'      Keyword.Reserved
+' '           Text
+"'"           Literal.String
+'/sys'        Literal.String
+"'"           Literal.String
+'\n\n\n'      Text
+
+'# IPv6address    =  hexpart [ ":" IPv4address ]' Comment.Single
+'\n'          Text
+
+'# IPv4address    =  1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT' Comment.Single
+'\n'          Text
+
+'# hexpart        =  [ hexseq ] [ "::" [ hexseq ] ]' Comment.Single
+'\n'          Text
+
+'# hexseq         =  hex4 *( ":" hex4)' Comment.Single
+'\n'          Text
+
+'# hex4           =  1*4HEXDIG' Comment.Single
+'\n'          Text
+
+'hexpart'     Name
+' '           Text
+'='           Operator
+' '           Text
+"r'"          Literal.String
+'({0}|)(?:::({0}|)|)' Literal.String
+"'"           Literal.String
+'.'           Operator
+'format'      Name
+' '           Text
+"r'"          Literal.String
+'(?:['        Literal.String
+'\\'          Literal.String
+'da-f]{1,4})(?::[' Literal.String
+'\\'          Literal.String
+'da-f]{1,4})*' Literal.String
+"'"           Literal.String
+'\n'          Text
+
+'addrv4'      Name
+'  '          Text
+'='           Operator
+' '           Text
+"r'"          Literal.String
+'('           Literal.String
+'\\'          Literal.String
+'d{1,3})'     Literal.String
+'\\'          Literal.String
+'.('          Literal.String
+'\\'          Literal.String
+'d{1,3})'     Literal.String
+'\\'          Literal.String
+'.('          Literal.String
+'\\'          Literal.String
+'d{1,3})'     Literal.String
+'\\'          Literal.String
+'.('          Literal.String
+'\\'          Literal.String
+'d{1,3})'     Literal.String
+"'"           Literal.String
+'\n'          Text
+
+'addrv6'      Name
+'  '          Text
+'='           Operator
+' '           Text
+'re'          Name
+'.'           Operator
+'compile'     Name
+' '           Text
+'$'           Operator
+' '           Text
+"r'"          Literal.String
+'(?i)(?:{})(?::{})?$' Literal.String
+"'"           Literal.String
+'.'           Operator
+'format'      Name
+' '           Text
+'hexpart'     Name
+' '           Text
+'addrv4'      Name
+'\n\n\n'      Text
+
+'# Parse a base-N number given a list of its digits.' Comment.Single
+'\n'          Text
+
+'#'           Comment.Single
+'\n'          Text
+
+'# :param q: the number of digits in that numeral system' Comment.Single
+'\n'          Text
+
+'#'           Comment.Single
+'\n'          Text
+
+'# :param digits: an iterable of integers in range [0..q]' Comment.Single
+'\n'          Text
+
+'#'           Comment.Single
+'\n'          Text
+
+'# :return: a decimal integer' Comment.Single
+'\n'          Text
+
+'#'           Comment.Single
+'\n'          Text
+
+'base_n'      Name
+' '           Text
+'='           Operator
+' '           Text
+'q'           Name
+' '           Text
+'digits'      Name
+' '           Text
+'->'          Operator
+' '           Text
+'foldl'       Name
+' '           Text
+'('           Punctuation
+'x'           Name
+' '           Text
+'y'           Name
+' '           Text
+'->'          Operator
+' '           Text
+'x'           Name
+' '           Text
+'*'           Operator
+' '           Text
+'q'           Name
+' '           Text
+'+'           Operator
+' '           Text
+'y'           Name
+')'           Punctuation
+' '           Text
+'0'           Literal.Number.Integer
+' '           Text
+'digits'      Name
+'\n\n\n'      Text
+
+'# Parse a sequence of hexadecimal numbers' Comment.Single
+'\n'          Text
+
+'#'           Comment.Single
+'\n'          Text
+
+'# :param q: a string of colon-separated base-16 integers' Comment.Single
+'\n'          Text
+
+'#'           Comment.Single
+'\n'          Text
+
+'# :return: an iterable of Python ints' Comment.Single
+'\n'          Text
+
+'#'           Comment.Single
+'\n'          Text
+
+'unhex'       Name
+' '           Text
+'='           Operator
+' '           Text
+'q'           Name
+' '           Text
+'->'          Operator
+' '           Text
+'q'           Name
+' '           Text
+'and'         Operator.Word
+' '           Text
+'map'         Name.Builtin
+' '           Text
+'('           Punctuation
+'p'           Name
+' '           Text
+'->'          Operator
+' '           Text
+'int'         Name.Builtin
+' '           Text
+'p'           Name
+' '           Text
+'16'          Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'('           Punctuation
+'q'           Name
+'.'           Operator
+'split'       Name
+' '           Text
+"'"           Literal.String
+':'           Literal.String
+"'"           Literal.String
+')'           Punctuation
+'\n\n\n'      Text
+
+'# Parse an IPv6 address as specified in RFC 4291.' Comment.Single
+'\n'          Text
+
+'#'           Comment.Single
+'\n'          Text
+
+'# :param address: a string, obviously.' Comment.Single
+'\n'          Text
+
+'#'           Comment.Single
+'\n'          Text
+
+'# :return: an integer which, written in binary form, points to the same node.' Comment.Single
+'\n'          Text
+
+'#'           Comment.Single
+'\n'          Text
+
+'inet_pton6'  Name
+' '           Text
+'='           Operator
+' '           Text
+'address'     Name
+' '           Text
+'->'          Operator
+'\n  '        Text
+'not'         Keyword.Reserved
+' '           Text
+'('           Punctuation
+'match'       Name
+' '           Text
+'='           Operator
+' '           Text
+'addrv6'      Name
+'.'           Operator
+'match'       Name
+' '           Text
+'address'     Name
+')'           Punctuation
+' '           Text
+'=>'          Operator
+' '           Text
+'raise'       Keyword.Reserved
+' '           Text
+'$'           Operator
+' '           Text
+'ValueError'  Name.Exception
+' '           Text
+"'"           Literal.String
+'not a valid IPv6 address' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'start'       Name
+','           Operator
+' '           Text
+'end'         Name
+','           Operator
+' '           Text
+'*'           Operator
+'ipv4'        Name
+' '           Text
+'='           Operator
+' '           Text
+'match'       Name
+'.'           Operator
+'groups'      Name
+'!'           Operator
+'\n\n  '      Text
+'is_ipv4'     Name
+' '           Text
+'='           Operator
+' '           Text
+'not'         Keyword.Reserved
+' '           Text
+'$'           Operator
+' '           Text
+'None'        Name.Builtin.Pseudo
+' '           Text
+'in'          Operator.Word
+' '           Text
+'ipv4'        Name
+'\n  '        Text
+'shift'       Name
+'   '         Text
+'='           Operator
+' '           Text
+'('           Punctuation
+'7'           Literal.Number.Integer
+' '           Text
+'-'           Operator
+' '           Text
+'start'       Name
+'.'           Operator
+'count'       Name
+' '           Text
+"'"           Literal.String
+':'           Literal.String
+"'"           Literal.String
+' '           Text
+'-'           Operator
+' '           Text
+'2'           Literal.Number.Integer
+' '           Text
+'*'           Operator
+' '           Text
+'is_ipv4'     Name
+')'           Punctuation
+' '           Text
+'*'           Operator
+' '           Text
+'16'          Literal.Number.Integer
+'\n\n  '      Text
+'('           Punctuation
+'end'         Name
+' '           Text
+'is'          Operator.Word
+' '           Text
+'None'        Name.Builtin.Pseudo
+' '           Text
+'and'         Operator.Word
+' '           Text
+'shift'       Name
+')'           Punctuation
+' '           Text
+'or'          Operator.Word
+' '           Text
+'shift'       Name
+' '           Text
+'<'           Operator
+' '           Text
+'0'           Literal.Number.Integer
+' '           Text
+'=>'          Operator
+' '           Text
+'raise'       Keyword.Reserved
+' '           Text
+'$'           Operator
+' '           Text
+'ValueError'  Name.Exception
+' '           Text
+"'"           Literal.String
+'not a valid IPv6 address' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'hexaddr'     Name
+' '           Text
+'='           Operator
+' '           Text
+'('           Punctuation
+'base_n'      Name
+' '           Text
+'0x10000'     Literal.Number.Hex
+' '           Text
+'('           Punctuation
+'unhex'       Name
+' '           Text
+'start'       Name
+')'           Punctuation
+' '           Text
+'<<'          Operator
+' '           Text
+'shift'       Name
+')'           Punctuation
+' '           Text
+'+'           Operator
+' '           Text
+'base_n'      Name
+' '           Text
+'0x10000'     Literal.Number.Hex
+' '           Text
+'('           Punctuation
+'unhex'       Name
+' '           Text
+'$'           Operator
+' '           Text
+'end'         Name
+' '           Text
+'or'          Operator.Word
+' '           Text
+"'"           Literal.String
+"'"           Literal.String
+')'           Punctuation
+'\n  '        Text
+'if'          Keyword.Reserved
+' '           Text
+'('           Punctuation
+'is_ipv4'     Name
+' '           Text
+'=>'          Operator
+' '           Text
+'('           Punctuation
+'hexaddr'     Name
+' '           Text
+'<<'          Operator
+' '           Text
+'32'          Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'+'           Operator
+' '           Text
+'base_n'      Name
+' '           Text
+'0x100'       Literal.Number.Hex
+' '           Text
+'('           Punctuation
+'map'         Name.Builtin
+' '           Text
+'int'         Name.Builtin
+' '           Text
+'ipv4'        Name
+')'           Punctuation
+')'           Punctuation
+' '           Text
+'('           Punctuation
+'otherwise'   Keyword.Reserved
+' '           Text
+'=>'          Operator
+' '           Text
+'hexaddr'     Name
+')'           Punctuation
+'\n\n\n'      Text
+
+'inet6_type'  Name
+' '           Text
+'='           Operator
+' '           Text
+'q'           Name
+' '           Text
+'->'          Operator
+' '           Text
+'if'          Keyword.Reserved
+'\n  '        Text
+'q'           Name
+' '           Text
+'=='          Operator
+' '           Text
+'0'           Literal.Number.Integer
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'unspecified' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'q'           Name
+' '           Text
+'=='          Operator
+' '           Text
+'1'           Literal.Number.Integer
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'loopback'    Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+'  '          Text
+'32'          Literal.Number.Integer
+')'           Punctuation
+'              ' Text
+'=='          Operator
+' '           Text
+'0x000000000000ffff' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'IPv4-mapped' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+'  '          Text
+'64'          Literal.Number.Integer
+')'           Punctuation
+'              ' Text
+'=='          Operator
+' '           Text
+'0xfe80000000000000' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'link-local'  Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+' '           Text
+'120'         Literal.Number.Integer
+')'           Punctuation
+'              ' Text
+'!='          Operator
+' '           Text
+'0x00000000000000ff' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'general unicast' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+' '           Text
+'112'         Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'%'           Operator
+' '           Text
+'('           Punctuation
+'1'           Literal.Number.Integer
+' '           Text
+'<<'          Operator
+' '           Text
+'4'           Literal.Number.Integer
+')'           Punctuation
+'   '         Text
+'=='          Operator
+' '           Text
+'0x0000000000000000' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'multicast w/ reserved scope value' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+' '           Text
+'112'         Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'%'           Operator
+' '           Text
+'('           Punctuation
+'1'           Literal.Number.Integer
+' '           Text
+'<<'          Operator
+' '           Text
+'4'           Literal.Number.Integer
+')'           Punctuation
+'   '         Text
+'=='          Operator
+' '           Text
+'0x000000000000000f' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'multicast w/ reserved scope value' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+' '           Text
+'112'         Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'%'           Operator
+' '           Text
+'('           Punctuation
+'1'           Literal.Number.Integer
+' '           Text
+'<<'          Operator
+' '           Text
+'4'           Literal.Number.Integer
+')'           Punctuation
+'   '         Text
+'=='          Operator
+' '           Text
+'0x0000000000000001' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'interface-local multicast' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+' '           Text
+'112'         Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'%'           Operator
+' '           Text
+'('           Punctuation
+'1'           Literal.Number.Integer
+' '           Text
+'<<'          Operator
+' '           Text
+'4'           Literal.Number.Integer
+')'           Punctuation
+'   '         Text
+'=='          Operator
+' '           Text
+'0x0000000000000004' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'admin-local multicast' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+' '           Text
+'112'         Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'%'           Operator
+' '           Text
+'('           Punctuation
+'1'           Literal.Number.Integer
+' '           Text
+'<<'          Operator
+' '           Text
+'4'           Literal.Number.Integer
+')'           Punctuation
+'   '         Text
+'=='          Operator
+' '           Text
+'0x0000000000000005' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'site-local multicast' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+' '           Text
+'112'         Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'%'           Operator
+' '           Text
+'('           Punctuation
+'1'           Literal.Number.Integer
+' '           Text
+'<<'          Operator
+' '           Text
+'4'           Literal.Number.Integer
+')'           Punctuation
+'   '         Text
+'=='          Operator
+' '           Text
+'0x0000000000000008' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'organization-local multicast' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+' '           Text
+'112'         Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'%'           Operator
+' '           Text
+'('           Punctuation
+'1'           Literal.Number.Integer
+' '           Text
+'<<'          Operator
+' '           Text
+'4'           Literal.Number.Integer
+')'           Punctuation
+'   '         Text
+'=='          Operator
+' '           Text
+'0x000000000000000e' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'global multicast' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+' '           Text
+'112'         Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'%'           Operator
+' '           Text
+'('           Punctuation
+'1'           Literal.Number.Integer
+' '           Text
+'<<'          Operator
+' '           Text
+'4'           Literal.Number.Integer
+')'           Punctuation
+'   '         Text
+'!='          Operator
+' '           Text
+'0x0000000000000002' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'multicast w/ unknown scope value' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'('           Punctuation
+'q'           Name
+' '           Text
+'>>'          Operator
+'  '          Text
+'24'          Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'%'           Operator
+' '           Text
+'('           Punctuation
+'1'           Literal.Number.Integer
+' '           Text
+'<<'          Operator
+' '           Text
+'112'         Literal.Number.Integer
+')'           Punctuation
+' '           Text
+'=='          Operator
+' '           Text
+'0x00000000000001ff' Literal.Number.Hex
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'solicited-node multicast' Literal.String
+"'"           Literal.String
+'\n  '        Text
+'otherwise'   Keyword.Reserved
+' '           Text
+'=>'          Operator
+' '           Text
+"'"           Literal.String
+'link-local multicast' Literal.String
+"'"           Literal.String
+'\n\n\n'      Text
+
+'print'       Name.Builtin
+' '           Text
+'$'           Operator
+' '           Text
+'('           Punctuation
+'x'           Name
+' '           Text
+'->'          Operator
+' '           Text
+'inet6_type'  Name
+' '           Text
+'x'           Name
+','           Operator
+' '           Text
+'hex'         Name.Builtin
+' '           Text
+'x'           Name
+')'           Punctuation
+' '           Text
+'$'           Operator
+' '           Text
+'inet_pton6'  Name
+' '           Text
+'$'           Operator
+' '           Text
+'sys'         Name
+'.'           Operator
+'stdin'       Name
+'.'           Operator
+'read'        Name
+'!.'          Operator
+'strip'       Name
+'!'           Operator
+'\n'          Text