Rewrite the Dylan Lexer and provide a lexer for Dylan LID files.

Fixes include:
* Dylan is not case sensitive.
* Support for other number literal formats (hex, octal, binary).
* Support nested multi-line comments.
* Correct various issues with matching of 'names'.
* Support highlighting of macro definitions better.
* Support keyword / required keyword syntax in class definitions.
* Support each-subclass slot allocation.
* Handle the | operator.
* Change what tokens are mapped to what types and reduce the Christmas tree look of the output a bit.
* Explicitly list and handle the various builtins defined in the DRM.

1 files changed, 86 insertions, 4 deletions

diff --git a/tests/examplefiles/classes.dylan b/tests/examplefiles/classes.dylan
index 6dd55ff2..90cafdf6 100644
--- a/tests/examplefiles/classes.dylan
+++ b/tests/examplefiles/classes.dylan
@@ -1,12 +1,25 @@
+module: sample
+comment: for make sure that does not highlight per word.
+
 define class <car> (<object>)
   slot serial-number :: <integer> = unique-serial-number();
-  slot model-name :: <string>,
+  constant slot model-name :: <string>,
     required-init-keyword: model:;
-  slot has-sunroof? :: <boolean>,
+  each-subclass slot has-sunroof? :: <boolean>,
     init-keyword: sunroof?:,
     init-value: #f;
+  keyword foo:;
+  required keyword bar:;
 end class <car>;
 
+define class <flying-car> (<car>)
+end class <flying-car>;
+
+let flying-car = make(<flying-car>);
+let car? :: <car?> = #f;
+let prefixed-car :: <vehicles/car> = #f;
+let model :: <car-911> = #f;
+
 define constant $empty-string = "";
 define constant $escaped-backslash = '\\';
 define constant $escaped-single-quote = '\'';
@@ -31,10 +44,79 @@ define method foo() => _ :: <boolean>;
   #t
 end method;
 
-define method \+()
-end;
+define method \+
+    (offset1 :: <time-offset>, offset2 :: <time-offset>)
+ => (sum :: <time-offset>)
+  let sum = offset1.total-seconds + offset2.total-seconds;
+  make(<time-offset>, total-seconds: sum);
+end method \+;
+
+define method bar ()
+  1 | 2 & 3
+end
+
+if (bar)
+  1
+elseif (foo)
+  2
+else
+  3
+end if;
+
+select (foo by instance?)
+  <integer> => 1
+  otherwise => 3
+end select;
+
+/* multi
+   line
+   comment
+*/
+
+/* multi line comments
+  /* can be */
+  nested */
 
 define constant $symbol = #"hello";
 define variable *vector* = #[3.5, 5]
 define constant $list = #(1, 2);
 define constant $pair = #(1 . "foo")
+
+let octal-number = #o238;
+let hex-number = #x3890ADEF;
+let binary-number = #b1010;
+let float-exponent = 3.5e10;
+
+block (return)
+  with-lock (lock)
+    return();
+  end;
+exception (e :: <error>)
+    format-out("Oh no");
+cleanup
+    return();
+afterwards
+    format-out("Hello");
+end;
+
+define macro repeat
+  { repeat ?:body end }
+   => { block (?=stop!)
+          local method again() ?body; again() end;
+          again();
+        end }
+end macro repeat;
+
+define macro with-decoded-seconds
+  {
+    with-decoded-seconds
+      (?max:variable, ?min:variable, ?sec:variable = ?time:expression)
+      ?:body
+    end
+  }
+    => {
+         let (?max, ?min, ?sec) = decode-total-seconds(?time);
+         ?body
+       }
+end macro;
+