Hashtable-based findKeyword function

1 files changed, 22 insertions, 10 deletions

diff --git a/tokenizer.sml b/tokenizer.sml
index 41f8d38..104ae61 100644
--- a/tokenizer.sml
+++ b/tokenizer.sml
@@ -8,7 +8,6 @@ structure Tokenizer:> TOKENIZER = struct
     IntConst of intType * string * intSfx |
     FloatConst of string * floatSfx
 
-
   datatype token =
     Invalid |
     NewLine |
@@ -500,15 +499,28 @@ structure Tokenizer:> TOKENIZER = struct
     s
   end
 
-  fun findKeyword s =
-    case List.find
-      (fn (_, repr) =>
-          String.sub (repr, 0) = kwPrefix andalso
-            String.extract (repr, 1, NONE) = s)
-      tokenRepr
-    of
-      SOME (tk, _) => tk
-    | NONE => Id s
+  fun keywordHashtableGen () =
+  let
+    open Hashtable
+    val table = create 128
+    val () =
+      List.app
+        (fn (tk, repr) =>
+            if String.sub (repr, 0) = kwPrefix then
+              insert table (String.extract (repr, 1, NONE)) tk
+            else
+              ())
+        tokenRepr
+  in
+    table
+  end
+
+  val keywordHashtable = lazy keywordHashtableGen
+
+  fun findKeyword str =
+    case Hashtable.lookup (keywordHashtable ()) str of
+      NONE => Id str
+    | SOME tk => tk
 
   fun idParser () (stream, startOff) c =
   let