Beginning of the preprocessor

1 files changed, 17 insertions, 21 deletions

diff --git a/tokenizer.sml b/tokenizer.sml
index 104ae61..10c2e77 100644
--- a/tokenizer.sml
+++ b/tokenizer.sml
@@ -10,6 +10,7 @@ structure Tokenizer:> TOKENIZER = struct
 
   datatype token =
     Invalid |
+    EOS |
     NewLine |
 
     Num of numConst |
@@ -265,8 +266,8 @@ structure Tokenizer:> TOKENIZER = struct
   fun getSfxReprSimple sfx buf =
     getSfxRepr sfx buf (fn () => raise SuffixWithoutRepr)
 
-  val printToken = fn
-      Id s => print $ "id:" ^ s
+  val token2str = fn
+      Id s => "id:" ^ s
     | Num (IntConst (it, str, sfx)) =>
       let
         val intType =
@@ -275,22 +276,20 @@ structure Tokenizer:> TOKENIZER = struct
           | ItOct => "0"
           | ItHex => "0x"
       in
-        print intType;
-        print str;
-        print $ "`" ^ getSfxReprSimple sfx intSuffixRepr ^ "`"
+        intType ^ str ^ "`" ^ getSfxReprSimple sfx intSuffixRepr ^ "`"
       end
-    | Num (FloatConst (str, sfx)) => (
-        print str;
-        print $ "`" ^ getSfxReprSimple sfx floatSuffixRepr ^ "`"
-    )
-    | CharConst (repr, _) => print repr
+    | Num (FloatConst (str, sfx)) =>
+        str ^ "`" ^ getSfxReprSimple sfx floatSuffixRepr ^ "`"
+    | CharConst (repr, _) => repr
     | StringConst s =>
-        print $ "\"" ^ s ^ "\""
+        "\"" ^ s ^ "\""
     | v =>
       case List.find (fn (x, _) => x = v) tokenRepr of
-      SOME (_, repr) => print repr
+      SOME (_, repr) => repr
       | NONE => raise TokenWithoutRepr
 
+  fun printToken tk = print $ token2str tk
+
   fun isIdStart c = Char.isAlpha c orelse c = #"_"
   fun isIdBody c = Char.isAlphaNum c orelse c = #"_"
 
@@ -766,14 +765,12 @@ structure Tokenizer:> TOKENIZER = struct
     | #"\n" => (NONE, stream)
     | #"x" => parseHexSeq stream
     | c =>
-        if isOctal c then
-          parseOctalSeq stream c
-        else
-          raiseErr0 "unknown escape sequence"
+      if isOctal c then
+        parseOctalSeq stream c
+      else
+        raiseErr0 "unknown escape sequence"
   end
 
-  fun stringCut s = String.extract (s, 1, SOME $ String.size s - 2)
-
   datatype SeqParseState = SeqInit | SeqStart | SeqValue of int | SeqTerm
 
   datatype seqParseMode = SpmChr | SpmStr
@@ -821,7 +818,7 @@ structure Tokenizer:> TOKENIZER = struct
             if mode = SpmChr then
               CharConst (s, v)
             else
-              StringConst $ stringCut s
+              StringConst $ String.extract (s, 1, SOME $ String.size s - 2)
         in
           (SeqTerm, SOME $ term (finishSeqRead startOff stream) v, stream)
         end
@@ -998,7 +995,7 @@ structure Tokenizer:> TOKENIZER = struct
     aux [] stream
   end
 
-  fun printTokens tkl fname =
+  fun debugPrint tkl fname =
   let
     fun print' line _ ((_, NewLine) :: tks) =
       print' (line + 1) true tks
@@ -1018,5 +1015,4 @@ structure Tokenizer:> TOKENIZER = struct
     print' 1 true tkl;
     print "\n"
   end
-
 end