diff options
Diffstat (limited to 'tokenizer.sml')
| -rw-r--r-- | tokenizer.sml | 38 |
1 files changed, 17 insertions, 21 deletions
diff --git a/tokenizer.sml b/tokenizer.sml index 104ae61..10c2e77 100644 --- a/tokenizer.sml +++ b/tokenizer.sml @@ -10,6 +10,7 @@ structure Tokenizer:> TOKENIZER = struct datatype token = Invalid | + EOS | NewLine | Num of numConst | @@ -265,8 +266,8 @@ structure Tokenizer:> TOKENIZER = struct fun getSfxReprSimple sfx buf = getSfxRepr sfx buf (fn () => raise SuffixWithoutRepr) - val printToken = fn - Id s => print $ "id:" ^ s + val token2str = fn + Id s => "id:" ^ s | Num (IntConst (it, str, sfx)) => let val intType = @@ -275,22 +276,20 @@ structure Tokenizer:> TOKENIZER = struct | ItOct => "0" | ItHex => "0x" in - print intType; - print str; - print $ "`" ^ getSfxReprSimple sfx intSuffixRepr ^ "`" + intType ^ str ^ "`" ^ getSfxReprSimple sfx intSuffixRepr ^ "`" end - | Num (FloatConst (str, sfx)) => ( - print str; - print $ "`" ^ getSfxReprSimple sfx floatSuffixRepr ^ "`" - ) - | CharConst (repr, _) => print repr + | Num (FloatConst (str, sfx)) => + str ^ "`" ^ getSfxReprSimple sfx floatSuffixRepr ^ "`" + | CharConst (repr, _) => repr | StringConst s => - print $ "\"" ^ s ^ "\"" + "\"" ^ s ^ "\"" | v => case List.find (fn (x, _) => x = v) tokenRepr of - SOME (_, repr) => print repr + SOME (_, repr) => repr | NONE => raise TokenWithoutRepr + fun printToken tk = print $ token2str tk + fun isIdStart c = Char.isAlpha c orelse c = #"_" fun isIdBody c = Char.isAlphaNum c orelse c = #"_" @@ -766,14 +765,12 @@ structure Tokenizer:> TOKENIZER = struct | #"\n" => (NONE, stream) | #"x" => parseHexSeq stream | c => - if isOctal c then - parseOctalSeq stream c - else - raiseErr0 "unknown escape sequence" + if isOctal c then + parseOctalSeq stream c + else + raiseErr0 "unknown escape sequence" end - fun stringCut s = String.extract (s, 1, SOME $ String.size s - 2) - datatype SeqParseState = SeqInit | SeqStart | SeqValue of int | SeqTerm datatype seqParseMode = SpmChr | SpmStr @@ -821,7 +818,7 @@ structure Tokenizer:> TOKENIZER = struct if mode = SpmChr then CharConst (s, v) else - StringConst $ stringCut s + StringConst $ String.extract (s, 1, SOME $ String.size s - 2) in (SeqTerm, SOME $ term (finishSeqRead startOff stream) v, stream) end @@ -998,7 +995,7 @@ structure Tokenizer:> TOKENIZER = struct aux [] stream end - fun printTokens tkl fname = + fun debugPrint tkl fname = let fun print' line _ ((_, NewLine) :: tks) = print' (line + 1) true tks @@ -1018,5 +1015,4 @@ structure Tokenizer:> TOKENIZER = struct print' 1 true tkl; print "\n" end - end |