diff options
Diffstat (limited to 'tokenizer.fun')
| -rw-r--r-- | tokenizer.fun | 17 |
1 files changed, 13 insertions, 4 deletions
diff --git a/tokenizer.fun b/tokenizer.fun index 5cca203..4cb5d1d 100644 --- a/tokenizer.fun +++ b/tokenizer.fun @@ -29,6 +29,7 @@ struct kwConst | kwContinue | kwDefault | + kwDo | kwDouble | kwElse | kwEnum | @@ -36,6 +37,7 @@ struct kwFloat | kwFor | kwGoto | + kwIf | kwInt | kwLong | kwRegister | @@ -43,6 +45,7 @@ struct kwShort | kwSigned | kwSizeof | + kwStatic | kwStruct | kwSwitch | kwTypedef | @@ -50,6 +53,7 @@ struct kwUnsigned | kwVoid | kwVolatile | + kwWhile | LParen | RParen | @@ -121,7 +125,7 @@ struct CppError | CppPragma - val kwPrefix = #"@" + val kwPrefix = #"`" val cppPrefix = #"$" type fullToken = S.pos * token @@ -153,6 +157,7 @@ struct (kwConst, &"const"), (kwContinue, &"continue"), (kwDefault, &"default"), + (kwDo, &"do"), (kwDouble, &"double"), (kwElse, &"else"), (kwEnum, &"enum"), @@ -161,12 +166,14 @@ struct (kwFor, &"for"), (kwGoto, &"goto"), (kwInt, &"int"), + (kwIf, &"if"), (kwLong, &"long"), (kwRegister, &"register"), (kwReturn, &"return"), (kwShort, &"short"), (kwSigned, &"signed"), (kwSizeof, &"sizeof"), + (kwStatic, &"static"), (kwStruct, &"struct"), (kwSwitch, &"switch"), (kwTypedef, &"typedef"), @@ -174,6 +181,7 @@ struct (kwUnsigned, &"unsigned"), (kwVoid, &"void"), (kwVolatile, &"volatile"), + (kwWhile, &"while"), (LParen, "("), (RParen, ")"), @@ -271,7 +279,7 @@ struct getSfxRepr sfx buf (fn () => raise SuffixWithoutRepr) val token2str = fn - Id s => "id:" ^ s + Id s => s | Num (IntConst (it, str, sfx)) => let val intType = @@ -280,10 +288,10 @@ struct | ItOct => "0" | ItHex => "0x" in - intType ^ str ^ "`" ^ getSfxReprSimple sfx intSuffixRepr ^ "`" + intType ^ str ^ getSfxReprSimple sfx intSuffixRepr end | Num (FloatConst (str, sfx)) => - str ^ "`" ^ getSfxReprSimple sfx floatSuffixRepr ^ "`" + str ^ getSfxReprSimple sfx floatSuffixRepr | CharConst (repr, _) => repr | StringConst s => "\"" ^ s ^ "\"" @@ -860,6 +868,7 @@ struct | NONE => raise ExpectedCppDir end | formCppDir kwElse = CppElse + | formCppDir kwIf = CppIf | formCppDir _ = raise ExpectedCppDir fun handleCppDir tk prevPos stream = |