diff options
Diffstat (limited to 'tokenizer.fun')
| -rw-r--r-- | tokenizer.fun | 14 |
1 files changed, 7 insertions, 7 deletions
diff --git a/tokenizer.fun b/tokenizer.fun index 07bd08d..14f4289 100644 --- a/tokenizer.fun +++ b/tokenizer.fun @@ -14,7 +14,7 @@ struct Id of int | CharConst of int * Word64.word | Num of int | - Strlit of int | + Strlit of int * int | kwBreak | kwCase | @@ -280,7 +280,7 @@ struct | PpcInclude (dir, arg) => Printf out `"#include(" `dir `", " `arg `")" % | CharConst (repr, _) => Printf out ?repr % - | Strlit id => Printf out ?id % + | Strlit (id, _) => Printf out ?id % | v => case List.find (fn (x, _) => x = v) tokenRepr of SOME (_, repr) => @@ -606,21 +606,21 @@ struct val startOff = S.getOffset stream - 1 val (pos, stream) = S.getPosRaw startOff stream - fun collect stream = + fun collect size stream = let val (c, stream) = getMaybeBackslashed stream in case c of Reg #"\000" => error pos "unfinished string literal" - | Reg #"\"" => (S.getOffset stream, stream) - | _ => collect stream + | Reg #"\"" => (S.getOffset stream, size, stream) + | _ => collect (size + 1) stream end - val (endOff, stream) = collect stream + val (endOff, size, stream) = collect 0 stream val s = S.getSubstr startOff endOff stream val id = ST.getId symtab s in - (Strlit id, pos, stream) + (Strlit (id, size), pos, stream) end fun parseId symtab _ stream = |