diff options
| author | Vladimir Azarov <avm@intermediate-node.net> | 2025-05-31 19:30:21 +0200 |
|---|---|---|
| committer | Vladimir Azarov <avm@intermediate-node.net> | 2025-05-31 19:30:38 +0200 |
| commit | 546a5861526192a908f2aa2bfc3cfe4f3f3baf43 (patch) | |
| tree | acfe627e088bdba54a42e786d3b6b7053ec56fca /tokenizer.fun | |
| parent | 868e6313e3824d68b3121c5c95c7f29bc088c0e9 (diff) | |
Proper constant parsing
Diffstat (limited to 'tokenizer.fun')
| -rw-r--r-- | tokenizer.fun | 15 |
1 files changed, 7 insertions, 8 deletions
diff --git a/tokenizer.fun b/tokenizer.fun index 2adfd5c..b0df510 100644 --- a/tokenizer.fun +++ b/tokenizer.fun @@ -11,10 +11,9 @@ struct NewLine | MacroEnd of int | - Num of string | - Id of int | CharConst of int * int | + Num of int | Strlit of int | kwBreak | @@ -275,12 +274,11 @@ struct end z in case tk of - Id id => Printf out ?id % + Id id | Num id => Printf out ?id % | MacroEnd mid => Printf out `"mend(" ?mid `")" % | NewLine => Printf out `"\\n" % | PpcInclude (dir, arg) => Printf out `"#include(" `dir `", " `arg `")" % - | Num s => Printf out `s % | CharConst (repr, _) => Printf out ?repr % | Strlit id => Printf out ?id % | v => @@ -646,7 +644,7 @@ struct (Id id, stream) end - fun parseNumber dx stream = + fun parseNumber symtab dx stream = let fun collect stream = let @@ -669,8 +667,9 @@ struct val (endOff, stream) = collect stream val s = S.getSubstr startOff endOff stream + val id = ST.getId symtab s in - (Num s, pos, stream) + (Num id, pos, stream) end fun getDir stream = OS.Path.getParent o S.getFname $ stream @@ -807,9 +806,9 @@ struct else if isNondigit c then @-> $ parseId symtab else if isDigit c then - parseNumber 1 stream + parseNumber symtab 1 stream else if c = #"." andalso isDigit c1 then - parseNumber 2 stream + parseNumber symtab 2 stream else if c = #"'" then parseCharConst symtab stream else if c = #"\"" then |