diff options
| author | Vladimir Azarov <avm@intermediate-node.net> | 2025-05-26 21:06:51 +0200 |
|---|---|---|
| committer | Vladimir Azarov <avm@intermediate-node.net> | 2025-05-26 21:06:51 +0200 |
| commit | 3a32398248e5593b1b536c837478cab276f7aebf (patch) | |
| tree | 55ff4004a06278c84908668c26d39d0fc3098a08 /tokenizer.sig | |
| parent | 6f3fa80b37ca5f8d992f5d6f66aee77ead303bf4 (diff) | |
Simpler tokenizer
Diffstat (limited to 'tokenizer.sig')
| -rw-r--r-- | tokenizer.sig | 14 |
1 files changed, 3 insertions, 11 deletions
diff --git a/tokenizer.sig b/tokenizer.sig index 67666a4..f221ab3 100644 --- a/tokenizer.sig +++ b/tokenizer.sig @@ -2,25 +2,17 @@ signature TOKENIZER = sig structure ST: SYMTAB structure S: STREAM - datatype intType = ItDec | ItOct | ItHex - datatype intSfx = IsNone | IsU | IsL | IsUL | IsLL | IsULL - datatype floatSfx = FsNone | FsF | FsL - - datatype numConst = - IntConst of intType * string * intSfx | - FloatConst of string * floatSfx - datatype token = Invalid | EOS | NewLine | MacroEnd of int | - Num of numConst | + Num of string | Id of int | - CharConst of string * int | - StringConst of string | + CharConst of int * int | + Strlit of int | kwBreak | kwCase | |