From 3a32398248e5593b1b536c837478cab276f7aebf Mon Sep 17 00:00:00 2001 From: Vladimir Azarov Date: Mon, 26 May 2025 21:06:51 +0200 Subject: Simpler tokenizer --- tokenizer.sig | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) (limited to 'tokenizer.sig') diff --git a/tokenizer.sig b/tokenizer.sig index 67666a4..f221ab3 100644 --- a/tokenizer.sig +++ b/tokenizer.sig @@ -2,25 +2,17 @@ signature TOKENIZER = sig structure ST: SYMTAB structure S: STREAM - datatype intType = ItDec | ItOct | ItHex - datatype intSfx = IsNone | IsU | IsL | IsUL | IsLL | IsULL - datatype floatSfx = FsNone | FsF | FsL - - datatype numConst = - IntConst of intType * string * intSfx | - FloatConst of string * floatSfx - datatype token = Invalid | EOS | NewLine | MacroEnd of int | - Num of numConst | + Num of string | Id of int | - CharConst of string * int | - StringConst of string | + CharConst of int * int | + Strlit of int | kwBreak | kwCase | -- cgit v1.2.3