diff options
| author | Vladimir Azarov <avm@intermediate-node.net> | 2025-04-04 18:24:46 +0200 |
|---|---|---|
| committer | Vladimir Azarov <avm@intermediate-node.net> | 2025-04-04 18:24:46 +0200 |
| commit | 7b29b31648fd737e7bbc007f480b799add91bc6b (patch) | |
| tree | e724c15c959d98ece73c186b82a61100f4e8d06a /tokenizer.sig | |
| parent | d7d4830443f1e385af862462f976553c8a9033e1 (diff) | |
Beginning of the preprocessor
Diffstat (limited to 'tokenizer.sig')
| -rw-r--r-- | tokenizer.sig | 123 |
1 files changed, 121 insertions, 2 deletions
diff --git a/tokenizer.sig b/tokenizer.sig index ffbec52..245cfbe 100644 --- a/tokenizer.sig +++ b/tokenizer.sig @@ -1,13 +1,132 @@ signature TOKENIZER = sig - type token + datatype intType = ItDec | ItOct | ItHex + datatype intSfx = IsNone | IsU | IsL | IsUL | IsLL | IsULL + datatype floatSfx = FsNone | FsF | FsL + + datatype numConst = + IntConst of intType * string * intSfx | + FloatConst of string * floatSfx + + datatype token = + Invalid | + EOS | + NewLine | + + Num of numConst | + + Id of string | + CharConst of string * int | + StringConst of string | + + kwBreak | + kwCase | + kwChar | + kwConst | + kwContinue | + kwDefault | + kwDouble | + kwElse | + kwEnum | + kwExtern | + kwFloat | + kwFor | + kwGoto | + kwInt | + kwLong | + kwRegister | + kwReturn | + kwShort | + kwSigned | + kwSizeof | + kwStruct | + kwSwitch | + kwTypedef | + kwUnion | + kwUnsigned | + kwVoid | + kwVolatile | + + LParen | + RParen | + LBracket | + RBracket | + LBrace | + RBrace | + + QuestionMark | + Colon | + Coma | + Semicolon | + + Arrow | + Plus | + DoublePlus| + Minus | + DoubleMinus | + Ampersand | + Asterisk | + Slash | + Tilde | + ExclMark | + Percent | + DoubleGreater | + DoubleLess | + Greater | + Less | + EqualSign | + LessEqualSign | + GreaterEqualSign | + DoubleEqualSign | + ExclMarkEqualSign | + Cap | + VerticalBar | + DoubleAmpersand | + DoubleVerticalBar | + + AsteriskEqualSign | + SlashEqualSign | + PercentEqualSign | + PlusEqualSign | + MinusEqualSign | + DoubleLessEqualSign | + DoubleGreaterEqualSign | + AmpersandEqualSign | + CapEqualSign | + VerticalBarEqualSign | + + Hash | + DoubleHash | + + Dot | + DoubleDot | + TripleDot | + + CommentStart | + + CppInclude | + CppDefine | + CppUndef | + CppIf | + CppIfdef | + CppIfndef | + CppElse | + CppElif | + CppEndif | + CppWarning | + CppError | + CppPragma + type fullToken = Stream.pos * token (* Fatal. both may be thrown by tokenize *) exception FsmTableIsTooSmall exception TkErrorAug of Stream.ppos * string + val getToken: Stream.t -> fullToken option * Stream.t + val tokenize: Stream.t -> fullToken list + val token2str: token -> string val printToken: token -> unit - val printTokens: fullToken list -> string -> unit + val debugPrint: fullToken list -> string -> unit end |