signature TOKENIZER = sig
  datatype intType = ItDec | ItOct | ItHex
  datatype intSfx = IsNone | IsU | IsL | IsUL | IsLL | IsULL
  datatype floatSfx = FsNone | FsF | FsL

  datatype numConst =
    IntConst of intType * string * intSfx |
    FloatConst of string * floatSfx

  datatype token =
    Invalid |
    EOS |
    NewLine |

    Num of numConst |

    Id of string |
    CharConst of string * int |
    StringConst of string |

    kwBreak |
    kwCase |
    kwChar |
    kwConst |
    kwContinue |
    kwDefault |
    kwDouble |
    kwElse |
    kwEnum |
    kwExtern |
    kwFloat |
    kwFor |
    kwGoto |
    kwInt |
    kwLong |
    kwRegister |
    kwReturn |
    kwShort |
    kwSigned |
    kwSizeof |
    kwStruct |
    kwSwitch |
    kwTypedef |
    kwUnion |
    kwUnsigned |
    kwVoid |
    kwVolatile |

    LParen |
    RParen |
    LBracket |
    RBracket |
    LBrace |
    RBrace |

    QuestionMark |
    Colon |
    Coma |
    Semicolon |

    Arrow |
    Plus |
    DoublePlus|
    Minus |
    DoubleMinus |
    Ampersand |
    Asterisk |
    Slash |
    Tilde |
    ExclMark |
    Percent |
    DoubleGreater |
    DoubleLess |
    Greater |
    Less |
    EqualSign |
    LessEqualSign |
    GreaterEqualSign |
    DoubleEqualSign |
    ExclMarkEqualSign |
    Cap |
    VerticalBar |
    DoubleAmpersand |
    DoubleVerticalBar |

    AsteriskEqualSign |
    SlashEqualSign |
    PercentEqualSign |
    PlusEqualSign |
    MinusEqualSign |
    DoubleLessEqualSign |
    DoubleGreaterEqualSign |
    AmpersandEqualSign |
    CapEqualSign |
    VerticalBarEqualSign |

    Hash |
    DoubleHash |

    Dot |
    DoubleDot |
    TripleDot |

    CommentStart |

    CppInclude |
    CppDefine |
    CppUndef |
    CppIf |
    CppIfdef |
    CppIfndef |
    CppElse |
    CppElif |
    CppEndif |
    CppWarning |
    CppError |
    CppPragma

  type fullToken = Stream.pos * token

  (* Fatal. both may be thrown by tokenize *)
  exception FsmTableIsTooSmall
  exception TkErrorAug of Stream.ppos * string

  val getToken: Stream.t -> fullToken option * Stream.t

  val tokenize: Stream.t -> fullToken list
  val token2str: token -> string
  val printToken: token -> unit

  val debugPrint: fullToken list -> string -> unit
end