signature TOKENIZER = sig
  structure ST: SYMTAB
  structure S: STREAM

  datatype token =
    Invalid |
    EOS |
    NewLine |
    MacroEnd of int |

    Id of int |
    CharConst of int * Word64.word |
    Num of int |
    Strlit of int * int |

    kwBreak |
    kwCase |
    kwChar |
    kwConst |
    kwContinue |
    kwDefault |
    kwDo |
    kwDouble |
    kwElse |
    kwEnum |
    kwExtern |
    kwFloat |
    kwFor |
    kwGoto |
    kwIf |
    kwInt |
    kwLong |
    kwRegister |
    kwReturn |
    kwShort |
    kwSigned |
    kwSizeof |
    kwStatic |
    kwStruct |
    kwSwitch |
    kwTypedef |
    kwUnion |
    kwUnsigned |
    kwVoid |
    kwVolatile |
    kwWhile |

    LParen |
    RParen |
    LBracket |
    RBracket |
    LBrace |
    RBrace |

    QuestionMark |
    Colon |
    Comma |
    Semicolon |

    Arrow |
    Plus |
    DoublePlus|
    Minus |
    DoubleMinus |
    Ampersand |
    Asterisk |
    Slash |
    Tilde |
    ExclMark |
    Percent |
    DoubleGreater |
    DoubleLess |
    Greater |
    Less |
    EqualSign |
    LessEqualSign |
    GreaterEqualSign |
    DoubleEqualSign |
    ExclMarkEqualSign |
    Cap |
    VerticalBar |
    DoubleAmpersand |
    DoubleVerticalBar |

    AsteriskEqualSign |
    SlashEqualSign |
    PercentEqualSign |
    PlusEqualSign |
    MinusEqualSign |
    DoubleLessEqualSign |
    DoubleGreaterEqualSign |
    AmpersandEqualSign |
    CapEqualSign |
    VerticalBarEqualSign |

    Hash |
    DoubleHash |

    Dot |
    DoubleDot |
    TripleDot |

    CommentStart |

    PpcInclude of string * string |
    PpcDefine |
    PpcUndef |
    PpcIf |
    PpcIfdef |
    PpcIfndef |
    PpcElse |
    PpcElif |
    PpcEndif |
    PpcWarning |
    PpcError |
    PpcPragma

  val initSymtab: unit -> token ST.t

  val getToken: token ST.t -> S.t -> token * S.pos * S.t
  val Ptk: (token ST.t, token, 'a, 'b, 'c) a2printer

  val isPpcDir: token -> bool
  val debugPrint: string -> unit
end