signature TOKENIZER = sig
  structure S: STREAM

  datatype intType = ItDec | ItOct | ItHex
  datatype intSfx = IsNone | IsU | IsL | IsUL | IsLL | IsULL
  datatype floatSfx = FsNone | FsF | FsL

  datatype numConst =
    IntConst of intType * string * intSfx |
    FloatConst of string * floatSfx

  datatype token =
    Invalid |
    EOS |
    NewLine |
    MacroEnd of string |

    Num of numConst |

    Id of string |
    CharConst of string * int |
    StringConst of string |

    kwBreak |
    kwCase |
    kwChar |
    kwConst |
    kwContinue |
    kwDefault |
    kwDo |
    kwDouble |
    kwElse |
    kwEnum |
    kwExtern |
    kwFloat |
    kwFor |
    kwGoto |
    kwIf |
    kwInt |
    kwLong |
    kwRegister |
    kwReturn |
    kwShort |
    kwSigned |
    kwSizeof |
    kwStatic |
    kwStruct |
    kwSwitch |
    kwTypedef |
    kwUnion |
    kwUnsigned |
    kwVoid |
    kwVolatile |
    kwWhile |

    LParen |
    RParen |
    LBracket |
    RBracket |
    LBrace |
    RBrace |

    QuestionMark |
    Colon |
    Coma |
    Semicolon |

    Arrow |
    Plus |
    DoublePlus|
    Minus |
    DoubleMinus |
    Ampersand |
    Asterisk |
    Slash |
    Tilde |
    ExclMark |
    Percent |
    DoubleGreater |
    DoubleLess |
    Greater |
    Less |
    EqualSign |
    LessEqualSign |
    GreaterEqualSign |
    DoubleEqualSign |
    ExclMarkEqualSign |
    Cap |
    VerticalBar |
    DoubleAmpersand |
    DoubleVerticalBar |

    AsteriskEqualSign |
    SlashEqualSign |
    PercentEqualSign |
    PlusEqualSign |
    MinusEqualSign |
    DoubleLessEqualSign |
    DoubleGreaterEqualSign |
    AmpersandEqualSign |
    CapEqualSign |
    VerticalBarEqualSign |

    Hash |
    DoubleHash |

    Dot |
    DoubleDot |
    TripleDot |

    CommentStart |

    PpcInclude of string * string |
    PpcDefine |
    PpcUndef |
    PpcIf |
    PpcIfdef |
    PpcIfndef |
    PpcElse |
    PpcElif |
    PpcEndif |
    PpcWarning |
    PpcError |
    PpcPragma

  val getToken: S.t -> token * S.pos * S.t
  val Ptk: (token, 'a, 'b) a1printer

  val isPpcDir: token -> bool
  val debugPrint: string -> unit
end