signature TOKENIZER = sig
  type token
  type fullToken = Stream.pos * token

  (* Fatal. both may be thrown by tokenize *)
  exception FsmTableIsTooSmall
  exception TkErrorAug of Stream.ppos * string

  val tokenize: Stream.t -> fullToken list -> fullToken list * Stream.fileInfo
  val printToken: token -> unit
end