signature TOKENIZER = sig
  type token
  type fullToken = Stream.pos * token

  (* Fatal. both may be thrown by tokenize *)
  exception FsmTableIsTooSmall
  exception TkErrorAug of Stream.convPos * string

  val tokenize: Stream.t -> fullToken list -> fullToken list * Stream.filesInfo

  val printToken: token -> unit
end