blob: 5956273442964d9d5be109468f9649c36d30312a (
plain)
1
2
3
4
5
6
7
8
9
10
11
|
signature TOKENIZER = sig
type token
type fullToken = Stream.pos * token
(* Fatal. both may be thrown by tokenize *)
exception FsmTableIsTooSmall
exception TkErrorAug of Stream.ppos * string
val tokenize: Stream.t -> fullToken list -> fullToken list * Stream.fileInfo
val printToken: token -> unit
end
|
