blob: ffbec52f3d82c07c08fcf08f10efc0d73b0871d3 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
|
signature TOKENIZER = sig
type token
type fullToken = Stream.pos * token
(* Fatal. both may be thrown by tokenize *)
exception FsmTableIsTooSmall
exception TkErrorAug of Stream.ppos * string
val tokenize: Stream.t -> fullToken list
val printToken: token -> unit
val printTokens: fullToken list -> string -> unit
end
|
