1 files changed, 12 insertions, 0 deletions
diff --git a/tokenizer.sig b/tokenizer.sig
new file mode 100644
index 0000000..c31a7e6
--- /dev/null
+++ b/tokenizer.sig
@@ -0,0 +1,12 @@
+signature TOKENIZER = sig
+  type token
+  type fullToken = Stream.pos * token
+
+  (* Fatal. both may be thrown by tokenize *)
+  exception FsmTableIsTooSmall
+  exception TkErrorAug of Stream.convPos * string
+
+  val tokenize: Stream.t -> fullToken list -> fullToken list * Stream.filesInfo
+
+  val printToken: token -> unit
+end