From 87217fe5ba58f5199d30586b5d9bec104dece445 Mon Sep 17 00:00:00 2001
From: Vladimir Azarov <avm@intermediate-node.net>
Date: Mon, 24 Mar 2025 21:51:16 +0100
Subject: Partition into structures

---
 tokenizer.sig | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 tokenizer.sig

(limited to 'tokenizer.sig')

diff --git a/tokenizer.sig b/tokenizer.sig
new file mode 100644
index 0000000..c31a7e6
--- /dev/null
+++ b/tokenizer.sig
@@ -0,0 +1,12 @@
+signature TOKENIZER = sig
+  type token
+  type fullToken = Stream.pos * token
+
+  (* Fatal. both may be thrown by tokenize *)
+  exception FsmTableIsTooSmall
+  exception TkErrorAug of Stream.convPos * string
+
+  val tokenize: Stream.t -> fullToken list -> fullToken list * Stream.filesInfo
+
+  val printToken: token -> unit
+end
-- 
cgit v1.2.3