From 3a32398248e5593b1b536c837478cab276f7aebf Mon Sep 17 00:00:00 2001
From: Vladimir Azarov <avm@intermediate-node.net>
Date: Mon, 26 May 2025 21:06:51 +0200
Subject: Simpler tokenizer

---
 stream.sml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'stream.sml')

diff --git a/stream.sml b/stream.sml
index 3e18f2e..4d58911 100644
--- a/stream.sml
+++ b/stream.sml
@@ -37,8 +37,8 @@ structure Stream :> STREAM = struct
   end z
 
   fun getchar (S as { contents, off, ... }: t) =
-    (SOME $ String.sub (contents, off), updateStream S s#off (off + 1) %)
-      handle Subscript => (NONE, S)
+    (String.sub (contents, off), updateStream S s#off (off + 1) %)
+      handle Subscript => (chr 0, S)
 
   fun ungetc ({ off = 0, ... }: t) =
       raise UngetcError
@@ -100,7 +100,7 @@ structure Stream :> STREAM = struct
         (line, col)
       else
         calcPos (curOff + 1)
-            (if String.sub (contents,curOff) = #"\n" then (line + 1, 1)
+            (if String.sub (contents, curOff) = #"\n" then (line + 1, 1)
                else (line, col + 1))
     val (line, col) = calcPos prevOff (line, col)
   in
@@ -111,6 +111,8 @@ structure Stream :> STREAM = struct
   fun getPos (S as { off, ... }: t) =
     getPosRaw off S
 
+  fun getPosDisc s = #1 $ getPos s
+
   fun EOFpos (S as { contents, ... }: t) =
     getPosRaw (String.size contents) S
 
-- 
cgit v1.2.3