Simpler tokenizer

1 files changed, 5 insertions, 3 deletions

diff --git a/stream.sml b/stream.sml
index 3e18f2e..4d58911 100644
--- a/stream.sml
+++ b/stream.sml
@@ -37,8 +37,8 @@ structure Stream :> STREAM = struct
   end z
 
   fun getchar (S as { contents, off, ... }: t) =
-    (SOME $ String.sub (contents, off), updateStream S s#off (off + 1) %)
-      handle Subscript => (NONE, S)
+    (String.sub (contents, off), updateStream S s#off (off + 1) %)
+      handle Subscript => (chr 0, S)
 
   fun ungetc ({ off = 0, ... }: t) =
       raise UngetcError
@@ -100,7 +100,7 @@ structure Stream :> STREAM = struct
         (line, col)
       else
         calcPos (curOff + 1)
-            (if String.sub (contents,curOff) = #"\n" then (line + 1, 1)
+            (if String.sub (contents, curOff) = #"\n" then (line + 1, 1)
                else (line, col + 1))
     val (line, col) = calcPos prevOff (line, col)
   in
@@ -111,6 +111,8 @@ structure Stream :> STREAM = struct
   fun getPos (S as { off, ... }: t) =
     getPosRaw off S
 
+  fun getPosDisc s = #1 $ getPos s
+
   fun EOFpos (S as { contents, ... }: t) =
     getPosRaw (String.size contents) S