Some basic parsing, with interned strings.

src/syntax/token_stream.rs

@@ -0,0 +1,99 @@
+use crate::syntax::tokens::Token;
+use crate::util::istring::InternedString;
+use logos::{Logos,SpannedIter};
+use std::fs::File;
+use std::io;
+use std::io::Read;
+
+pub struct TokenStream<'s> {
+    filename: InternedString,
+    lexer: SpannedIter<'s, Token>,
+}
+
+impl<'s> TokenStream<'s> {
+    pub fn new(filename: &str, s: &'s str) -> TokenStream<'s> {
+        TokenStream {
+            filename: InternedString::new(filename),
+            lexer: Token::lexer(s).spanned()
+        }
+    }
+
+    pub fn from_file(filename: &str, buffer: &'s mut String) -> io::Result<TokenStream<'s>> {
+        let mut file = File::open(filename)?;
+        file.read_to_string(buffer)?;
+        Ok(TokenStream::new(filename, buffer))
+    }
+}
+
+#[derive(Clone,Debug,PartialEq)]
+pub enum Location {
+    InFile(InternedString, usize),
+    Manufactured
+}
+
+impl Location {
+    fn new(filename: InternedString, offset: usize) -> Location {
+        Location::InFile(filename, offset)
+    }
+}
+
+impl Default for Location {
+    fn default() -> Self {
+        Location::Manufactured
+    }
+}
+
+#[derive(Debug,PartialEq)]
+pub struct LexerError {
+    filename: InternedString,
+    offset: usize
+}
+
+#[cfg(test)]
+impl LexerError {
+    fn new(filename: InternedString, offset: usize) -> LexerError {
+        LexerError{ filename, offset, }
+    }
+}
+
+type LocatedToken = Result<(Location, Token, Location),LexerError>;
+
+impl<'s> Iterator for TokenStream<'s> {
+    type Item = LocatedToken;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.lexer.next() {
+            None => None,
+            Some((Token::Error, span)) => {
+                Some(Err(LexerError {
+                    filename: self.filename,
+                    offset: span.start,
+                }))
+            }
+            Some((token, span)) => {
+                let start = Location::new(self.filename, span.start);
+                let end = Location::new(self.filename, span.end);
+                Some(Ok((start, token, end)))
+            }
+        }
+    }
+}
+
+#[test]
+fn stream_works() {
+    let fname = InternedString::new("<file>");
+    let mut lex0 = TokenStream::new("<file>", "y = x + 1//foo");
+    assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 0), Token::var("y"), Location::new(fname, 1)))));
+    assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 2), Token::Equals, Location::new(fname, 3)))));
+    assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 4), Token::var("x"), Location::new(fname, 5)))));
+    assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 6), Token::Operator('+'), Location::new(fname, 7)))));
+    assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 8), Token::Number((None, 1)), Location::new(fname, 9)))));
+    assert_eq!(lex0.next(), None);
+}
+
+#[test]
+fn errors_work() {
+    let fname = InternedString::new("<file>");
+    let mut lex0 = TokenStream::new("<file>", "\u{2639}");
+    assert_eq!(lex0.next(), Some(Err(LexerError::new(fname, 0))));
+}