use crate::syntax::tokens::Token; use crate::util::istring::InternedString; use logos::{Logos,SpannedIter}; use std::fs::File; use std::io; use std::io::Read; pub struct TokenStream<'s> { filename: InternedString, lexer: SpannedIter<'s, Token>, } impl<'s> TokenStream<'s> { pub fn new(filename: &str, s: &'s str) -> TokenStream<'s> { TokenStream { filename: InternedString::new(filename), lexer: Token::lexer(s).spanned() } } pub fn from_file(filename: &str, buffer: &'s mut String) -> io::Result> { let mut file = File::open(filename)?; file.read_to_string(buffer)?; Ok(TokenStream::new(filename, buffer)) } } #[derive(Clone,Debug,PartialEq)] pub enum Location { InFile(InternedString, usize), Manufactured } impl Location { fn new(filename: InternedString, offset: usize) -> Location { Location::InFile(filename, offset) } } impl Default for Location { fn default() -> Self { Location::Manufactured } } #[derive(Debug,PartialEq)] pub struct LexerError { filename: InternedString, offset: usize } #[cfg(test)] impl LexerError { fn new(filename: InternedString, offset: usize) -> LexerError { LexerError{ filename, offset, } } } type LocatedToken = Result<(Location, Token, Location),LexerError>; impl<'s> Iterator for TokenStream<'s> { type Item = LocatedToken; fn next(&mut self) -> Option { match self.lexer.next() { None => None, Some((Token::Error, span)) => { Some(Err(LexerError { filename: self.filename, offset: span.start, })) } Some((token, span)) => { let start = Location::new(self.filename, span.start); let end = Location::new(self.filename, span.end); Some(Ok((start, token, end))) } } } } #[test] fn stream_works() { let fname = InternedString::new(""); let mut lex0 = TokenStream::new("", "y = x + 1//foo"); assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 0), Token::var("y"), Location::new(fname, 1))))); assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 2), Token::Equals, Location::new(fname, 3))))); assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 4), Token::var("x"), Location::new(fname, 5))))); assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 6), Token::Operator('+'), Location::new(fname, 7))))); assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 8), Token::Number((None, 1)), Location::new(fname, 9))))); assert_eq!(lex0.next(), None); } #[test] fn errors_work() { let fname = InternedString::new(""); let mut lex0 = TokenStream::new("", "\u{2639}"); assert_eq!(lex0.next(), Some(Err(LexerError::new(fname, 0)))); }