use crate::syntax::tokens::Token; use logos::{Logos, SpannedIter}; use std::fmt; use thiserror::Error; pub struct TokenStream<'s> { file_idx: usize, lexer: SpannedIter<'s, Token>, } impl<'s> TokenStream<'s> { pub fn new(file_idx: usize, s: &'s str) -> TokenStream<'s> { TokenStream { file_idx, lexer: Token::lexer(s).spanned(), } } } #[derive(Clone, Debug, PartialEq, Eq)] pub enum Location { InFile(usize, usize), Manufactured, } impl fmt::Display for Location { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Location::InFile(s, off) => write!(f, "{}:{}", s, off), Location::Manufactured => write!(f, ""), } } } impl Location { fn new(file_idx: usize, offset: usize) -> Location { Location::InFile(file_idx, offset) } } impl Default for Location { fn default() -> Self { Location::Manufactured } } type LocatedToken = Result<(Location, Token, Location), LexerError>; impl<'s> Iterator for TokenStream<'s> { type Item = LocatedToken; fn next(&mut self) -> Option { match self.lexer.next() { None => None, Some((Token::Error, span)) => Some(Err(LexerError::new(self.file_idx, span.start))), Some((token, span)) => { let start = Location::new(self.file_idx, span.start); let end = Location::new(self.file_idx, span.end); Some(Ok((start, token, end))) } } } } #[test] fn stream_works() { let fidx = 42; let mut lex0 = TokenStream::new(42, "y = x + 1//foo"); assert_eq!( lex0.next(), Some(Ok(( Location::new(fidx, 0), Token::var("y"), Location::new(fidx, 1) ))) ); assert_eq!( lex0.next(), Some(Ok(( Location::new(fidx, 2), Token::Equals, Location::new(fidx, 3) ))) ); assert_eq!( lex0.next(), Some(Ok(( Location::new(fidx, 4), Token::var("x"), Location::new(fidx, 5) ))) ); assert_eq!( lex0.next(), Some(Ok(( Location::new(fidx, 6), Token::Operator('+'), Location::new(fidx, 7) ))) ); assert_eq!( lex0.next(), Some(Ok(( Location::new(fidx, 8), Token::Number((None, 1)), Location::new(fidx, 9) ))) ); assert_eq!(lex0.next(), None); } #[test] fn errors_work() { let fidx = 2; let mut lex0 = TokenStream::new(2, "\u{2639}"); assert_eq!(lex0.next(), Some(Err(LexerError::new(fidx, 0)))); }