99 lines
2.9 KiB
Rust
99 lines
2.9 KiB
Rust
use crate::syntax::tokens::Token;
|
|
use crate::util::istring::InternedString;
|
|
use logos::{Logos,SpannedIter};
|
|
use std::fs::File;
|
|
use std::io;
|
|
use std::io::Read;
|
|
|
|
pub struct TokenStream<'s> {
|
|
filename: InternedString,
|
|
lexer: SpannedIter<'s, Token>,
|
|
}
|
|
|
|
impl<'s> TokenStream<'s> {
|
|
pub fn new(filename: &str, s: &'s str) -> TokenStream<'s> {
|
|
TokenStream {
|
|
filename: InternedString::new(filename),
|
|
lexer: Token::lexer(s).spanned()
|
|
}
|
|
}
|
|
|
|
pub fn from_file(filename: &str, buffer: &'s mut String) -> io::Result<TokenStream<'s>> {
|
|
let mut file = File::open(filename)?;
|
|
file.read_to_string(buffer)?;
|
|
Ok(TokenStream::new(filename, buffer))
|
|
}
|
|
}
|
|
|
|
#[derive(Clone,Debug,PartialEq)]
|
|
pub enum Location {
|
|
InFile(InternedString, usize),
|
|
Manufactured
|
|
}
|
|
|
|
impl Location {
|
|
fn new(filename: InternedString, offset: usize) -> Location {
|
|
Location::InFile(filename, offset)
|
|
}
|
|
}
|
|
|
|
impl Default for Location {
|
|
fn default() -> Self {
|
|
Location::Manufactured
|
|
}
|
|
}
|
|
|
|
#[derive(Debug,PartialEq)]
|
|
pub struct LexerError {
|
|
filename: InternedString,
|
|
offset: usize
|
|
}
|
|
|
|
#[cfg(test)]
|
|
impl LexerError {
|
|
fn new(filename: InternedString, offset: usize) -> LexerError {
|
|
LexerError{ filename, offset, }
|
|
}
|
|
}
|
|
|
|
type LocatedToken = Result<(Location, Token, Location),LexerError>;
|
|
|
|
impl<'s> Iterator for TokenStream<'s> {
|
|
type Item = LocatedToken;
|
|
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
match self.lexer.next() {
|
|
None => None,
|
|
Some((Token::Error, span)) => {
|
|
Some(Err(LexerError {
|
|
filename: self.filename,
|
|
offset: span.start,
|
|
}))
|
|
}
|
|
Some((token, span)) => {
|
|
let start = Location::new(self.filename, span.start);
|
|
let end = Location::new(self.filename, span.end);
|
|
Some(Ok((start, token, end)))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn stream_works() {
|
|
let fname = InternedString::new("<file>");
|
|
let mut lex0 = TokenStream::new("<file>", "y = x + 1//foo");
|
|
assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 0), Token::var("y"), Location::new(fname, 1)))));
|
|
assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 2), Token::Equals, Location::new(fname, 3)))));
|
|
assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 4), Token::var("x"), Location::new(fname, 5)))));
|
|
assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 6), Token::Operator('+'), Location::new(fname, 7)))));
|
|
assert_eq!(lex0.next(), Some(Ok((Location::new(fname, 8), Token::Number((None, 1)), Location::new(fname, 9)))));
|
|
assert_eq!(lex0.next(), None);
|
|
}
|
|
|
|
#[test]
|
|
fn errors_work() {
|
|
let fname = InternedString::new("<file>");
|
|
let mut lex0 = TokenStream::new("<file>", "\u{2639}");
|
|
assert_eq!(lex0.next(), Some(Err(LexerError::new(fname, 0))));
|
|
} |