Adam Wick
b2f6b12ced
This implements a full compiler, with both static compilation and JIT support, for the world's simplest and silliest programming language. You can do math, and print variables. That's it. On the bright side, it implements every part of the compiler, from the lexer and parser; through analysis and simplification; and into a reasonable code generator. This should be a good jumping off point for adding more advanced features. Tests, including proptests, are included to help avoid regressions.
118 lines
2.7 KiB
Rust
118 lines
2.7 KiB
Rust
use crate::syntax::tokens::Token;
|
|
use logos::{Logos, SpannedIter};
|
|
use std::fmt;
|
|
use thiserror::Error;
|
|
|
|
pub struct TokenStream<'s> {
|
|
file_idx: usize,
|
|
lexer: SpannedIter<'s, Token>,
|
|
}
|
|
|
|
impl<'s> TokenStream<'s> {
|
|
pub fn new(file_idx: usize, s: &'s str) -> TokenStream<'s> {
|
|
TokenStream {
|
|
file_idx,
|
|
lexer: Token::lexer(s).spanned(),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub enum Location {
|
|
InFile(usize, usize),
|
|
Manufactured,
|
|
}
|
|
|
|
impl fmt::Display for Location {
|
|
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
|
match self {
|
|
Location::InFile(s, off) => write!(f, "{}:{}", s, off),
|
|
Location::Manufactured => write!(f, "<manufactured>"),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Location {
|
|
fn new(file_idx: usize, offset: usize) -> Location {
|
|
Location::InFile(file_idx, offset)
|
|
}
|
|
}
|
|
|
|
impl Default for Location {
|
|
fn default() -> Self {
|
|
Location::Manufactured
|
|
}
|
|
}
|
|
|
|
type LocatedToken = Result<(Location, Token, Location), LexerError>;
|
|
|
|
impl<'s> Iterator for TokenStream<'s> {
|
|
type Item = LocatedToken;
|
|
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
match self.lexer.next() {
|
|
None => None,
|
|
Some((Token::Error, span)) => Some(Err(LexerError::new(self.file_idx, span.start))),
|
|
Some((token, span)) => {
|
|
let start = Location::new(self.file_idx, span.start);
|
|
let end = Location::new(self.file_idx, span.end);
|
|
Some(Ok((start, token, end)))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn stream_works() {
|
|
let fidx = 42;
|
|
let mut lex0 = TokenStream::new(42, "y = x + 1//foo");
|
|
assert_eq!(
|
|
lex0.next(),
|
|
Some(Ok((
|
|
Location::new(fidx, 0),
|
|
Token::var("y"),
|
|
Location::new(fidx, 1)
|
|
)))
|
|
);
|
|
assert_eq!(
|
|
lex0.next(),
|
|
Some(Ok((
|
|
Location::new(fidx, 2),
|
|
Token::Equals,
|
|
Location::new(fidx, 3)
|
|
)))
|
|
);
|
|
assert_eq!(
|
|
lex0.next(),
|
|
Some(Ok((
|
|
Location::new(fidx, 4),
|
|
Token::var("x"),
|
|
Location::new(fidx, 5)
|
|
)))
|
|
);
|
|
assert_eq!(
|
|
lex0.next(),
|
|
Some(Ok((
|
|
Location::new(fidx, 6),
|
|
Token::Operator('+'),
|
|
Location::new(fidx, 7)
|
|
)))
|
|
);
|
|
assert_eq!(
|
|
lex0.next(),
|
|
Some(Ok((
|
|
Location::new(fidx, 8),
|
|
Token::Number((None, 1)),
|
|
Location::new(fidx, 9)
|
|
)))
|
|
);
|
|
assert_eq!(lex0.next(), None);
|
|
}
|
|
|
|
#[test]
|
|
fn errors_work() {
|
|
let fidx = 2;
|
|
let mut lex0 = TokenStream::new(2, "\u{2639}");
|
|
assert_eq!(lex0.next(), Some(Err(LexerError::new(fidx, 0))));
|
|
}
|