✍️ Switch to a handwritten lexer and parser. #1

Open
acw wants to merge 33 commits from handwritten-lexer into master
5 changed files with 556 additions and 71 deletions
Showing only changes of commit f6bf3dd639 - Show all commits

View File

@@ -110,13 +110,14 @@ pub enum ExportClass {
#[derive(Debug)] #[derive(Debug)]
pub enum Statement { pub enum Statement {
Binding(BindingStmt), Binding(BindingStmt),
Expression(Expression),
} }
#[derive(Debug)] #[derive(Debug)]
pub struct BindingStmt { pub struct BindingStmt {
location: Location, location: Location,
mutable: bool, mutable: bool,
variable: String, variable: Name,
value: Expression, value: Expression,
} }
@@ -126,7 +127,17 @@ pub enum Expression {
Reference(Name), Reference(Name),
EnumerationValue(Name, Name, Option<Box<Expression>>), EnumerationValue(Name, Name, Option<Box<Expression>>),
StructureValue(Name, Vec<FieldValue>), StructureValue(Name, Vec<FieldValue>),
Conditional(ConditionalExpr),
Call(Box<Expression>, CallKind, Vec<Expression>), Call(Box<Expression>, CallKind, Vec<Expression>),
Block(Location, Vec<Statement>),
}
#[derive(Debug)]
pub struct ConditionalExpr {
location: Location,
test: Box<Expression>,
consequent: Box<Expression>,
alternative: Option<Box<Expression>>,
} }
#[derive(Debug)] #[derive(Debug)]

View File

@@ -1,4 +1,4 @@
use crate::syntax::{Located, Location}; use crate::syntax::Location;
use std::cmp; use std::cmp;
use std::fmt; use std::fmt;
use std::hash; use std::hash;

View File

@@ -7,7 +7,9 @@ pub struct Parser<'a> {
file_id: usize, file_id: usize,
lexer: Lexer<'a>, lexer: Lexer<'a>,
known_tokens: Vec<LocatedToken>, known_tokens: Vec<LocatedToken>,
precedence_table: HashMap<String, (u8, u8)>, prefix_precedence_table: HashMap<String, u8>,
infix_precedence_table: HashMap<String, (u8, u8)>,
postfix_precedence_table: HashMap<String, u8>,
} }
pub enum Associativity { pub enum Associativity {
@@ -20,7 +22,7 @@ impl<'a> Parser<'a> {
/// Create a new parser from the given file index and lexer. /// Create a new parser from the given file index and lexer.
/// ///
/// The file index will be used for annotating locations and for /// The file index will be used for annotating locations and for
/// error messages. If you don't care about either, you can use /// error messages. If you don't care about either, you can use
/// 0 with no loss of functionality. (Obviously, it will be harder /// 0 with no loss of functionality. (Obviously, it will be harder
/// to create quality error messages, but you already knew that.) /// to create quality error messages, but you already knew that.)
pub fn new(file_id: usize, lexer: Lexer<'a>) -> Parser<'a> { pub fn new(file_id: usize, lexer: Lexer<'a>) -> Parser<'a> {
@@ -28,7 +30,9 @@ impl<'a> Parser<'a> {
file_id, file_id,
lexer, lexer,
known_tokens: vec![], known_tokens: vec![],
precedence_table: HashMap::new(), prefix_precedence_table: HashMap::new(),
infix_precedence_table: HashMap::new(),
postfix_precedence_table: HashMap::new(),
} }
} }
@@ -38,7 +42,7 @@ impl<'a> Parser<'a> {
&mut self, &mut self,
operator: S, operator: S,
associativity: Associativity, associativity: Associativity,
level: u8 level: u8,
) { ) {
let actual_associativity = match associativity { let actual_associativity = match associativity {
Associativity::Left => (level * 2, (level * 2) + 1), Associativity::Left => (level * 2, (level * 2) + 1),
@@ -46,11 +50,22 @@ impl<'a> Parser<'a> {
Associativity::None => (level * 2, level * 2), Associativity::None => (level * 2, level * 2),
}; };
self.precedence_table.insert(operator.to_string(), actual_associativity); self.infix_precedence_table
.insert(operator.to_string(), actual_associativity);
}
pub fn add_prefix_precedence<S: ToString>(&mut self, operator: S, level: u8) {
self.prefix_precedence_table
.insert(operator.to_string(), level * 2);
}
pub fn add_postfix_precedence<S: ToString>(&mut self, operator: S, level: u8) {
self.postfix_precedence_table
.insert(operator.to_string(), level * 2);
} }
fn get_precedence(&self, name: &String) -> (u8, u8) { fn get_precedence(&self, name: &String) -> (u8, u8) {
match self.precedence_table.get(name) { match self.infix_precedence_table.get(name) {
None => (19, 20), None => (19, 20),
Some(x) => *x, Some(x) => *x,
} }
@@ -588,14 +603,17 @@ impl<'a> Parser<'a> {
} }
pub fn parse_expression(&mut self) -> Result<Expression, ParserError> { pub fn parse_expression(&mut self) -> Result<Expression, ParserError> {
let next = self.next()?.ok_or_else(|| let next = self
self.bad_eof("looking for an expression"))?; .next()?
.ok_or_else(|| self.bad_eof("looking for an expression"))?;
self.save(next.clone()); self.save(next.clone());
match next.token { match next.token {
Token::ValueName(x) if x == "match" => self.parse_match_expression(), Token::ValueName(x) if x == "match" => self.parse_match_expression(),
Token::ValueName(x) if x == "if" => self.parse_if_expression(), Token::ValueName(x) if x == "if" => {
_ => self.parse_infix(0), Ok(Expression::Conditional(self.parse_if_expression()?))
}
_ => self.parse_arithmetic(0),
} }
} }
@@ -603,12 +621,244 @@ impl<'a> Parser<'a> {
unimplemented!() unimplemented!()
} }
pub fn parse_if_expression(&mut self) -> Result<Expression, ParserError> { pub fn parse_if_expression(&mut self) -> Result<ConditionalExpr, ParserError> {
unimplemented!() let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for an 'if' to start conditional"))?;
if !matches!(next.token, Token::ValueName(ref x) if x == "if") {
return Err(ParserError::UnexpectedToken {
file_id: self.file_id,
span: next.span,
token: next.token,
expected: "an 'if' to start a conditional",
});
}
let start = self.to_location(next.span);
let test = self.parse_arithmetic(0)?;
let consequent = self.parse_block()?;
let maybe_else = self.next()?;
let (alternative, location) = match maybe_else {
Some(LocatedToken {
token: Token::ValueName(ref n),
..
}) if n == "else" => {
let expr = self.parse_block()?;
let location = match expr {
Expression::Block(ref l, _) => l.clone(),
_ => panic!("How did parse_block not return a block?!"),
};
(Some(Box::new(expr)), location)
}
_ => {
let location = match consequent {
Expression::Block(ref l, _) => l.clone(),
_ => panic!("How did parse_block not return a block?!"),
};
(None, location)
}
};
Ok(ConditionalExpr {
location,
test: Box::new(test),
consequent: Box::new(consequent),
alternative,
})
} }
pub fn parse_infix(&mut self, level: u8) -> Result<Expression, ParserError> { pub fn parse_block(&mut self) -> Result<Expression, ParserError> {
let mut lhs = self.parse_base_expression()?; let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for open brace to start block"))?;
if !matches!(next.token, Token::OpenBrace) {
return Err(ParserError::UnexpectedToken {
file_id: self.file_id,
span: next.span,
token: next.token,
expected: "an open brace to start a block",
});
}
let start = self.to_location(next.span);
let mut statements = vec![];
let mut ended_with_expr = false;
while let Some((stmt, terminal)) = self.parse_statement()? {
statements.push(stmt);
if terminal {
ended_with_expr = true;
break;
}
}
let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for statement or block close"))?;
if !matches!(next.token, Token::CloseBrace) {
return Err(ParserError::UnexpectedToken {
file_id: self.file_id,
span: next.span,
token: next.token,
expected: "a close brace to end a block",
});
}
let end = self.to_location(next.span);
if !ended_with_expr {
let void_name = Name::new(end.clone(), "%prim%void");
let void_ref = Expression::Reference(void_name);
let void_call = Expression::Call(Box::new(void_ref), CallKind::Normal, vec![]);
statements.push(Statement::Expression(void_call));
}
Ok(Expression::Block(start.extend_to(&end), statements))
}
pub fn parse_statement(&mut self) -> Result<Option<(Statement, bool)>, ParserError> {
loop {
let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for a statement or close brace"))?;
match next.token {
Token::CloseBrace => {
self.save(next);
return Ok(None);
}
Token::ValueName(ref l) if l == "let" => {
self.save(next);
return Ok(Some((Statement::Binding(self.parse_let()?), false)));
}
_ => {
self.save(next);
let expr = Statement::Expression(self.parse_expression()?);
let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for semicolon or close brace"))?;
if matches!(next.token, Token::Semi) {
return Ok(Some((expr, false)));
} else {
self.save(next);
return Ok(Some((expr, true)));
}
}
}
}
}
pub fn parse_let(&mut self) -> Result<BindingStmt, ParserError> {
let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for a let for a binding statement"))?;
if !matches!(next.token, Token::ValueName(ref n) if n == "let") {
self.save(next.clone());
return Err(ParserError::UnexpectedToken {
file_id: self.file_id,
span: next.span,
token: next.token,
expected: "a 'let' to open a binding statement",
});
}
let start = self.to_location(next.span);
let next = self
.next()?
.ok_or_else(|| self.bad_eof("'mut' or a variable name"))?;
let mutable = matches!(next.token, Token::ValueName(ref n) if n == "mut");
if !mutable {
self.save(next);
}
let next = self
.next()?
.ok_or_else(|| self.bad_eof("a variable name"))?;
let variable = match next.token {
Token::ValueName(v) => Name::new(self.to_location(next.span), v),
_ => {
return Err(ParserError::UnexpectedToken {
file_id: self.file_id,
span: next.span,
token: next.token,
expected: "a variable name for the let binding",
});
}
};
let next = self
.next()?
.ok_or_else(|| self.bad_eof("an '=' after a variable name in a binding"))?;
if !matches!(next.token, Token::OperatorName(ref x) if x == "=") {
return Err(ParserError::UnexpectedToken {
file_id: self.file_id,
span: next.span,
token: next.token,
expected: "an '=' after the variable name in a let binding",
});
}
let value = self.parse_expression()?;
let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for terminal semicolon for let statement"))?;
if !matches!(next.token, Token::Semi) {
return Err(ParserError::UnexpectedToken {
file_id: self.file_id,
span: next.span,
token: next.token,
expected: "a semicolon to finish a let statement",
});
}
let end = self.to_location(next.span);
Ok(BindingStmt {
location: start.extend_to(&end),
mutable,
variable,
value,
})
}
pub fn parse_arithmetic(&mut self, level: u8) -> Result<Expression, ParserError> {
// start by checking for prefix operators.
let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for arithmetic expression"))?;
let mut lhs = if let Token::OperatorName(ref n) = next.token {
if let Some(pre_prec) = self.prefix_precedence_table.get(n) {
if *pre_prec < level {
self.save(next.clone());
return Err(ParserError::UnexpectedToken {
file_id: self.file_id,
span: next.span,
token: next.token,
expected: "a base expression of a tighter-binding prefix operator",
});
}
let rhs = self.parse_arithmetic(*pre_prec)?;
let opname = Name::new(self.to_location(next.span), n);
let op_expr = Expression::Reference(opname);
Expression::Call(Box::new(op_expr), CallKind::Prefix, vec![rhs])
} else {
self.save(next);
self.parse_base_expression()?
}
} else {
self.save(next);
self.parse_base_expression()?
};
loop { loop {
let Some(next) = self.next()? else { let Some(next) = self.next()? else {
@@ -621,7 +871,21 @@ impl<'a> Parser<'a> {
let args = self.parse_call_arguments()?; let args = self.parse_call_arguments()?;
lhs = Expression::Call(Box::new(lhs), CallKind::Normal, args); lhs = Expression::Call(Box::new(lhs), CallKind::Normal, args);
} }
Token::OperatorName(ref n) => { Token::OperatorName(ref n) => {
if let Some(postprec) = self.postfix_precedence_table.get(n) {
if *postprec < level {
self.save(next);
break;
}
let opname = Name::new(self.to_location(next.span), n);
let op_expr = Expression::Reference(opname);
lhs = Expression::Call(Box::new(op_expr), CallKind::Postfix, vec![lhs]);
continue;
}
let (left_pr, right_pr) = self.get_precedence(&n); let (left_pr, right_pr) = self.get_precedence(&n);
if left_pr < level { if left_pr < level {
@@ -629,13 +893,14 @@ impl<'a> Parser<'a> {
break; break;
} }
let rhs = self.parse_infix(right_pr)?; let rhs = self.parse_arithmetic(right_pr)?;
let name = Name::new(self.to_location(next.span), n); let name = Name::new(self.to_location(next.span), n);
let opref = Box::new(Expression::Reference(name)); let opref = Box::new(Expression::Reference(name));
let args = vec![lhs, rhs]; let args = vec![lhs, rhs];
lhs = Expression::Call(opref, CallKind::Infix, args); lhs = Expression::Call(opref, CallKind::Infix, args);
} }
_ => { _ => {
self.save(next); self.save(next);
return Ok(lhs); return Ok(lhs);
@@ -647,43 +912,48 @@ impl<'a> Parser<'a> {
} }
fn parse_call_arguments(&mut self) -> Result<Vec<Expression>, ParserError> { fn parse_call_arguments(&mut self) -> Result<Vec<Expression>, ParserError> {
let next = self.next()?.ok_or_else(|| self.bad_eof( let next = self
"looking for open paren for function arguments"))?; .next()?
.ok_or_else(|| self.bad_eof("looking for open paren for function arguments"))?;
if !matches!(next.token, Token::OpenParen) { if !matches!(next.token, Token::OpenParen) {
return Err(ParserError::UnexpectedToken { return Err(ParserError::UnexpectedToken {
file_id: self.file_id, file_id: self.file_id,
span: next.span, span: next.span,
token: next.token, token: next.token,
expected: "open paren for call arguments", expected: "open paren for call arguments",
}); });
} }
let mut args = vec![]; let mut args = vec![];
loop { loop {
let next = self.next()?.ok_or_else(|| self.bad_eof( let next = self.next()?.ok_or_else(|| {
"looking for an expression or close paren in function arguments"))?; self.bad_eof("looking for an expression or close paren in function arguments")
})?;
if matches!(next.token, Token::CloseParen) { if matches!(next.token, Token::CloseParen) {
break; break;
} }
self.save(next); self.save(next);
let argument = self.parse_infix(0)?; let argument = self.parse_arithmetic(0)?;
args.push(argument); args.push(argument);
let next = self.next()?.ok_or_else(|| self.bad_eof( let next = self.next()?.ok_or_else(|| {
"looking for comma or close paren in function arguments"))?; self.bad_eof("looking for comma or close paren in function arguments")
})?;
match next.token { match next.token {
Token::Comma => continue, Token::Comma => continue,
Token::CloseParen => break, Token::CloseParen => break,
_ => return Err(ParserError::UnexpectedToken { _ => {
file_id: self.file_id, return Err(ParserError::UnexpectedToken {
span: next.span, file_id: self.file_id,
token: next.token, span: next.span,
expected: "comma or close paren in function arguments", token: next.token,
}), expected: "comma or close paren in function arguments",
});
}
} }
} }
@@ -700,7 +970,10 @@ impl<'a> Parser<'a> {
.ok_or_else(|| self.bad_eof("looking for an expression"))?; .ok_or_else(|| self.bad_eof("looking for an expression"))?;
match next.token { match next.token {
Token::OpenBrace => unimplemented!(), Token::OpenBrace => {
self.save(next);
return self.parse_block();
}
Token::OpenParen => { Token::OpenParen => {
let inner = self.parse_expression()?; let inner = self.parse_expression()?;

View File

@@ -1,5 +1,6 @@
use crate::syntax::error::ParserError;
use crate::syntax::parse::Parser; use crate::syntax::parse::Parser;
use crate::syntax::tokens::Lexer; use crate::syntax::tokens::{Lexer, Token};
use crate::syntax::*; use crate::syntax::*;
#[test] #[test]
@@ -513,9 +514,7 @@ fn structure_value() {
f2.as_printed() == "bar" && f2.as_printed() == "bar" &&
matches!(v1, Expression::Value(ConstantValue::Integer(_,_))) && matches!(v1, Expression::Value(ConstantValue::Integer(_,_))) &&
matches!(v2, Expression::Value(ConstantValue::String(_,_)))))); matches!(v2, Expression::Value(ConstantValue::String(_,_))))));
assert!(matches!( assert!(matches!(parse_st("Foo{ foo: 1,, bar: \"foo\", }"), Err(_)));
parse_st("Foo{ foo: 1,, bar: \"foo\", }"),
Err(_)));
} }
#[test] #[test]
@@ -528,7 +527,6 @@ fn infix_and_precedence() {
result.parse_expression() result.parse_expression()
}; };
assert!(matches!( assert!(matches!(
parse_ex("0"), parse_ex("0"),
Ok(Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value, .. }))) Ok(Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value, .. })))
@@ -659,15 +657,9 @@ fn calls() {
Expression::Reference(a), Expression::Reference(a),
Expression::Reference(b), Expression::Reference(b),
] if a.as_printed() == "a" && b.as_printed() == "b"))); ] if a.as_printed() == "a" && b.as_printed() == "b")));
assert!(matches!( assert!(matches!(parse_ex("f(,a,b,)"), Err(_)));
parse_ex("f(,a,b,)"), assert!(matches!(parse_ex("f(a,,b,)"), Err(_)));
Err(_))); assert!(matches!(parse_ex("f(a,b,,)"), Err(_)));
assert!(matches!(
parse_ex("f(a,,b,)"),
Err(_)));
assert!(matches!(
parse_ex("f(a,b,,)"),
Err(_)));
assert!(matches!( assert!(matches!(
parse_ex("f()()"), parse_ex("f()()"),
@@ -749,23 +741,190 @@ fn calls() {
matches!(v2, ConstantValue::Integer(_, IntegerWithBase{ value: 2, .. })))))); matches!(v2, ConstantValue::Integer(_, IntegerWithBase{ value: 2, .. }))))));
assert!(matches!( assert!(matches!(
parse_ex("a + b(2 + 3) * c"), parse_ex("a + b(2 + 3) * c"),
Ok(Expression::Call(plus, CallKind::Infix, pargs)) if Ok(Expression::Call(plus, CallKind::Infix, pargs)) if
matches!(plus.as_ref(), Expression::Reference(n) if n.as_printed() == "+") &&
matches!(pargs.as_slice(), [
Expression::Reference(a),
Expression::Call(times, CallKind::Infix, targs)
] if a.as_printed() == "a" &&
matches!(times.as_ref(), Expression::Reference(n) if n.as_printed() == "*") &&
matches!(targs.as_slice(), [
Expression::Call(b, CallKind::Normal, bargs),
Expression::Reference(c),
] if c.as_printed() == "c" &&
matches!(b.as_ref(), Expression::Reference(n) if n.as_printed() == "b") &&
matches!(bargs.as_slice(), [Expression::Call(plus, CallKind::Infix, pargs)] if
matches!(plus.as_ref(), Expression::Reference(n) if n.as_printed() == "+") && matches!(plus.as_ref(), Expression::Reference(n) if n.as_printed() == "+") &&
matches!(pargs.as_slice(), [ matches!(pargs.as_slice(), [
Expression::Reference(a), Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value: 2, .. })),
Expression::Call(times, CallKind::Infix, targs) Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value: 3, .. }))
] if a.as_printed() == "a" && ]))))));
matches!(times.as_ref(), Expression::Reference(n) if n.as_printed() == "*") && }
matches!(targs.as_slice(), [
Expression::Call(b, CallKind::Normal, bargs), #[test]
Expression::Reference(c), fn prefix_and_postfix() {
] if c.as_printed() == "c" && let parse_ex = |str| {
matches!(b.as_ref(), Expression::Reference(n) if n.as_printed() == "b") && let lexer = Lexer::from(str);
matches!(bargs.as_slice(), [Expression::Call(plus, CallKind::Infix, pargs)] if let mut result = Parser::new(0, lexer);
matches!(plus.as_ref(), Expression::Reference(n) if n.as_printed() == "+") && result.add_infix_precedence("+", parse::Associativity::Left, 4);
matches!(pargs.as_slice(), [ result.add_infix_precedence("*", parse::Associativity::Left, 8);
Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value: 2, .. })), result.add_prefix_precedence("++", 6);
Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value: 3, .. })) result.add_postfix_precedence("++", 6);
])))))); result.add_prefix_precedence("--", 7);
result.add_postfix_precedence("--", 7);
result.parse_expression()
};
assert!(matches!(
parse_ex("++a"),
Ok(Expression::Call(pp, CallKind::Prefix, args)) if
matches!(pp.as_ref(), Expression::Reference(n) if n.as_printed() == "++") &&
matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "a")));
assert!(matches!(
parse_ex("a--"),
Ok(Expression::Call(pp, CallKind::Postfix, args)) if
matches!(pp.as_ref(), Expression::Reference(n) if n.as_printed() == "--") &&
matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "a")));
// the prefix is weaker than the postfix, so it should be the outside
// operatotr
assert!(matches!(
parse_ex("++a--"),
Ok(Expression::Call(pp, CallKind::Prefix, args)) if
matches!(pp.as_ref(), Expression::Reference(n) if n.as_printed() == "++") &&
matches!(args.as_slice(), [Expression::Call(mm, CallKind::Postfix, args)] if
matches!(mm.as_ref(), Expression::Reference(n) if n.as_printed() == "--") &&
matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "a"))));
// the prefix is stronger than the postfix, so it should be the inside
// operator
assert!(matches!(
parse_ex("--a++"),
Ok(Expression::Call(pp, CallKind::Postfix, args)) if
matches!(pp.as_ref(), Expression::Reference(n) if n.as_printed() == "++") &&
matches!(args.as_slice(), [Expression::Call(mm, CallKind::Prefix, args)] if
matches!(mm.as_ref(), Expression::Reference(n) if n.as_printed() == "--") &&
matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "a"))));
assert!(matches!(
parse_ex("a++ + b"),
Ok(Expression::Call(p, CallKind::Infix, args)) if
matches!(p.as_ref(), Expression::Reference(n) if n.as_printed() == "+") &&
matches!(args.as_slice(), [
Expression::Call(mm, CallKind::Postfix, args),
Expression::Reference(n)
] if n.as_printed() == "b" &&
matches!(mm.as_ref(), Expression::Reference(n) if n.as_printed() == "++") &&
matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "a"))));
assert!(matches!(
parse_ex("a + ++ b"),
Ok(Expression::Call(p, CallKind::Infix, args)) if
matches!(p.as_ref(), Expression::Reference(n) if n.as_printed() == "+") &&
matches!(args.as_slice(), [
Expression::Reference(n),
Expression::Call(mm, CallKind::Prefix, args),
] if n.as_printed() == "a" &&
matches!(mm.as_ref(), Expression::Reference(n) if n.as_printed() == "++") &&
matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "b"))));
assert!(matches!(
parse_ex("a * ++ b"),
Err(ParserError::UnexpectedToken{ token: Token::OperatorName(pp), .. })
if pp == "++"));
}
#[test]
fn blocks() {
let parse_ex = |str| {
let lexer = Lexer::from(str);
let mut result = Parser::new(0, lexer);
result.parse_expression()
};
assert!(matches!(
parse_ex("{}"),
Ok(Expression::Block(_, void)) if
matches!(void.as_slice(), [Statement::Expression(call)] if
matches!(call, Expression::Call(void, CallKind::Normal, vargs) if
matches!(void.as_ref(), Expression::Reference(n) if
n.as_printed() == "%prim%void") &&
vargs.is_empty()))));
assert!(matches!(
parse_ex("{ x }"),
Ok(Expression::Block(_, x)) if
matches!(x.as_slice(), [Statement::Expression(Expression::Reference(n))] if
n.as_printed() == "x")));
assert!(matches!(
parse_ex("{ x; }"),
Ok(Expression::Block(_, x)) if
matches!(x.as_slice(), [
Statement::Expression(Expression::Reference(n)),
Statement::Expression(Expression::Call(primv, CallKind::Normal, vargs)),
] if n.as_printed() == "x" && vargs.is_empty() &&
matches!(primv.as_ref(), Expression::Reference(n) if
n.as_printed() == "%prim%void"))));
assert!(matches!(
parse_ex("{ x; y }"),
Ok(Expression::Block(_, x)) if
matches!(x.as_slice(), [
Statement::Expression(Expression::Reference(x)),
Statement::Expression(Expression::Reference(y)),
] if x.as_printed() == "x" && y.as_printed() == "y")));
}
#[test]
fn bindings() {
let parse_ex = |str| {
let lexer = Lexer::from(str);
let mut result = Parser::new(0, lexer);
result.parse_expression()
};
assert!(matches!(
parse_ex("{ let x = y; }"),
Ok(Expression::Block(_, x)) if
matches!(x.as_slice(), [Statement::Binding(b), Statement::Expression(_)] if
!b.mutable &&
b.variable.as_printed() == "x" &&
matches!(b.value, Expression::Reference(ref n) if n.as_printed() == "y"))));
}
#[test]
fn conditionals() {
let parse_ex = |str| {
let lexer = Lexer::from(str);
let mut result = Parser::new(0, lexer);
result.parse_expression()
};
assert!(matches!(
parse_ex("if x { y } else { z }"),
Ok(Expression::Conditional(cond)) if
matches!(cond.test.as_ref(), Expression::Reference(n) if n.as_printed() == "x") &&
matches!(cond.consequent.as_ref(), Expression::Block(_, cs) if
matches!(cs.as_slice(), [Statement::Expression(Expression::Reference(n))] if
n.as_printed() == "y")) &&
matches!(cond.alternative.as_ref(), Some(expr) if
matches!(expr.as_ref(), Expression::Block(_, ast) if
matches!(ast.as_slice(), [Statement::Expression(Expression::Reference(n))] if
n.as_printed() == "z")))));
assert!(matches!(
parse_ex("if x { y }"),
Ok(Expression::Conditional(cond)) if
matches!(cond.test.as_ref(), Expression::Reference(n) if n.as_printed() == "x") &&
matches!(cond.consequent.as_ref(), Expression::Block(_, cs) if
matches!(cs.as_slice(), [Statement::Expression(Expression::Reference(n))] if
n.as_printed() == "y")) &&
cond.alternative.is_none()));
assert!(matches!(parse_ex("if x v { z }"), Err(_)));
assert!(matches!(
parse_ex("if x + y { z }"),
Ok(Expression::Conditional(cond)) if
matches!(cond.test.as_ref(), Expression::Call(_, CallKind::Infix, _))));
} }

View File

@@ -476,10 +476,33 @@ impl<'a> LexerState<'a> {
token: Token::OperatorName("-".into()), token: Token::OperatorName("-".into()),
span: token_start_offset..token_start_offset + 1, span: token_start_offset..token_start_offset + 1,
})), })),
Some((end, '>')) => Ok(Some(LocatedToken { Some((end, '>')) => {
token: Token::Arrow, let Some((pbloc, peekaboo)) = self.next_char() else {
span: token_start_offset..end, return Ok(Some(LocatedToken {
})), token: Token::Arrow,
span: token_start_offset..end,
}));
};
let is_operator = !peekaboo.is_alphanumeric()
&& !peekaboo.is_whitespace()
&& !peekaboo.is_control();
if is_operator {
self.parse_identifier(
token_start_offset,
format!("->{peekaboo}"),
|c| !c.is_alphanumeric() && !c.is_whitespace() && !c.is_control(),
Token::OperatorName,
)
} else {
self.stash_char(pbloc, peekaboo);
Ok(Some(LocatedToken {
token: Token::Arrow,
span: token_start_offset..end,
}))
}
}
Some((_, c)) if !c.is_alphanumeric() && !c.is_whitespace() && !c.is_control() => self Some((_, c)) if !c.is_alphanumeric() && !c.is_whitespace() && !c.is_control() => self
.parse_identifier( .parse_identifier(
token_start_offset, token_start_offset,
@@ -665,3 +688,22 @@ fn can_separate_pieces() {
assert_eq!(Some(Token::ValueName("b".into())), next_token()); assert_eq!(Some(Token::ValueName("b".into())), next_token());
assert_eq!(None, next_token()); assert_eq!(None, next_token());
} }
#[test]
fn arrow_requires_nonop() {
let mut lexer = Lexer::from("->");
let mut next_token = move || lexer.next().map(|x| x.expect("Can read valid token").token);
assert_eq!(Some(Token::Arrow), next_token());
let mut lexer = Lexer::from("->*");
let mut next_token = move || lexer.next().map(|x| x.expect("Can read valid token").token);
assert_eq!(Some(Token::OperatorName("->*".into())), next_token());
let mut lexer = Lexer::from("->*x");
let mut next_token = move || lexer.next().map(|x| x.expect("Can read valid token").token);
assert_eq!(Some(Token::OperatorName("->*".into())), next_token());
let mut lexer = Lexer::from("->x");
let mut next_token = move || lexer.next().map(|x| x.expect("Can read valid token").token);
assert_eq!(Some(Token::Arrow), next_token());
}