diff --git a/src/syntax.rs b/src/syntax.rs index dfcf197..35ae376 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -110,13 +110,14 @@ pub enum ExportClass { #[derive(Debug)] pub enum Statement { Binding(BindingStmt), + Expression(Expression), } #[derive(Debug)] pub struct BindingStmt { location: Location, mutable: bool, - variable: String, + variable: Name, value: Expression, } @@ -126,7 +127,17 @@ pub enum Expression { Reference(Name), EnumerationValue(Name, Name, Option>), StructureValue(Name, Vec), + Conditional(ConditionalExpr), Call(Box, CallKind, Vec), + Block(Location, Vec), +} + +#[derive(Debug)] +pub struct ConditionalExpr { + location: Location, + test: Box, + consequent: Box, + alternative: Option>, } #[derive(Debug)] diff --git a/src/syntax/name.rs b/src/syntax/name.rs index 5dd90e0..fbfdb5f 100644 --- a/src/syntax/name.rs +++ b/src/syntax/name.rs @@ -1,4 +1,4 @@ -use crate::syntax::{Located, Location}; +use crate::syntax::Location; use std::cmp; use std::fmt; use std::hash; diff --git a/src/syntax/parse.rs b/src/syntax/parse.rs index 7161071..ddb7c2e 100644 --- a/src/syntax/parse.rs +++ b/src/syntax/parse.rs @@ -7,7 +7,9 @@ pub struct Parser<'a> { file_id: usize, lexer: Lexer<'a>, known_tokens: Vec, - precedence_table: HashMap, + prefix_precedence_table: HashMap, + infix_precedence_table: HashMap, + postfix_precedence_table: HashMap, } pub enum Associativity { @@ -20,7 +22,7 @@ impl<'a> Parser<'a> { /// Create a new parser from the given file index and lexer. /// /// The file index will be used for annotating locations and for - /// error messages. If you don't care about either, you can use + /// error messages. If you don't care about either, you can use /// 0 with no loss of functionality. (Obviously, it will be harder /// to create quality error messages, but you already knew that.) pub fn new(file_id: usize, lexer: Lexer<'a>) -> Parser<'a> { @@ -28,7 +30,9 @@ impl<'a> Parser<'a> { file_id, lexer, known_tokens: vec![], - precedence_table: HashMap::new(), + prefix_precedence_table: HashMap::new(), + infix_precedence_table: HashMap::new(), + postfix_precedence_table: HashMap::new(), } } @@ -38,7 +42,7 @@ impl<'a> Parser<'a> { &mut self, operator: S, associativity: Associativity, - level: u8 + level: u8, ) { let actual_associativity = match associativity { Associativity::Left => (level * 2, (level * 2) + 1), @@ -46,11 +50,22 @@ impl<'a> Parser<'a> { Associativity::None => (level * 2, level * 2), }; - self.precedence_table.insert(operator.to_string(), actual_associativity); + self.infix_precedence_table + .insert(operator.to_string(), actual_associativity); + } + + pub fn add_prefix_precedence(&mut self, operator: S, level: u8) { + self.prefix_precedence_table + .insert(operator.to_string(), level * 2); + } + + pub fn add_postfix_precedence(&mut self, operator: S, level: u8) { + self.postfix_precedence_table + .insert(operator.to_string(), level * 2); } fn get_precedence(&self, name: &String) -> (u8, u8) { - match self.precedence_table.get(name) { + match self.infix_precedence_table.get(name) { None => (19, 20), Some(x) => *x, } @@ -588,14 +603,17 @@ impl<'a> Parser<'a> { } pub fn parse_expression(&mut self) -> Result { - let next = self.next()?.ok_or_else(|| - self.bad_eof("looking for an expression"))?; + let next = self + .next()? + .ok_or_else(|| self.bad_eof("looking for an expression"))?; self.save(next.clone()); match next.token { Token::ValueName(x) if x == "match" => self.parse_match_expression(), - Token::ValueName(x) if x == "if" => self.parse_if_expression(), - _ => self.parse_infix(0), + Token::ValueName(x) if x == "if" => { + Ok(Expression::Conditional(self.parse_if_expression()?)) + } + _ => self.parse_arithmetic(0), } } @@ -603,12 +621,244 @@ impl<'a> Parser<'a> { unimplemented!() } - pub fn parse_if_expression(&mut self) -> Result { - unimplemented!() + pub fn parse_if_expression(&mut self) -> Result { + let next = self + .next()? + .ok_or_else(|| self.bad_eof("looking for an 'if' to start conditional"))?; + if !matches!(next.token, Token::ValueName(ref x) if x == "if") { + return Err(ParserError::UnexpectedToken { + file_id: self.file_id, + span: next.span, + token: next.token, + expected: "an 'if' to start a conditional", + }); + } + let start = self.to_location(next.span); + + let test = self.parse_arithmetic(0)?; + let consequent = self.parse_block()?; + + let maybe_else = self.next()?; + let (alternative, location) = match maybe_else { + Some(LocatedToken { + token: Token::ValueName(ref n), + .. + }) if n == "else" => { + let expr = self.parse_block()?; + let location = match expr { + Expression::Block(ref l, _) => l.clone(), + _ => panic!("How did parse_block not return a block?!"), + }; + + (Some(Box::new(expr)), location) + } + + _ => { + let location = match consequent { + Expression::Block(ref l, _) => l.clone(), + _ => panic!("How did parse_block not return a block?!"), + }; + + (None, location) + } + }; + + Ok(ConditionalExpr { + location, + test: Box::new(test), + consequent: Box::new(consequent), + alternative, + }) } - pub fn parse_infix(&mut self, level: u8) -> Result { - let mut lhs = self.parse_base_expression()?; + pub fn parse_block(&mut self) -> Result { + let next = self + .next()? + .ok_or_else(|| self.bad_eof("looking for open brace to start block"))?; + if !matches!(next.token, Token::OpenBrace) { + return Err(ParserError::UnexpectedToken { + file_id: self.file_id, + span: next.span, + token: next.token, + expected: "an open brace to start a block", + }); + } + let start = self.to_location(next.span); + + let mut statements = vec![]; + let mut ended_with_expr = false; + + while let Some((stmt, terminal)) = self.parse_statement()? { + statements.push(stmt); + if terminal { + ended_with_expr = true; + break; + } + } + + let next = self + .next()? + .ok_or_else(|| self.bad_eof("looking for statement or block close"))?; + if !matches!(next.token, Token::CloseBrace) { + return Err(ParserError::UnexpectedToken { + file_id: self.file_id, + span: next.span, + token: next.token, + expected: "a close brace to end a block", + }); + } + let end = self.to_location(next.span); + + if !ended_with_expr { + let void_name = Name::new(end.clone(), "%prim%void"); + let void_ref = Expression::Reference(void_name); + let void_call = Expression::Call(Box::new(void_ref), CallKind::Normal, vec![]); + statements.push(Statement::Expression(void_call)); + } + + Ok(Expression::Block(start.extend_to(&end), statements)) + } + + pub fn parse_statement(&mut self) -> Result, ParserError> { + loop { + let next = self + .next()? + .ok_or_else(|| self.bad_eof("looking for a statement or close brace"))?; + + match next.token { + Token::CloseBrace => { + self.save(next); + return Ok(None); + } + + Token::ValueName(ref l) if l == "let" => { + self.save(next); + return Ok(Some((Statement::Binding(self.parse_let()?), false))); + } + + _ => { + self.save(next); + let expr = Statement::Expression(self.parse_expression()?); + + let next = self + .next()? + .ok_or_else(|| self.bad_eof("looking for semicolon or close brace"))?; + + if matches!(next.token, Token::Semi) { + return Ok(Some((expr, false))); + } else { + self.save(next); + return Ok(Some((expr, true))); + } + } + } + } + } + + pub fn parse_let(&mut self) -> Result { + let next = self + .next()? + .ok_or_else(|| self.bad_eof("looking for a let for a binding statement"))?; + if !matches!(next.token, Token::ValueName(ref n) if n == "let") { + self.save(next.clone()); + return Err(ParserError::UnexpectedToken { + file_id: self.file_id, + span: next.span, + token: next.token, + expected: "a 'let' to open a binding statement", + }); + } + let start = self.to_location(next.span); + + let next = self + .next()? + .ok_or_else(|| self.bad_eof("'mut' or a variable name"))?; + let mutable = matches!(next.token, Token::ValueName(ref n) if n == "mut"); + if !mutable { + self.save(next); + } + + let next = self + .next()? + .ok_or_else(|| self.bad_eof("a variable name"))?; + let variable = match next.token { + Token::ValueName(v) => Name::new(self.to_location(next.span), v), + _ => { + return Err(ParserError::UnexpectedToken { + file_id: self.file_id, + span: next.span, + token: next.token, + expected: "a variable name for the let binding", + }); + } + }; + + let next = self + .next()? + .ok_or_else(|| self.bad_eof("an '=' after a variable name in a binding"))?; + if !matches!(next.token, Token::OperatorName(ref x) if x == "=") { + return Err(ParserError::UnexpectedToken { + file_id: self.file_id, + span: next.span, + token: next.token, + expected: "an '=' after the variable name in a let binding", + }); + } + + let value = self.parse_expression()?; + + let next = self + .next()? + .ok_or_else(|| self.bad_eof("looking for terminal semicolon for let statement"))?; + if !matches!(next.token, Token::Semi) { + return Err(ParserError::UnexpectedToken { + file_id: self.file_id, + span: next.span, + token: next.token, + expected: "a semicolon to finish a let statement", + }); + } + let end = self.to_location(next.span); + + Ok(BindingStmt { + location: start.extend_to(&end), + mutable, + variable, + value, + }) + } + + pub fn parse_arithmetic(&mut self, level: u8) -> Result { + // start by checking for prefix operators. + let next = self + .next()? + .ok_or_else(|| self.bad_eof("looking for arithmetic expression"))?; + + let mut lhs = if let Token::OperatorName(ref n) = next.token { + if let Some(pre_prec) = self.prefix_precedence_table.get(n) { + if *pre_prec < level { + self.save(next.clone()); + return Err(ParserError::UnexpectedToken { + file_id: self.file_id, + span: next.span, + token: next.token, + expected: "a base expression of a tighter-binding prefix operator", + }); + } + + let rhs = self.parse_arithmetic(*pre_prec)?; + let opname = Name::new(self.to_location(next.span), n); + let op_expr = Expression::Reference(opname); + + Expression::Call(Box::new(op_expr), CallKind::Prefix, vec![rhs]) + } else { + self.save(next); + self.parse_base_expression()? + } + } else { + self.save(next); + self.parse_base_expression()? + }; loop { let Some(next) = self.next()? else { @@ -621,7 +871,21 @@ impl<'a> Parser<'a> { let args = self.parse_call_arguments()?; lhs = Expression::Call(Box::new(lhs), CallKind::Normal, args); } + Token::OperatorName(ref n) => { + if let Some(postprec) = self.postfix_precedence_table.get(n) { + if *postprec < level { + self.save(next); + break; + } + + let opname = Name::new(self.to_location(next.span), n); + let op_expr = Expression::Reference(opname); + + lhs = Expression::Call(Box::new(op_expr), CallKind::Postfix, vec![lhs]); + continue; + } + let (left_pr, right_pr) = self.get_precedence(&n); if left_pr < level { @@ -629,13 +893,14 @@ impl<'a> Parser<'a> { break; } - let rhs = self.parse_infix(right_pr)?; + let rhs = self.parse_arithmetic(right_pr)?; let name = Name::new(self.to_location(next.span), n); let opref = Box::new(Expression::Reference(name)); let args = vec![lhs, rhs]; lhs = Expression::Call(opref, CallKind::Infix, args); } + _ => { self.save(next); return Ok(lhs); @@ -647,43 +912,48 @@ impl<'a> Parser<'a> { } fn parse_call_arguments(&mut self) -> Result, ParserError> { - let next = self.next()?.ok_or_else(|| self.bad_eof( - "looking for open paren for function arguments"))?; + let next = self + .next()? + .ok_or_else(|| self.bad_eof("looking for open paren for function arguments"))?; if !matches!(next.token, Token::OpenParen) { return Err(ParserError::UnexpectedToken { - file_id: self.file_id, - span: next.span, - token: next.token, - expected: "open paren for call arguments", + file_id: self.file_id, + span: next.span, + token: next.token, + expected: "open paren for call arguments", }); } - let mut args = vec![]; + let mut args = vec![]; loop { - let next = self.next()?.ok_or_else(|| self.bad_eof( - "looking for an expression or close paren in function arguments"))?; + let next = self.next()?.ok_or_else(|| { + self.bad_eof("looking for an expression or close paren in function arguments") + })?; if matches!(next.token, Token::CloseParen) { break; } self.save(next); - let argument = self.parse_infix(0)?; + let argument = self.parse_arithmetic(0)?; args.push(argument); - let next = self.next()?.ok_or_else(|| self.bad_eof( - "looking for comma or close paren in function arguments"))?; + let next = self.next()?.ok_or_else(|| { + self.bad_eof("looking for comma or close paren in function arguments") + })?; match next.token { Token::Comma => continue, Token::CloseParen => break, - _ => return Err(ParserError::UnexpectedToken { - file_id: self.file_id, - span: next.span, - token: next.token, - expected: "comma or close paren in function arguments", - }), + _ => { + return Err(ParserError::UnexpectedToken { + file_id: self.file_id, + span: next.span, + token: next.token, + expected: "comma or close paren in function arguments", + }); + } } } @@ -700,7 +970,10 @@ impl<'a> Parser<'a> { .ok_or_else(|| self.bad_eof("looking for an expression"))?; match next.token { - Token::OpenBrace => unimplemented!(), + Token::OpenBrace => { + self.save(next); + return self.parse_block(); + } Token::OpenParen => { let inner = self.parse_expression()?; diff --git a/src/syntax/parser_tests.rs b/src/syntax/parser_tests.rs index 8be3eff..5960671 100644 --- a/src/syntax/parser_tests.rs +++ b/src/syntax/parser_tests.rs @@ -1,5 +1,6 @@ +use crate::syntax::error::ParserError; use crate::syntax::parse::Parser; -use crate::syntax::tokens::Lexer; +use crate::syntax::tokens::{Lexer, Token}; use crate::syntax::*; #[test] @@ -513,9 +514,7 @@ fn structure_value() { f2.as_printed() == "bar" && matches!(v1, Expression::Value(ConstantValue::Integer(_,_))) && matches!(v2, Expression::Value(ConstantValue::String(_,_)))))); - assert!(matches!( - parse_st("Foo{ foo: 1,, bar: \"foo\", }"), - Err(_))); + assert!(matches!(parse_st("Foo{ foo: 1,, bar: \"foo\", }"), Err(_))); } #[test] @@ -528,7 +527,6 @@ fn infix_and_precedence() { result.parse_expression() }; - assert!(matches!( parse_ex("0"), Ok(Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value, .. }))) @@ -659,15 +657,9 @@ fn calls() { Expression::Reference(a), Expression::Reference(b), ] if a.as_printed() == "a" && b.as_printed() == "b"))); - assert!(matches!( - parse_ex("f(,a,b,)"), - Err(_))); - assert!(matches!( - parse_ex("f(a,,b,)"), - Err(_))); - assert!(matches!( - parse_ex("f(a,b,,)"), - Err(_))); + assert!(matches!(parse_ex("f(,a,b,)"), Err(_))); + assert!(matches!(parse_ex("f(a,,b,)"), Err(_))); + assert!(matches!(parse_ex("f(a,b,,)"), Err(_))); assert!(matches!( parse_ex("f()()"), @@ -749,23 +741,190 @@ fn calls() { matches!(v2, ConstantValue::Integer(_, IntegerWithBase{ value: 2, .. })))))); assert!(matches!( - parse_ex("a + b(2 + 3) * c"), - Ok(Expression::Call(plus, CallKind::Infix, pargs)) if + parse_ex("a + b(2 + 3) * c"), + Ok(Expression::Call(plus, CallKind::Infix, pargs)) if + matches!(plus.as_ref(), Expression::Reference(n) if n.as_printed() == "+") && + matches!(pargs.as_slice(), [ + Expression::Reference(a), + Expression::Call(times, CallKind::Infix, targs) + ] if a.as_printed() == "a" && + matches!(times.as_ref(), Expression::Reference(n) if n.as_printed() == "*") && + matches!(targs.as_slice(), [ + Expression::Call(b, CallKind::Normal, bargs), + Expression::Reference(c), + ] if c.as_printed() == "c" && + matches!(b.as_ref(), Expression::Reference(n) if n.as_printed() == "b") && + matches!(bargs.as_slice(), [Expression::Call(plus, CallKind::Infix, pargs)] if matches!(plus.as_ref(), Expression::Reference(n) if n.as_printed() == "+") && matches!(pargs.as_slice(), [ - Expression::Reference(a), - Expression::Call(times, CallKind::Infix, targs) - ] if a.as_printed() == "a" && - matches!(times.as_ref(), Expression::Reference(n) if n.as_printed() == "*") && - matches!(targs.as_slice(), [ - Expression::Call(b, CallKind::Normal, bargs), - Expression::Reference(c), - ] if c.as_printed() == "c" && - matches!(b.as_ref(), Expression::Reference(n) if n.as_printed() == "b") && - matches!(bargs.as_slice(), [Expression::Call(plus, CallKind::Infix, pargs)] if - matches!(plus.as_ref(), Expression::Reference(n) if n.as_printed() == "+") && - matches!(pargs.as_slice(), [ - Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value: 2, .. })), - Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value: 3, .. })) - ])))))); + Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value: 2, .. })), + Expression::Value(ConstantValue::Integer(_, IntegerWithBase{ value: 3, .. })) + ])))))); +} + +#[test] +fn prefix_and_postfix() { + let parse_ex = |str| { + let lexer = Lexer::from(str); + let mut result = Parser::new(0, lexer); + result.add_infix_precedence("+", parse::Associativity::Left, 4); + result.add_infix_precedence("*", parse::Associativity::Left, 8); + result.add_prefix_precedence("++", 6); + result.add_postfix_precedence("++", 6); + result.add_prefix_precedence("--", 7); + result.add_postfix_precedence("--", 7); + result.parse_expression() + }; + + assert!(matches!( + parse_ex("++a"), + Ok(Expression::Call(pp, CallKind::Prefix, args)) if + matches!(pp.as_ref(), Expression::Reference(n) if n.as_printed() == "++") && + matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "a"))); + + assert!(matches!( + parse_ex("a--"), + Ok(Expression::Call(pp, CallKind::Postfix, args)) if + matches!(pp.as_ref(), Expression::Reference(n) if n.as_printed() == "--") && + matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "a"))); + + // the prefix is weaker than the postfix, so it should be the outside + // operatotr + assert!(matches!( + parse_ex("++a--"), + Ok(Expression::Call(pp, CallKind::Prefix, args)) if + matches!(pp.as_ref(), Expression::Reference(n) if n.as_printed() == "++") && + matches!(args.as_slice(), [Expression::Call(mm, CallKind::Postfix, args)] if + matches!(mm.as_ref(), Expression::Reference(n) if n.as_printed() == "--") && + matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "a")))); + + // the prefix is stronger than the postfix, so it should be the inside + // operator + assert!(matches!( + parse_ex("--a++"), + Ok(Expression::Call(pp, CallKind::Postfix, args)) if + matches!(pp.as_ref(), Expression::Reference(n) if n.as_printed() == "++") && + matches!(args.as_slice(), [Expression::Call(mm, CallKind::Prefix, args)] if + matches!(mm.as_ref(), Expression::Reference(n) if n.as_printed() == "--") && + matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "a")))); + + assert!(matches!( + parse_ex("a++ + b"), + Ok(Expression::Call(p, CallKind::Infix, args)) if + matches!(p.as_ref(), Expression::Reference(n) if n.as_printed() == "+") && + matches!(args.as_slice(), [ + Expression::Call(mm, CallKind::Postfix, args), + Expression::Reference(n) + ] if n.as_printed() == "b" && + matches!(mm.as_ref(), Expression::Reference(n) if n.as_printed() == "++") && + matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "a")))); + + assert!(matches!( + parse_ex("a + ++ b"), + Ok(Expression::Call(p, CallKind::Infix, args)) if + matches!(p.as_ref(), Expression::Reference(n) if n.as_printed() == "+") && + matches!(args.as_slice(), [ + Expression::Reference(n), + Expression::Call(mm, CallKind::Prefix, args), + ] if n.as_printed() == "a" && + matches!(mm.as_ref(), Expression::Reference(n) if n.as_printed() == "++") && + matches!(args.as_slice(), [Expression::Reference(n)] if n.as_printed() == "b")))); + + assert!(matches!( + parse_ex("a * ++ b"), + Err(ParserError::UnexpectedToken{ token: Token::OperatorName(pp), .. }) + if pp == "++")); +} + +#[test] +fn blocks() { + let parse_ex = |str| { + let lexer = Lexer::from(str); + let mut result = Parser::new(0, lexer); + result.parse_expression() + }; + + assert!(matches!( + parse_ex("{}"), + Ok(Expression::Block(_, void)) if + matches!(void.as_slice(), [Statement::Expression(call)] if + matches!(call, Expression::Call(void, CallKind::Normal, vargs) if + matches!(void.as_ref(), Expression::Reference(n) if + n.as_printed() == "%prim%void") && + vargs.is_empty())))); + assert!(matches!( + parse_ex("{ x }"), + Ok(Expression::Block(_, x)) if + matches!(x.as_slice(), [Statement::Expression(Expression::Reference(n))] if + n.as_printed() == "x"))); + assert!(matches!( + parse_ex("{ x; }"), + Ok(Expression::Block(_, x)) if + matches!(x.as_slice(), [ + Statement::Expression(Expression::Reference(n)), + Statement::Expression(Expression::Call(primv, CallKind::Normal, vargs)), + ] if n.as_printed() == "x" && vargs.is_empty() && + matches!(primv.as_ref(), Expression::Reference(n) if + n.as_printed() == "%prim%void")))); + assert!(matches!( + parse_ex("{ x; y }"), + Ok(Expression::Block(_, x)) if + matches!(x.as_slice(), [ + Statement::Expression(Expression::Reference(x)), + Statement::Expression(Expression::Reference(y)), + ] if x.as_printed() == "x" && y.as_printed() == "y"))); +} + +#[test] +fn bindings() { + let parse_ex = |str| { + let lexer = Lexer::from(str); + let mut result = Parser::new(0, lexer); + result.parse_expression() + }; + + assert!(matches!( + parse_ex("{ let x = y; }"), + Ok(Expression::Block(_, x)) if + matches!(x.as_slice(), [Statement::Binding(b), Statement::Expression(_)] if + !b.mutable && + b.variable.as_printed() == "x" && + matches!(b.value, Expression::Reference(ref n) if n.as_printed() == "y")))); +} + +#[test] +fn conditionals() { + let parse_ex = |str| { + let lexer = Lexer::from(str); + let mut result = Parser::new(0, lexer); + result.parse_expression() + }; + + assert!(matches!( + parse_ex("if x { y } else { z }"), + Ok(Expression::Conditional(cond)) if + matches!(cond.test.as_ref(), Expression::Reference(n) if n.as_printed() == "x") && + matches!(cond.consequent.as_ref(), Expression::Block(_, cs) if + matches!(cs.as_slice(), [Statement::Expression(Expression::Reference(n))] if + n.as_printed() == "y")) && + matches!(cond.alternative.as_ref(), Some(expr) if + matches!(expr.as_ref(), Expression::Block(_, ast) if + matches!(ast.as_slice(), [Statement::Expression(Expression::Reference(n))] if + n.as_printed() == "z"))))); + + assert!(matches!( + parse_ex("if x { y }"), + Ok(Expression::Conditional(cond)) if + matches!(cond.test.as_ref(), Expression::Reference(n) if n.as_printed() == "x") && + matches!(cond.consequent.as_ref(), Expression::Block(_, cs) if + matches!(cs.as_slice(), [Statement::Expression(Expression::Reference(n))] if + n.as_printed() == "y")) && + cond.alternative.is_none())); + + assert!(matches!(parse_ex("if x v { z }"), Err(_))); + + assert!(matches!( + parse_ex("if x + y { z }"), + Ok(Expression::Conditional(cond)) if + matches!(cond.test.as_ref(), Expression::Call(_, CallKind::Infix, _)))); } diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 08f5885..a990409 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -476,10 +476,33 @@ impl<'a> LexerState<'a> { token: Token::OperatorName("-".into()), span: token_start_offset..token_start_offset + 1, })), - Some((end, '>')) => Ok(Some(LocatedToken { - token: Token::Arrow, - span: token_start_offset..end, - })), + Some((end, '>')) => { + let Some((pbloc, peekaboo)) = self.next_char() else { + return Ok(Some(LocatedToken { + token: Token::Arrow, + span: token_start_offset..end, + })); + }; + let is_operator = !peekaboo.is_alphanumeric() + && !peekaboo.is_whitespace() + && !peekaboo.is_control(); + + if is_operator { + self.parse_identifier( + token_start_offset, + format!("->{peekaboo}"), + |c| !c.is_alphanumeric() && !c.is_whitespace() && !c.is_control(), + Token::OperatorName, + ) + } else { + self.stash_char(pbloc, peekaboo); + + Ok(Some(LocatedToken { + token: Token::Arrow, + span: token_start_offset..end, + })) + } + } Some((_, c)) if !c.is_alphanumeric() && !c.is_whitespace() && !c.is_control() => self .parse_identifier( token_start_offset, @@ -665,3 +688,22 @@ fn can_separate_pieces() { assert_eq!(Some(Token::ValueName("b".into())), next_token()); assert_eq!(None, next_token()); } + +#[test] +fn arrow_requires_nonop() { + let mut lexer = Lexer::from("->"); + let mut next_token = move || lexer.next().map(|x| x.expect("Can read valid token").token); + assert_eq!(Some(Token::Arrow), next_token()); + + let mut lexer = Lexer::from("->*"); + let mut next_token = move || lexer.next().map(|x| x.expect("Can read valid token").token); + assert_eq!(Some(Token::OperatorName("->*".into())), next_token()); + + let mut lexer = Lexer::from("->*x"); + let mut next_token = move || lexer.next().map(|x| x.expect("Can read valid token").token); + assert_eq!(Some(Token::OperatorName("->*".into())), next_token()); + + let mut lexer = Lexer::from("->x"); + let mut next_token = move || lexer.next().map(|x| x.expect("Can read valid token").token); + assert_eq!(Some(Token::Arrow), next_token()); +}