✍️ Switch to a handwritten lexer and parser. #1

Open
acw wants to merge 33 commits from handwritten-lexer into master
3 changed files with 360 additions and 15 deletions
Showing only changes of commit 7bd242a641 - Show all commits

View File

@@ -139,7 +139,33 @@ pub struct MatchExpr {
}
#[derive(Debug)]
pub struct MatchCase {}
pub struct MatchCase {
pub pattern: Pattern,
pub consequent: Expression,
}
#[derive(Debug)]
pub enum Pattern {
Constant(ConstantValue),
Variable(Name),
EnumerationValue(EnumerationPattern),
Structure(StructurePattern),
}
#[derive(Debug)]
pub struct EnumerationPattern {
pub location: Location,
pub type_name: Name,
pub variant_name: Name,
pub argument: Option<Box<Pattern>>,
}
#[derive(Debug)]
pub struct StructurePattern {
pub location: Location,
pub type_name: Name,
pub fields: Vec<(Name, Option<Pattern>)>,
}
#[derive(Debug)]
pub enum CallKind {

View File

@@ -682,7 +682,256 @@ impl<'lexer> Parser<'lexer> {
}
fn parse_match_case(&mut self) -> Result<Option<MatchCase>, ParserError> {
unimplemented!()
// skip over anything we can just skip
loop {
let peeked = self
.next()?
.ok_or_else(|| self.bad_eof("looking for match case"))?;
if matches!(peeked.token, Token::Comma) {
continue;
}
let stop = matches!(peeked.token, Token::CloseBrace);
self.save(peeked);
if stop {
return Ok(None);
}
break;
}
let pattern = self.parse_pattern()?;
let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for an open brace after 'match'"))?;
if !matches!(next.token, Token::Arrow) {
return Err(ParserError::UnexpectedToken {
file: self.file.clone(),
span: next.span,
token: next.token,
expected: "an arrow after a pattern, as part of a match case",
});
}
let consequent = self.parse_expression()?;
Ok(Some(MatchCase {
pattern,
consequent,
}))
}
pub fn parse_pattern(&mut self) -> Result<Pattern, ParserError> {
if let Ok(constant) = self.parse_constant() {
return Ok(Pattern::Constant(constant));
}
let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for a pattern to match"))?;
match next.token {
Token::ValueName(x) => {
let name = Name::new(self.to_location(next.span), x);
Ok(Pattern::Variable(name))
}
Token::TypeName(x) => {
let type_name = Name::new(self.to_location(next.span.clone()), x);
let start = self.to_location(next.span);
let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for a pattern to match"))?;
match next.token {
Token::OpenBrace => {
let mut fields = vec![];
while let Some(field_pattern) = self.parse_field_pattern()? {
fields.push(field_pattern)
}
let final_brace = self.next()?.ok_or_else(|| {
self.bad_eof("looking for closing brace in structure pattern.")
})?;
if !matches!(final_brace.token, Token::CloseBrace) {
return Err(ParserError::UnexpectedToken {
file: self.file.clone(),
span: final_brace.span,
token: final_brace.token,
expected: "closing brace in structure pattern",
});
}
let final_brace_location = self.to_location(final_brace.span);
let structure_pattern = StructurePattern {
location: start.extend_to(&final_brace_location),
type_name,
fields,
};
Ok(Pattern::Structure(structure_pattern))
}
Token::Colon => {
let second_colon = self.next()?.ok_or_else(|| {
self.bad_eof("looking for second colon in an enumeration pattern")
})?;
if !matches!(second_colon.token, Token::Colon) {
return Err(ParserError::UnexpectedToken {
file: self.file.clone(),
span: second_colon.span,
token: second_colon.token,
expected: "second colon in an enumeration pattern",
});
}
let vname = self.next()?.ok_or_else(|| {
self.bad_eof("looking for enumeration value name in pattern")
})?;
let variant_name = match vname.token {
Token::TypeName(s) => {
let loc = self.to_location(vname.span.clone());
Name::new(loc, s)
}
_ => {
return Err(ParserError::UnexpectedToken {
file: self.file.clone(),
span: vname.span,
token: vname.token,
expected: "enumeration value name in pattern",
});
}
};
let mut final_location = self.to_location(vname.span);
let argument = if let Some(maybe_paren) = self.next()? {
if matches!(maybe_paren.token, Token::OpenParen) {
let sub_pattern = self.parse_pattern()?;
let tok = self.next()?.ok_or_else(|| {
self.bad_eof("looking for close paren after enum value argument")
})?;
if !matches!(tok.token, Token::CloseParen) {
return Err(ParserError::UnexpectedToken {
file: self.file.clone(),
span: tok.span,
token: tok.token,
expected: "close paren after enum value argument",
});
}
final_location = self.to_location(tok.span);
Some(Box::new(sub_pattern))
} else {
None
}
} else {
None
};
let location = start.extend_to(&final_location);
let pattern = EnumerationPattern {
location,
type_name,
variant_name,
argument,
};
Ok(Pattern::EnumerationValue(pattern))
}
_ => Err(ParserError::UnexpectedToken {
file: self.file.clone(),
span: next.span,
token: next.token,
expected: "An '::' or '{' after a type name in a pattern",
}),
}
}
_ => Err(ParserError::UnexpectedToken {
file: self.file.clone(),
span: next.span,
token: next.token,
expected: "The start of a pattern: a variable name or type name",
}),
}
}
fn parse_field_pattern(&mut self) -> Result<Option<(Name, Option<Pattern>)>, ParserError> {
let next = self
.next()?
.ok_or_else(|| self.bad_eof("looking for structure pattern field name"))?;
let name = match next.token {
Token::CloseBrace => {
self.save(next);
return Ok(None);
}
Token::ValueName(s) => Name::new(self.to_location(next.span), s),
_ => {
return Err(ParserError::UnexpectedToken {
file: self.file.clone(),
span: next.span,
token: next.token,
expected: "a field name in a structure pattern",
});
}
};
let next = self.next()?.ok_or_else(|| {
self.bad_eof("looking for colon, comma, or brace after structure field name in pattern")
})?;
let sub_pattern = match next.token {
Token::Comma => None,
Token::CloseBrace => {
self.save(next);
None
}
Token::Colon => {
let subpattern = self.parse_pattern()?;
let next = self
.next()?
.ok_or_else(|| self.bad_eof(
"looking for comma or close brace after structure field"))?;
match next.token {
Token::Comma => {}
Token::CloseBrace => self.save(next),
_ => return Err(ParserError::UnexpectedToken {
file: self.file.clone(),
span: next.span,
token: next.token,
expected: "comma or close brace after structure field"
}),
}
Some(subpattern)
}
_ => {
return Err(ParserError::UnexpectedToken {
file: self.file.clone(),
span: next.span,
token: next.token,
expected: "colon, comma, or brace after structure field name in pattern",
});
}
};
Ok(Some((name, sub_pattern)))
}
fn parse_if_expression(&mut self) -> Result<ConditionalExpr, ParserError> {
@@ -1120,6 +1369,7 @@ impl<'lexer> Parser<'lexer> {
};
let arg = if let Some(maybe_paren) = self.next()? {
if matches!(maybe_paren.token, Token::OpenParen) {
let expr = self.parse_expression()?;
let tok = self.next()?.ok_or_else(|| {
@@ -1135,6 +1385,10 @@ impl<'lexer> Parser<'lexer> {
}
Some(Box::new(expr))
} else {
self.save(maybe_paren);
None
}
} else {
None
};

View File

@@ -925,3 +925,68 @@ fn conditionals() {
Ok(Expression::Conditional(cond)) if
matches!(cond.test.as_ref(), Expression::Call(_, CallKind::Infix, _))));
}
#[test]
#[allow(clippy::get_first)]
fn patterns() {
let parse_pat = |str| {
let lexer = Lexer::from(str);
let mut result = Parser::new("test", lexer);
result.parse_pattern()
};
assert!(matches!(
parse_pat("1"),
Ok(Pattern::Constant(ConstantValue::Integer(_,
IntegerWithBase { value, .. }))) if
value == 1));
assert!(matches!(
parse_pat("x"),
Ok(Pattern::Variable(n)) if n.as_printed() == "x"));
assert!(matches!(
parse_pat("Cons::Pair(pair)"),
Ok(Pattern::EnumerationValue(EnumerationPattern{
type_name, variant_name, argument: Some(subpat), ..
})) if
type_name.as_printed() == "Cons" &&
variant_name.as_printed() == "Pair" &&
matches!(subpat.as_ref(), Pattern::Variable(p) if
p.as_printed() == "pair")));
assert!(matches!(
parse_pat("Structure{ field, other: something }"),
Ok(Pattern::Structure(StructurePattern { type_name, fields, .. })) if
type_name.as_printed() == "Structure" &&
fields.len() == 2 &&
matches!(fields.get(0), Some((n, None)) if n.as_printed() == "field") &&
matches!(fields.get(1), Some((n, Some(Pattern::Variable(s)))) if
n.as_printed() == "other" &&
s.as_printed() == "something")));
assert!(matches!(
parse_pat("Enumeration::Value(Structure { field, })"),
Ok(Pattern::EnumerationValue(EnumerationPattern {
type_name, variant_name, argument: Some(subpat), ..
})) if
type_name.as_printed() == "Enumeration" &&
variant_name.as_printed() == "Value" &&
matches!(subpat.as_ref(), Pattern::Structure(StructurePattern {
type_name, fields, ..
}) if
type_name.as_printed() == "Structure" &&
fields.len() == 1 &&
matches!(fields.first(), Some((f, None)) if
f.as_printed() == "field"))));
assert!(matches!(
parse_pat("Structure { field: Enumeration::Value, }"),
Ok(Pattern::Structure(StructurePattern {
type_name, fields, ..
})) if
type_name.as_printed() == "Structure" &&
fields.len() == 1 &&
matches!(fields.first(), Some((f, Some(subpat))) if
f.as_printed() == "field" &&
matches!(subpat, Pattern::EnumerationValue(EnumerationPattern {
type_name, variant_name, argument: None, ..
}) if
type_name.as_printed() == "Enumeration" &&
variant_name.as_printed() == "Value"))));
}