acw/ngr
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

checkpoint in reconstruction

Browse Source
This commit is contained in:
Adam Wick
2023-12-26 21:08:01 -08:00
parent 2c2268925a
commit e5db6640f2
21 changed files with 759 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
src/syntax/tokens.rs
View File

@@ -1,7 +1,6 @@
use internment::ArcIntern;
use logos::{Lexer, Logos};
use std::fmt;
use std::num::ParseIntError;
use thiserror::Error;
/// A single token of the input stream; used to help the parsing go down
@@ -26,6 +25,12 @@ use thiserror::Error;
/// trait, you should get back the exact same token.
#[derive(Logos, Clone, Debug, PartialEq, Eq)]
pub enum Token {
// we're actually just going to skip whitespace, though
#[regex(r"[ \t\r\n\f]+", logos::skip)]
// this is an extremely simple version of comments, just line
// comments. More complicated /* */ comments can be harder to
// implement, and didn't seem worth it at the time.
#[regex(r"//.*", logos::skip)]
// Our first set of tokens are simple characters that we're
// going to use to structure NGR programs.
#[token("=")]
@@ -87,18 +92,6 @@ pub enum Token {
// letter, too.
#[regex(r"[a-z][a-zA-Z0-9_]*", |v| ArcIntern::new(v.slice().to_string()))]
Variable(ArcIntern<String>),
// the next token will be an error token
#[error]
// we're actually just going to skip whitespace, though
#[regex(r"[ \t\r\n\f]+", logos::skip)]
// this is an extremely simple version of comments, just line
// comments. More complicated /* */ comments can be harder to
// implement, and didn't seem worth it at the time.
#[regex(r"//.*", logos::skip)]
/// This token represents that some core error happened in lexing;
/// possibly that something didn't match anything at all.
Error,
}
impl fmt::Display for Token {
@@ -137,7 +130,6 @@ impl fmt::Display for Token {
)
}
Token::Variable(s) => write!(f, "'{}'", s),
Token::Error => write!(f, "<error>"),
}
}
}
@@ -171,11 +163,13 @@ pub enum ConstantType {
I16 = 21,
I32 = 22,
I64 = 23,
Void = 255,
}
impl From<ConstantType> for cranelift_codegen::ir::Type {
fn from(value: ConstantType) -> Self {
match value {
ConstantType::Void => cranelift_codegen::ir::types::I64,
ConstantType::I8 | ConstantType::U8 => cranelift_codegen::ir::types::I8,
ConstantType::I16 | ConstantType::U16 => cranelift_codegen::ir::types::I16,
ConstantType::I32 | ConstantType::U32 => cranelift_codegen::ir::types::I32,
@@ -196,6 +190,7 @@ impl ConstantType {
/// Return the set of types that can be safely casted into this type.
pub fn safe_casts_to(self) -> Vec<ConstantType> {
match self {
ConstantType::Void => vec![ConstantType::Void],
ConstantType::I8 => vec![ConstantType::I8],
ConstantType::I16 => vec![ConstantType::I16, ConstantType::I8, ConstantType::U8],
ConstantType::I32 => vec![
@@ -243,6 +238,7 @@ impl ConstantType {
/// Return the name of the given type, as a string
pub fn name(&self) -> String {
match self {
ConstantType::Void => "void".to_string(),
ConstantType::I8 => "i8".to_string(),
ConstantType::I16 => "i16".to_string(),
ConstantType::I32 => "i32".to_string(),
@@ -286,7 +282,7 @@ impl TryFrom<i64> for ConstantType {
fn parse_number(
base: Option<u8>,
value: &Lexer<Token>,
) -> Result<(Option<u8>, Option<ConstantType>, u64), ParseIntError> {
) -> Result<(Option<u8>, Option<ConstantType>, u64), ()> {
let (radix, strval) = match base {
None => (10, value.slice()),
Some(radix) => (radix, &value.slice()[2..]),
@@ -312,13 +308,14 @@ fn parse_number(
(None, strval)
};
let intval = u64::from_str_radix(strval, radix as u32)?;
let intval = u64::from_str_radix(strval, radix as u32).map_err(|_| ())?;
Ok((base, declared_type, intval))
}
fn display_optional_type(otype: &Option<ConstantType>) -> &'static str {
match otype {
None => "",
Some(ConstantType::Void) => "void",
Some(ConstantType::I8) => "i8",
Some(ConstantType::I16) => "i16",
Some(ConstantType::I32) => "i32",
@@ -333,18 +330,18 @@ fn display_optional_type(otype: &Option<ConstantType>) -> &'static str {
#[test]
fn lex_numbers() {
let mut lex0 = Token::lexer("12 0b1100 0o14 0d12 0xc 12u8 0xci64// 9");
assert_eq!(lex0.next(), Some(Token::Number((None, None, 12))));
assert_eq!(lex0.next(), Some(Token::Number((Some(2), None, 12))));
assert_eq!(lex0.next(), Some(Token::Number((Some(8), None, 12))));
assert_eq!(lex0.next(), Some(Token::Number((Some(10), None, 12))));
assert_eq!(lex0.next(), Some(Token::Number((Some(16), None, 12))));
assert_eq!(lex0.next(), Some(Ok(Token::Number((None, None, 12)))));
assert_eq!(lex0.next(), Some(Ok(Token::Number((Some(2), None, 12)))));
assert_eq!(lex0.next(), Some(Ok(Token::Number((Some(8), None, 12)))));
assert_eq!(lex0.next(), Some(Ok(Token::Number((Some(10), None, 12)))));
assert_eq!(lex0.next(), Some(Ok(Token::Number((Some(16), None, 12)))));
assert_eq!(
lex0.next(),
Some(Token::Number((None, Some(ConstantType::U8), 12)))
Some(Ok(Token::Number((None, Some(ConstantType::U8), 12))))
);
assert_eq!(
lex0.next(),
Some(Token::Number((Some(16), Some(ConstantType::I64), 12)))
Some(Ok(Token::Number((Some(16), Some(ConstantType::I64), 12))))
);
assert_eq!(lex0.next(), None);
}
@@ -352,46 +349,52 @@ fn lex_numbers() {
#[test]
fn lex_symbols() {
let mut lex0 = Token::lexer("x + \t y * \n z // rest");
assert_eq!(lex0.next(), Some(Token::var("x")));
assert_eq!(lex0.next(), Some(Token::Operator('+')));
assert_eq!(lex0.next(), Some(Token::var("y")));
assert_eq!(lex0.next(), Some(Token::Operator('*')));
assert_eq!(lex0.next(), Some(Token::var("z")));
assert_eq!(lex0.next(), Some(Ok(Token::var("x"))));
assert_eq!(lex0.next(), Some(Ok(Token::Operator('+'))));
assert_eq!(lex0.next(), Some(Ok(Token::var("y"))));
assert_eq!(lex0.next(), Some(Ok(Token::Operator('*'))));
assert_eq!(lex0.next(), Some(Ok(Token::var("z"))));
assert_eq!(lex0.next(), None);
}
#[test]
fn lexer_spans() {
let mut lex0 = Token::lexer("y = x + 1//foo").spanned();
assert_eq!(lex0.next(), Some((Token::var("y"), 0..1)));
assert_eq!(lex0.next(), Some((Token::Equals, 2..3)));
assert_eq!(lex0.next(), Some((Token::var("x"), 4..5)));
assert_eq!(lex0.next(), Some((Token::Operator('+'), 6..7)));
assert_eq!(lex0.next(), Some((Token::Number((None, None, 1)), 8..9)));
assert_eq!(lex0.next(), Some((Ok(Token::var("y")), 0..1)));
assert_eq!(lex0.next(), Some((Ok(Token::Equals), 2..3)));
assert_eq!(lex0.next(), Some((Ok(Token::var("x")), 4..5)));
assert_eq!(lex0.next(), Some((Ok(Token::Operator('+')), 6..7)));
assert_eq!(
lex0.next(),
Some((Ok(Token::Number((None, None, 1))), 8..9))
);
assert_eq!(lex0.next(), None);
}
#[test]
fn further_spans() {
let mut lex0 = Token::lexer("x = 2i64 + 2i64;\ny = -x;\nprint y;").spanned();
assert_eq!(lex0.next(), Some((Token::var("x"), 0..1)));
assert_eq!(lex0.next(), Some((Token::Equals, 2..3)));
assert_eq!(lex0.next(), Some((Ok(Token::var("x")), 0..1)));
assert_eq!(lex0.next(), Some((Ok(Token::Equals), 2..3)));
assert_eq!(
lex0.next(),
Some((Token::Number((None, Some(ConstantType::I64), 2)), 4..8))
Some((Ok(Token::Number((None, Some(ConstantType::I64), 2))), 4..8))
);
assert_eq!(lex0.next(), Some((Token::Operator('+'), 9..10)));
assert_eq!(lex0.next(), Some((Ok(Token::Operator('+')), 9..10)));
assert_eq!(
lex0.next(),
Some((Token::Number((None, Some(ConstantType::I64), 2)), 11..15))
Some((
Ok(Token::Number((None, Some(ConstantType::I64), 2))),
11..15
))
);
assert_eq!(lex0.next(), Some((Token::Semi, 15..16)));
assert_eq!(lex0.next(), Some((Token::var("y"), 17..18)));
assert_eq!(lex0.next(), Some((Token::Equals, 19..20)));
assert_eq!(lex0.next(), Some((Token::Operator('-'), 21..22)));
assert_eq!(lex0.next(), Some((Token::var("x"), 22..23)));
assert_eq!(lex0.next(), Some((Token::Semi, 23..24)));
assert_eq!(lex0.next(), Some((Token::Print, 25..30)));
assert_eq!(lex0.next(), Some((Token::var("y"), 31..32)));
assert_eq!(lex0.next(), Some((Token::Semi, 32..33)));
assert_eq!(lex0.next(), Some((Ok(Token::Semi), 15..16)));
assert_eq!(lex0.next(), Some((Ok(Token::var("y")), 17..18)));
assert_eq!(lex0.next(), Some((Ok(Token::Equals), 19..20)));
assert_eq!(lex0.next(), Some((Ok(Token::Operator('-')), 21..22)));
assert_eq!(lex0.next(), Some((Ok(Token::var("x")), 22..23)));
assert_eq!(lex0.next(), Some((Ok(Token::Semi), 23..24)));
assert_eq!(lex0.next(), Some((Ok(Token::Print), 25..30)));
assert_eq!(lex0.next(), Some((Ok(Token::var("y")), 31..32)));
assert_eq!(lex0.next(), Some((Ok(Token::Semi), 32..33)));
}