Initial commit; a basic Logos lexer and some tests.

src/syntax/tokens.rs

@@ -0,0 +1,72 @@
+use logos::{Lexer, Logos};
+use std::num::ParseIntError;
+
+#[derive(Logos,Debug,PartialEq)]
+enum Token<'src> {
+    #[regex(r"[ \t\n\f]+", logos::skip)]
+    #[regex(r"//.*", logos::skip)]
+
+    #[token("=")]
+    Equals,
+
+    #[regex(r"[+\-*/]", |v| v.slice().chars().nth(0))]
+    Operator(char),
+
+    #[regex(r"0b[01]+", |v| parse_number(Some(2), v))]
+    #[regex(r"0o[0-7]+", |v| parse_number(Some(8), v))]
+    #[regex(r"0d[0-9]+", |v| parse_number(Some(10), v))]
+    #[regex(r"0x[0-9a-fA-F]+", |v| parse_number(Some(16), v))]
+    #[regex(r"[0-9]+", |v| parse_number(None, v))]
+    Number((Option<u8>, i128)),
+
+    #[regex(r"[a-z][a-zA-Z0-9_]*")]
+    Variable(&'src str),
+
+    #[error]
+    Error,
+}
+
+fn parse_number<'a,'src>(base: Option<u8>, value: &'a Lexer<'src, Token<'src>>) -> Result<(Option<u8>, i128), ParseIntError> {
+    let (radix, strval) = match base {
+       None => (10, value.slice()),
+       Some(radix) => (radix, &value.slice()[2..]),
+    };
+
+    println!("HERE! (radix {}, slice |{}|", radix, strval);
+    let intval = i128::from_str_radix(strval, radix as u32)?;
+    Ok((base, intval))
+}
+
+#[test]
+fn lex_numbers() {
+    let mut lex0 = Token::lexer("12 0b1100 0o14 0d12 0xc // 9");
+    assert_eq!(lex0.next(), Some(Token::Number((None, 12))));
+    assert_eq!(lex0.next(), Some(Token::Number((Some(2), 12))));
+    assert_eq!(lex0.next(), Some(Token::Number((Some(8), 12))));
+    assert_eq!(lex0.next(), Some(Token::Number((Some(10), 12))));
+    assert_eq!(lex0.next(), Some(Token::Number((Some(16), 12))));
+    assert_eq!(lex0.next(), None);
+}
+
+#[test]
+fn lex_symbols() {
+    let mut lex0 = Token::lexer("x + \t y * \n z // rest");
+    assert_eq!(lex0.next(), Some(Token::Variable("x")));
+    assert_eq!(lex0.next(), Some(Token::Operator('+')));
+    assert_eq!(lex0.next(), Some(Token::Variable("y")));
+    assert_eq!(lex0.next(), Some(Token::Operator('*')));
+    assert_eq!(lex0.next(), Some(Token::Variable("z")));
+    assert_eq!(lex0.next(), None);
+}
+
+#[test]
+fn lexer_spans() {
+    let mut lex0 = Token::lexer("y = x + 1//foo").spanned();
+    assert_eq!(lex0.next(), Some((Token::Variable("y"), 0..1)));
+    assert_eq!(lex0.next(), Some((Token::Equals, 2..3)));
+    assert_eq!(lex0.next(), Some((Token::Variable("x"), 4..5)));
+    assert_eq!(lex0.next(), Some((Token::Operator('+'), 6..7)));
+    assert_eq!(lex0.next(), Some((Token::Number((None, 1)), 8..9)));
+    assert_eq!(lex0.next(), None);
+}
+