Initial commit; a basic Logos lexer and some tests.

.gitignore

@@ -0,0 +1,2 @@
+/target
+Cargo.lock

Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "ngr"
+version = "0.1.0"
+authors = ["awick"]
+edition = "2018"
+
+[lib]
+name = "ngr"
+path = "src/lib.rs"
+
+[[bin]]
+name = "ngrc"
+path = "src/bin.rs"
+
+[dependencies]
+logos = "0.11.4"
+lalrpop-util = "0.19.0"
+
+[build-dependencies]
+lalrpop = "0.19.0"

build.rs

@@ -0,0 +1,5 @@
+extern crate lalrpop;
+
+fn main() {
+    lalrpop::process_root().unwrap();
+}

src/bin.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello, world!");
+}

src/lib.rs

@@ -0,0 +1 @@
+pub mod syntax;

src/syntax.rs

@@ -0,0 +1,5 @@
+//use lalrpop_util::lalrpop_mod;
+
+pub mod tokens;
+//lalrpop_mod!(pub parser);
+pub mod ast;

src/syntax/ast.rs

@@ -0,0 +1,14 @@
+pub enum Stmt {
+    Binding(String, Expr),
+    Expr(Expr),
+}
+
+pub enum Expr {
+    Value(Value),
+    Reference(String),
+    Primitive(String, Vec<Expr>),
+}
+
+pub enum Value {
+    Number(Option<u8>, i128)
+}

src/syntax/tokens.rs

@@ -0,0 +1,72 @@
+use logos::{Lexer, Logos};
+use std::num::ParseIntError;
+
+#[derive(Logos,Debug,PartialEq)]
+enum Token<'src> {
+    #[regex(r"[ \t\n\f]+", logos::skip)]
+    #[regex(r"//.*", logos::skip)]
+
+    #[token("=")]
+    Equals,
+
+    #[regex(r"[+\-*/]", |v| v.slice().chars().nth(0))]
+    Operator(char),
+
+    #[regex(r"0b[01]+", |v| parse_number(Some(2), v))]
+    #[regex(r"0o[0-7]+", |v| parse_number(Some(8), v))]
+    #[regex(r"0d[0-9]+", |v| parse_number(Some(10), v))]
+    #[regex(r"0x[0-9a-fA-F]+", |v| parse_number(Some(16), v))]
+    #[regex(r"[0-9]+", |v| parse_number(None, v))]
+    Number((Option<u8>, i128)),
+
+    #[regex(r"[a-z][a-zA-Z0-9_]*")]
+    Variable(&'src str),
+
+    #[error]
+    Error,
+}
+
+fn parse_number<'a,'src>(base: Option<u8>, value: &'a Lexer<'src, Token<'src>>) -> Result<(Option<u8>, i128), ParseIntError> {
+    let (radix, strval) = match base {
+       None => (10, value.slice()),
+       Some(radix) => (radix, &value.slice()[2..]),
+    };
+
+    println!("HERE! (radix {}, slice |{}|", radix, strval);
+    let intval = i128::from_str_radix(strval, radix as u32)?;
+    Ok((base, intval))
+}
+
+#[test]
+fn lex_numbers() {
+    let mut lex0 = Token::lexer("12 0b1100 0o14 0d12 0xc // 9");
+    assert_eq!(lex0.next(), Some(Token::Number((None, 12))));
+    assert_eq!(lex0.next(), Some(Token::Number((Some(2), 12))));
+    assert_eq!(lex0.next(), Some(Token::Number((Some(8), 12))));
+    assert_eq!(lex0.next(), Some(Token::Number((Some(10), 12))));
+    assert_eq!(lex0.next(), Some(Token::Number((Some(16), 12))));
+    assert_eq!(lex0.next(), None);
+}
+
+#[test]
+fn lex_symbols() {
+    let mut lex0 = Token::lexer("x + \t y * \n z // rest");
+    assert_eq!(lex0.next(), Some(Token::Variable("x")));
+    assert_eq!(lex0.next(), Some(Token::Operator('+')));
+    assert_eq!(lex0.next(), Some(Token::Variable("y")));
+    assert_eq!(lex0.next(), Some(Token::Operator('*')));
+    assert_eq!(lex0.next(), Some(Token::Variable("z")));
+    assert_eq!(lex0.next(), None);
+}
+
+#[test]
+fn lexer_spans() {
+    let mut lex0 = Token::lexer("y = x + 1//foo").spanned();
+    assert_eq!(lex0.next(), Some((Token::Variable("y"), 0..1)));
+    assert_eq!(lex0.next(), Some((Token::Equals, 2..3)));
+    assert_eq!(lex0.next(), Some((Token::Variable("x"), 4..5)));
+    assert_eq!(lex0.next(), Some((Token::Operator('+'), 6..7)));
+    assert_eq!(lex0.next(), Some((Token::Number((None, 1)), 8..9)));
+    assert_eq!(lex0.next(), None);
+}
+