Some basic parsing, with interned strings.

src/syntax/tokens.rs

@@ -1,14 +1,18 @@
+use crate::util::istring::InternedString;
 use logos::{Lexer, Logos};
 use std::num::ParseIntError;
 
-#[derive(Logos,Debug,PartialEq)]
-enum Token<'src> {
+#[derive(Logos,Clone,Debug,PartialEq)]
+pub enum Token {
     #[regex(r"[ \t\n\f]+", logos::skip)]
     #[regex(r"//.*", logos::skip)]
 
     #[token("=")]
     Equals,
 
+    #[token(";")]
+    Semi,
+
     #[regex(r"[+\-*/]", |v| v.slice().chars().nth(0))]
     Operator(char),
 
@@ -19,14 +23,21 @@ enum Token<'src> {
     #[regex(r"[0-9]+", |v| parse_number(None, v))]
     Number((Option<u8>, i128)),
 
-    #[regex(r"[a-z][a-zA-Z0-9_]*")]
-    Variable(&'src str),
+    #[regex(r"[a-z][a-zA-Z0-9_]*", |v| InternedString::new(v.slice()))]
+    Variable(InternedString),
 
     #[error]
     Error,
 }
 
-fn parse_number<'a,'src>(base: Option<u8>, value: &'a Lexer<'src, Token<'src>>) -> Result<(Option<u8>, i128), ParseIntError> {
+#[cfg(test)]
+impl Token {
+    pub(crate) fn var(s: &str) -> Token {
+        Token::Variable(InternedString::new(s))
+    }
+}
+
+fn parse_number<'a,'src>(base: Option<u8>, value: &'a Lexer<'src, Token>) -> Result<(Option<u8>, i128), ParseIntError> {
     let (radix, strval) = match base {
        None => (10, value.slice()),
        Some(radix) => (radix, &value.slice()[2..]),
@@ -51,20 +62,20 @@ fn lex_numbers() {
 #[test]
 fn lex_symbols() {
     let mut lex0 = Token::lexer("x + \t y * \n z // rest");
-    assert_eq!(lex0.next(), Some(Token::Variable("x")));
+    assert_eq!(lex0.next(), Some(Token::var("x")));
     assert_eq!(lex0.next(), Some(Token::Operator('+')));
-    assert_eq!(lex0.next(), Some(Token::Variable("y")));
+    assert_eq!(lex0.next(), Some(Token::var("y")));
     assert_eq!(lex0.next(), Some(Token::Operator('*')));
-    assert_eq!(lex0.next(), Some(Token::Variable("z")));
+    assert_eq!(lex0.next(), Some(Token::var("z")));
     assert_eq!(lex0.next(), None);
 }
 
 #[test]
 fn lexer_spans() {
     let mut lex0 = Token::lexer("y = x + 1//foo").spanned();
-    assert_eq!(lex0.next(), Some((Token::Variable("y"), 0..1)));
+    assert_eq!(lex0.next(), Some((Token::var("y"), 0..1)));
     assert_eq!(lex0.next(), Some((Token::Equals, 2..3)));
-    assert_eq!(lex0.next(), Some((Token::Variable("x"), 4..5)));
+    assert_eq!(lex0.next(), Some((Token::var("x"), 4..5)));
     assert_eq!(lex0.next(), Some((Token::Operator('+'), 6..7)));
     assert_eq!(lex0.next(), Some((Token::Number((None, 1)), 8..9)));
     assert_eq!(lex0.next(), None);