🤷 The initial version of the compiler, both static and JIT.

This implements a full compiler, with both static compilation and JIT
support, for the world's simplest and silliest programming language. You
can do math, and print variables. That's it. On the bright side, it
implements every part of the compiler, from the lexer and parser;
through analysis and simplification; and into a reasonable code
generator. This should be a good jumping off point for adding more
advanced features.

Tests, including proptests, are included to help avoid regressions.

src/syntax/token_stream.rs

@@ -0,0 +1,117 @@
+use crate::syntax::tokens::Token;
+use logos::{Logos, SpannedIter};
+use std::fmt;
+use thiserror::Error;
+
+pub struct TokenStream<'s> {
+    file_idx: usize,
+    lexer: SpannedIter<'s, Token>,
+}
+
+impl<'s> TokenStream<'s> {
+    pub fn new(file_idx: usize, s: &'s str) -> TokenStream<'s> {
+        TokenStream {
+            file_idx,
+            lexer: Token::lexer(s).spanned(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Location {
+    InFile(usize, usize),
+    Manufactured,
+}
+
+impl fmt::Display for Location {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Location::InFile(s, off) => write!(f, "{}:{}", s, off),
+            Location::Manufactured => write!(f, "<manufactured>"),
+        }
+    }
+}
+
+impl Location {
+    fn new(file_idx: usize, offset: usize) -> Location {
+        Location::InFile(file_idx, offset)
+    }
+}
+
+impl Default for Location {
+    fn default() -> Self {
+        Location::Manufactured
+    }
+}
+
+type LocatedToken = Result<(Location, Token, Location), LexerError>;
+
+impl<'s> Iterator for TokenStream<'s> {
+    type Item = LocatedToken;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.lexer.next() {
+            None => None,
+            Some((Token::Error, span)) => Some(Err(LexerError::new(self.file_idx, span.start))),
+            Some((token, span)) => {
+                let start = Location::new(self.file_idx, span.start);
+                let end = Location::new(self.file_idx, span.end);
+                Some(Ok((start, token, end)))
+            }
+        }
+    }
+}
+
+#[test]
+fn stream_works() {
+    let fidx = 42;
+    let mut lex0 = TokenStream::new(42, "y = x + 1//foo");
+    assert_eq!(
+        lex0.next(),
+        Some(Ok((
+            Location::new(fidx, 0),
+            Token::var("y"),
+            Location::new(fidx, 1)
+        )))
+    );
+    assert_eq!(
+        lex0.next(),
+        Some(Ok((
+            Location::new(fidx, 2),
+            Token::Equals,
+            Location::new(fidx, 3)
+        )))
+    );
+    assert_eq!(
+        lex0.next(),
+        Some(Ok((
+            Location::new(fidx, 4),
+            Token::var("x"),
+            Location::new(fidx, 5)
+        )))
+    );
+    assert_eq!(
+        lex0.next(),
+        Some(Ok((
+            Location::new(fidx, 6),
+            Token::Operator('+'),
+            Location::new(fidx, 7)
+        )))
+    );
+    assert_eq!(
+        lex0.next(),
+        Some(Ok((
+            Location::new(fidx, 8),
+            Token::Number((None, 1)),
+            Location::new(fidx, 9)
+        )))
+    );
+    assert_eq!(lex0.next(), None);
+}
+
+#[test]
+fn errors_work() {
+    let fidx = 2;
+    let mut lex0 = TokenStream::new(2, "\u{2639}");
+    assert_eq!(lex0.next(), Some(Err(LexerError::new(fidx, 0))));
+}