🧪 Add evaluation tests to ensure that passes retain NGR semantics. #2

Merged
acw merged 6 commits from acw/eval-tests into develop 2023-04-16 16:07:45 -07:00
20 changed files with 610 additions and 22 deletions

View File

@@ -26,6 +26,7 @@ pretty = { version = "^0.11.2", features = ["termcolor"] }
proptest = "^1.0.0"
rustyline = "^11.0.0"
target-lexicon = "^0.12.5"
tempfile = "^3.5.0"
thiserror = "^1.0.30"
[build-dependencies]

View File

@@ -1,12 +1,13 @@
#include <stdint.h>
#include <stdio.h>
#include <inttypes.h>
void print(char *variable_name, uint64_t value) {
printf("%s = %llu\n", variable_name, value);
void print(char *_ignore, char *variable_name, int64_t value) {
printf("%s = %" PRId64 "i64\n", variable_name, value);
}
void caller() {
print("x", 4);
print(NULL, "x", 4);
}
extern void gogogo();

View File

@@ -1,4 +1,5 @@
mod error;
mod eval;
mod into_crane;
mod runtime;
@@ -10,7 +11,7 @@ use cranelift_codegen::settings::Configurable;
use cranelift_codegen::{isa, settings};
use cranelift_jit::{JITBuilder, JITModule};
use cranelift_module::{default_libcall_names, DataContext, DataId, FuncId, Linkage, Module};
use cranelift_object::{object, ObjectBuilder, ObjectModule};
use cranelift_object::{ObjectBuilder, ObjectModule};
use target_lexicon::Triple;
const EMPTY_DATUM: [u8; 8] = [0; 8];
@@ -21,10 +22,11 @@ pub struct Backend<M: Module> {
runtime_functions: RuntimeFunctions,
defined_strings: HashMap<String, DataId>,
defined_symbols: HashMap<String, DataId>,
output_buffer: Option<String>,
}
impl Backend<JITModule> {
pub fn jit() -> Result<Self, BackendError> {
pub fn jit(output_buffer: Option<String>) -> Result<Self, BackendError> {
let platform = Triple::host();
let isa_builder = isa::lookup(platform.clone())?;
let mut settings_builder = settings::builder();
@@ -44,6 +46,7 @@ impl Backend<JITModule> {
runtime_functions,
defined_strings: HashMap::new(),
defined_symbols: HashMap::new(),
output_buffer,
})
}
@@ -69,23 +72,26 @@ impl Backend<ObjectModule> {
runtime_functions,
defined_strings: HashMap::new(),
defined_symbols: HashMap::new(),
output_buffer: None,
})
}
pub fn bytes(self) -> Result<Vec<u8>, object::write::Error> {
self.module.finish().emit()
pub fn bytes(self) -> Result<Vec<u8>, BackendError> {
self.module.finish().emit().map_err(Into::into)
}
}
impl<M: Module> Backend<M> {
pub fn define_string(&mut self, s: &str) -> Result<DataId, BackendError> {
let name = format!("<string_constant>{}", s);
let s0 = format!("{}\0", s);
let global_id = self
.module
.declare_data(&name, Linkage::Local, false, false)?;
let mut data_context = DataContext::new();
data_context.set_align(8);
data_context.define(s.to_owned().into_boxed_str().into_boxed_bytes());
data_context.define(s0.into_boxed_str().into_boxed_bytes());
self.module.define_data(global_id, &data_context)?;
self.defined_strings.insert(s.to_owned(), global_id);
Ok(global_id)
@@ -101,4 +107,20 @@ impl<M: Module> Backend<M> {
self.defined_symbols.insert(name, id);
Ok(id)
}
pub fn output_buffer_ptr(&mut self) -> *mut String {
if let Some(str) = self.output_buffer.as_mut() {
str as *mut String
} else {
std::ptr::null_mut()
}
}
pub fn output(self) -> String {
if let Some(s) = self.output_buffer {
s
} else {
String::new()
}
}
}

View File

@@ -18,6 +18,8 @@ pub enum BackendError {
SetError(#[from] SetError),
#[error(transparent)]
LookupError(#[from] LookupError),
#[error(transparent)]
Write(#[from] cranelift_object::object::write::Error),
}
impl From<BackendError> for Diagnostic<usize> {
@@ -41,6 +43,41 @@ impl From<BackendError> for Diagnostic<usize> {
BackendError::LookupError(me) => {
Diagnostic::error().with_message(format!("Internal error: {}", me))
}
BackendError::Write(me) => {
Diagnostic::error().with_message(format!("Cranelift object write error: {}", me))
}
}
}
}
impl PartialEq for BackendError {
fn eq(&self, other: &Self) -> bool {
match self {
BackendError::BuiltinError(a) => match other {
BackendError::BuiltinError(b) => a == b,
_ => false,
},
BackendError::CodegenError(_) => matches!(other, BackendError::CodegenError(_)),
BackendError::Cranelift(_) => matches!(other, BackendError::Cranelift(_)),
BackendError::LookupError(a) => match other {
BackendError::LookupError(b) => a == b,
_ => false,
},
BackendError::SetError(a) => match other {
BackendError::SetError(b) => a == b,
_ => false,
},
BackendError::VariableLookupFailure => other == &BackendError::VariableLookupFailure,
BackendError::Write(a) => match other {
BackendError::Write(b) => a == b,
_ => false,
},
}
}
}

106
src/backend/eval.rs Normal file
View File

@@ -0,0 +1,106 @@
use std::path::Path;
use crate::backend::Backend;
use crate::eval::EvalError;
use crate::ir::Program;
use cranelift_jit::JITModule;
use cranelift_object::ObjectModule;
use target_lexicon::Triple;
impl Backend<JITModule> {
pub fn eval(program: Program) -> Result<String, EvalError> {
let mut jitter = Backend::jit(Some(String::new()))?;
let function_id = jitter.compile_function("test", program)?;
jitter.module.finalize_definitions()?;
let compiled_bytes = jitter.bytes(function_id);
let compiled_function = unsafe { std::mem::transmute::<_, fn() -> ()>(compiled_bytes) };
compiled_function();
Ok(jitter.output())
}
}
impl Backend<ObjectModule> {
pub fn eval(program: Program) -> Result<String, EvalError> {
//use pretty::{Arena, Pretty};
//let allocator = Arena::<()>::new();
//program.pretty(&allocator).render(80, &mut std::io::stdout())?;
let mut backend = Self::object_file(Triple::host())?;
let my_directory = tempfile::tempdir()?;
let object_path = my_directory.path().join("object.o");
let executable_path = my_directory.path().join("test_executable");
backend.compile_function("gogogo", program)?;
let bytes = backend.bytes()?;
std::fs::write(&object_path, bytes)?;
Self::link(&object_path, &executable_path)?;
let output = std::process::Command::new(executable_path).output()?;
if output.stderr.is_empty() {
if output.status.success() {
Ok(std::string::String::from_utf8_lossy(&output.stdout).to_string())
} else {
Err(EvalError::IO(format!(
"Exitted with error code {}",
output.status
)))
}
} else {
Err(EvalError::IO(
std::string::String::from_utf8_lossy(&output.stderr).to_string(),
))
}
}
fn link(object_file: &Path, executable_path: &Path) -> Result<(), EvalError> {
use std::path::PathBuf;
let output = std::process::Command::new("clang")
.arg(
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("runtime")
.join("rts.c"),
)
.arg(object_file)
.arg("-o")
.arg(executable_path)
.output()?;
if !output.stderr.is_empty() {
return Err(EvalError::IO(
std::string::String::from_utf8_lossy(&output.stderr).to_string(),
));
}
Ok(())
}
}
proptest::proptest! {
#[test]
fn file_backend_works(program: Program) {
use crate::eval::PrimOpError;
let basic_result = program.eval();
#[cfg(target_family="windows")]
let basic_result = basic_result.map(|x| x.replace('\n', "\r\n"));
if !matches!(basic_result, Err(EvalError::PrimOp(PrimOpError::MathFailure(_)))) {
let compiled_result = Backend::<ObjectModule>::eval(program);
assert_eq!(basic_result, compiled_result);
}
}
#[test]
fn jit_backend_works(program: Program) {
use crate::eval::PrimOpError;
let basic_result = program.eval();
if !matches!(basic_result, Err(EvalError::PrimOp(PrimOpError::MathFailure(_)))) {
let compiled_result = Backend::<JITModule>::eval(program);
assert_eq!(basic_result, compiled_result);
}
}
}

View File

@@ -60,6 +60,8 @@ impl<M: Module> Backend<M> {
for stmt in program.statements.drain(..) {
match stmt {
Statement::Print(ann, var) => {
let buffer_ptr = self.output_buffer_ptr();
let buffer_ptr = builder.ins().iconst(types::I64, buffer_ptr as i64);
let local_name_ref = string_table.get(&var).unwrap();
let name_ptr = builder.ins().symbol_value(types::I64, *local_name_ref);
let val = ValueOrRef::Ref(ann, var).into_cranelift(
@@ -67,7 +69,9 @@ impl<M: Module> Backend<M> {
&variable_table,
&pre_defined_symbols,
)?;
builder.ins().call(print_func_ref, &[name_ptr, val]);
builder
.ins()
.call(print_func_ref, &[buffer_ptr, name_ptr, val]);
}
Statement::Binding(_, var_name, value) => {

View File

@@ -1,8 +0,0 @@
struct BackendObject {
}
impl BackendObject {
pub fn new() -> Result<Self, ()> {
unimplemented!()
}
}

View File

@@ -4,6 +4,7 @@ use cranelift_jit::JITBuilder;
use cranelift_module::{FuncId, Linkage, Module, ModuleResult};
use std::collections::HashMap;
use std::ffi::CStr;
use std::fmt::Write;
use target_lexicon::Triple;
use thiserror::Error;
@@ -12,17 +13,22 @@ pub struct RuntimeFunctions {
_referenced_functions: Vec<String>,
}
#[derive(Debug, Error)]
#[derive(Debug, Error, PartialEq)]
pub enum RuntimeFunctionError {
#[error("Could not find runtime function named '{0}'")]
CannotFindRuntimeFunction(String),
}
extern "C" fn runtime_print(name: *const i8, value: u64) {
extern "C" fn runtime_print(output_buffer: *mut String, name: *const i8, value: i64) {
let cstr = unsafe { CStr::from_ptr(name) };
let reconstituted = cstr.to_string_lossy();
if let Some(output_buffer) = unsafe { output_buffer.as_mut() } {
writeln!(output_buffer, "{} = {}i64", reconstituted, value).unwrap();
} else {
println!("{} = {}", reconstituted, value);
}
}
impl RuntimeFunctions {
pub fn new<M: Module>(platform: &Triple, module: &mut M) -> ModuleResult<RuntimeFunctions> {
@@ -36,7 +42,7 @@ impl RuntimeFunctions {
"print",
Linkage::Import,
&Signature {
params: vec![string_param, int64_param],
params: vec![string_param, string_param, int64_param],
returns: vec![],
call_conv: CallConv::triple_default(platform),
},

View File

@@ -48,7 +48,7 @@ impl<'a> RunLoop<'a> {
pub fn new(writer: &'a mut dyn WriteColor, config: Config) -> Result<Self, BackendError> {
Ok(RunLoop {
file_database: SimpleFiles::new(),
jitter: Backend::jit()?,
jitter: Backend::jit(None)?,
variable_binding_sites: HashMap::new(),
gensym_index: 1,
writer,

61
src/eval.rs Normal file
View File

@@ -0,0 +1,61 @@
mod env;
mod primop;
mod value;
use cranelift_module::ModuleError;
pub use env::{EvalEnvironment, LookupError};
pub use primop::PrimOpError;
pub use value::Value;
use crate::backend::BackendError;
#[derive(Debug, thiserror::Error)]
pub enum EvalError {
#[error(transparent)]
Lookup(#[from] LookupError),
#[error(transparent)]
PrimOp(#[from] PrimOpError),
#[error(transparent)]
Backend(#[from] BackendError),
#[error("IO error: {0}")]
IO(String),
#[error(transparent)]
Module(#[from] ModuleError),
}
impl From<std::io::Error> for EvalError {
fn from(value: std::io::Error) -> Self {
EvalError::IO(value.to_string())
}
}
impl PartialEq for EvalError {
fn eq(&self, other: &Self) -> bool {
match self {
EvalError::Lookup(a) => match other {
EvalError::Lookup(b) => a == b,
_ => false,
},
EvalError::PrimOp(a) => match other {
EvalError::PrimOp(b) => a == b,
_ => false,
},
EvalError::Backend(a) => match other {
EvalError::Backend(b) => a == b,
_ => false,
},
EvalError::IO(a) => match other {
EvalError::IO(b) => a == b,
_ => false,
},
EvalError::Module(a) => match other {
EvalError::Module(b) => a.to_string() == b.to_string(),
_ => false,
},
}
}
}

93
src/eval/env.rs Normal file
View File

@@ -0,0 +1,93 @@
use crate::eval::Value;
use internment::ArcIntern;
use std::sync::Arc;
pub struct EvalEnvironment {
inner: Arc<EvalEnvInternal>,
}
pub enum EvalEnvInternal {
Empty,
Value(ArcIntern<String>, Value, Arc<EvalEnvInternal>),
}
#[derive(Clone, Debug, PartialEq, thiserror::Error)]
pub enum LookupError {
#[error("Could not find variable '{0}' in environment")]
CouldNotFind(ArcIntern<String>),
}
impl Default for EvalEnvironment {
fn default() -> Self {
EvalEnvironment::empty()
}
}
impl EvalEnvironment {
pub fn empty() -> Self {
EvalEnvironment {
inner: Arc::new(EvalEnvInternal::Empty),
}
}
pub fn extend(&self, name: ArcIntern<String>, value: Value) -> Self {
EvalEnvironment {
inner: Arc::new(EvalEnvInternal::Value(name, value, self.inner.clone())),
}
}
pub fn lookup(&self, n: ArcIntern<String>) -> Result<Value, LookupError> {
self.inner.lookup(n)
}
}
impl EvalEnvInternal {
fn lookup(&self, n: ArcIntern<String>) -> Result<Value, LookupError> {
match self {
EvalEnvInternal::Empty => Err(LookupError::CouldNotFind(n)),
EvalEnvInternal::Value(name, value, _) if *name == n => Ok(value.clone()),
EvalEnvInternal::Value(_, _, rest) => rest.lookup(n),
}
}
}
#[cfg(test)]
mod tests {
use internment::ArcIntern;
use super::EvalEnvironment;
#[test]
fn simple_lookups() {
let tester = EvalEnvironment::default();
let tester = tester.extend(arced("foo"), 1i64.into());
let tester = tester.extend(arced("bar"), 2i64.into());
let tester = tester.extend(arced("goo"), 5i64.into());
assert_eq!(tester.lookup(arced("foo")), Ok(1.into()));
assert_eq!(tester.lookup(arced("bar")), Ok(2.into()));
assert_eq!(tester.lookup(arced("goo")), Ok(5.into()));
assert!(tester.lookup(arced("baz")).is_err());
}
#[test]
fn nested() {
let tester = EvalEnvironment::default();
let tester = tester.extend(arced("foo"), 1i64.into());
check_nested(&tester);
assert_eq!(tester.lookup(arced("foo")), Ok(1.into()));
assert!(tester.lookup(arced("bar")).is_err());
}
fn check_nested(env: &EvalEnvironment) {
let nested_env = env.extend(arced("bar"), 2i64.into());
assert_eq!(nested_env.lookup(arced("foo")), Ok(1.into()));
assert_eq!(nested_env.lookup(arced("bar")), Ok(2.into()));
}
fn arced(s: &str) -> ArcIntern<String> {
ArcIntern::new(s.to_string())
}
}

65
src/eval/primop.rs Normal file
View File

@@ -0,0 +1,65 @@
use crate::eval::value::Value;
#[derive(Clone, Debug, PartialEq, thiserror::Error)]
pub enum PrimOpError {
#[error("Math error (underflow or overflow) computing {0} operator")]
MathFailure(&'static str),
#[error("Type mismatch ({1} vs {2}) computing {0} operator")]
TypeMismatch(String, Value, Value),
#[error("Bad type for operator {0}: {1}")]
BadTypeFor(&'static str, Value),
#[error("Illegal number of arguments for {0}: {1} arguments found")]
BadArgCount(String, usize),
#[error("Unknown primitive operation {0}")]
UnknownPrimOp(String),
}
macro_rules! run_op {
($op: ident, $left: expr, $right: expr) => {
match $op {
"+" => $left
.checked_add($right)
.ok_or(PrimOpError::MathFailure("+"))
.map(Into::into),
"-" => $left
.checked_sub($right)
.ok_or(PrimOpError::MathFailure("+"))
.map(Into::into),
"*" => $left
.checked_mul($right)
.ok_or(PrimOpError::MathFailure("+"))
.map(Into::into),
"/" => $left
.checked_div($right)
.ok_or(PrimOpError::MathFailure("+"))
.map(Into::into),
_ => Err(PrimOpError::UnknownPrimOp($op.to_string())),
}
};
}
impl Value {
fn binary_op(operation: &str, left: &Value, right: &Value) -> Result<Value, PrimOpError> {
match left {
Value::I64(x) => match right {
Value::I64(y) => run_op!(operation, x, *y),
// _ => Err(PrimOpError::TypeMismatch(
// operation.to_string(),
// left.clone(),
// right.clone(),
// )),
},
}
}
pub fn calculate(operation: &str, values: Vec<Value>) -> Result<Value, PrimOpError> {
if values.len() == 2 {
Value::binary_op(operation, &values[0], &values[1])
} else {
Err(PrimOpError::BadArgCount(
operation.to_string(),
values.len(),
))
}
}
}

20
src/eval/value.rs Normal file
View File

@@ -0,0 +1,20 @@
use std::fmt::Display;
#[derive(Clone, Debug, PartialEq)]
pub enum Value {
I64(i64),
}
impl Display for Value {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Value::I64(x) => write!(f, "{}i64", x),
}
}
}
impl From<i64> for Value {
fn from(value: i64) -> Self {
Value::I64(value)
}
}

View File

@@ -1,4 +1,5 @@
mod ast;
mod eval;
mod from_syntax;
mod strings;

View File

@@ -1,10 +1,15 @@
use internment::ArcIntern;
use pretty::{DocAllocator, Pretty};
use proptest::{
prelude::Arbitrary,
strategy::{BoxedStrategy, Strategy},
};
use crate::syntax::Location;
type Variable = ArcIntern<String>;
#[derive(Debug)]
pub struct Program {
pub statements: Vec<Statement>,
}
@@ -28,6 +33,18 @@ where
}
}
impl Arbitrary for Program {
type Parameters = ();
type Strategy = BoxedStrategy<Self>;
fn arbitrary_with(args: Self::Parameters) -> Self::Strategy {
crate::syntax::Program::arbitrary_with(args)
.prop_map(|x| Program::from(x.simplify()))
.boxed()
}
}
#[derive(Debug)]
pub enum Statement {
Binding(Location, Variable, Expression),
Print(Location, Variable),
@@ -54,6 +71,7 @@ where
}
}
#[derive(Debug)]
pub enum Expression {
Value(Location, Value),
Reference(Location, Variable),
@@ -126,6 +144,7 @@ where
}
}
#[derive(Debug)]
pub enum ValueOrRef {
Value(Location, Value),
Ref(Location, ArcIntern<String>),
@@ -144,6 +163,7 @@ where
}
}
#[derive(Debug)]
pub enum Value {
Number(Option<u8>, i64),
}

77
src/ir/eval.rs Normal file
View File

@@ -0,0 +1,77 @@
use crate::eval::{EvalEnvironment, EvalError, Value};
use crate::ir::{Expression, Program, Statement};
use super::{Primitive, ValueOrRef};
impl Program {
pub fn eval(&self) -> Result<String, EvalError> {
let mut env = EvalEnvironment::empty();
let mut stdout = String::new();
for stmt in self.statements.iter() {
match stmt {
Statement::Binding(_, name, value) => {
let actual_value = value.eval(&env)?;
env = env.extend(name.clone(), actual_value);
}
Statement::Print(_, name) => {
let value = env.lookup(name.clone())?;
let line = format!("{} = {}\n", name, value);
stdout.push_str(&line);
}
}
}
Ok(stdout)
}
}
impl Expression {
fn eval(&self, env: &EvalEnvironment) -> Result<Value, EvalError> {
match self {
Expression::Value(_, v) => match v {
super::Value::Number(_, v) => Ok(Value::I64(*v)),
},
Expression::Reference(_, n) => Ok(env.lookup(n.clone())?),
Expression::Primitive(_, op, args) => {
let mut arg_values = Vec::with_capacity(args.len());
for arg in args.iter() {
match arg {
ValueOrRef::Ref(_, n) => arg_values.push(env.lookup(n.clone())?),
ValueOrRef::Value(_, super::Value::Number(_, v)) => {
arg_values.push(Value::I64(*v))
}
}
}
match op {
Primitive::Plus => Ok(Value::calculate("+", arg_values)?),
Primitive::Minus => Ok(Value::calculate("-", arg_values)?),
Primitive::Times => Ok(Value::calculate("*", arg_values)?),
Primitive::Divide => Ok(Value::calculate("/", arg_values)?),
}
}
}
}
}
#[test]
fn two_plus_three() {
let input = crate::syntax::Program::parse(0, "x = 2 + 3; print x;").expect("parse works");
let ir = Program::from(input.simplify());
let output = ir.eval().expect("runs successfully");
assert_eq!("x = 5i64\n", &output);
}
#[test]
fn lotsa_math() {
let input =
crate::syntax::Program::parse(0, "x = 2 + 3 * 10 / 5 - 1; print x;").expect("parse works");
let ir = Program::from(input.simplify());
let output = ir.eval().expect("runs successfully");
assert_eq!("x = 7i64\n", &output);
}

View File

@@ -71,3 +71,13 @@ impl From<syntax::Value> for ir::Value {
}
}
}
proptest::proptest! {
#[test]
fn translation_maintains_semantics(input: syntax::Program) {
let syntax_result = input.eval();
let ir = ir::Program::from(input.simplify());
let ir_result = ir.eval();
assert_eq!(syntax_result, ir_result);
}
}

View File

@@ -1,3 +1,4 @@
pub mod backend;
pub mod eval;
pub mod ir;
pub mod syntax;

View File

@@ -4,6 +4,7 @@ use logos::Logos;
mod arbitrary;
pub mod ast;
mod eval;
mod location;
mod simplify;
mod tokens;
@@ -269,4 +270,10 @@ proptest::proptest! {
let (errors, _) = program.validate();
prop_assert!(errors.is_empty());
}
#[test]
fn generated_run_or_overflow(program: Program) {
use crate::eval::{EvalError, PrimOpError};
assert!(matches!(program.eval(), Ok(_) | Err(EvalError::PrimOp(PrimOpError::MathFailure(_)))))
}
}

64
src/syntax/eval.rs Normal file
View File

@@ -0,0 +1,64 @@
use internment::ArcIntern;
use crate::eval::{EvalEnvironment, EvalError, Value};
use crate::syntax::{Expression, Program, Statement};
impl Program {
pub fn eval(&self) -> Result<String, EvalError> {
let mut env = EvalEnvironment::empty();
let mut stdout = String::new();
for stmt in self.statements.iter() {
match stmt {
Statement::Binding(_, name, value) => {
let actual_value = value.eval(&env)?;
env = env.extend(ArcIntern::new(name.clone()), actual_value);
}
Statement::Print(_, name) => {
let value = env.lookup(ArcIntern::new(name.clone()))?;
let line = format!("{} = {}\n", name, value);
stdout.push_str(&line);
}
}
}
Ok(stdout)
}
}
impl Expression {
fn eval(&self, env: &EvalEnvironment) -> Result<Value, EvalError> {
match self {
Expression::Value(_, v) => match v {
super::Value::Number(_, v) => Ok(Value::I64(*v)),
},
Expression::Reference(_, n) => Ok(env.lookup(ArcIntern::new(n.clone()))?),
Expression::Primitive(_, op, args) => {
let mut arg_values = Vec::with_capacity(args.len());
for arg in args.iter() {
arg_values.push(arg.eval(env)?);
}
Ok(Value::calculate(op, arg_values)?)
}
}
}
}
#[test]
fn two_plus_three() {
let input = Program::parse(0, "x = 2 + 3; print x;").expect("parse works");
let output = input.eval().expect("runs successfully");
assert_eq!("x = 5i64\n", &output);
}
#[test]
fn lotsa_math() {
let input = Program::parse(0, "x = 2 + 3 * 10 / 5 - 1; print x;").expect("parse works");
let output = input.eval().expect("runs successfully");
assert_eq!("x = 7i64\n", &output);
}