Update the lexer to match changes to the grammar

This commit is contained in:
Emi Simpson 2022-04-22 11:49:28 -04:00
parent 756bb883f0
commit 250d40dab7
Signed by: Emi
GPG key ID: A12F2C2FFDC3D847
7 changed files with 48 additions and 178 deletions

View file

@ -25,8 +25,8 @@
<rank5_expr> ::= <rank6_expr> { <rank5_op> <rank6_expr> } <rank5_expr> ::= <rank6_expr> { <rank5_op> <rank6_expr> }
<rank6_expr> ::= <rank7_expr> { <rank6_op> <rank7_expr> } <rank6_expr> ::= <rank7_expr> { <rank6_op> <rank7_expr> }
<rank7_expr> ::= <tightexpr> { <rank7_op> <tightexpr> } <rank7_expr> ::= <tightexpr> { <rank7_op> <tightexpr> }
<rank1_op> ::= LOr <rank1_op> ::= LOr | VBar
<rank2_op> ::= LAnd <rank2_op> ::= LAnd | Aro
<rank3_op> ::= Eq | NEq | LessThan | GreaterThan <rank3_op> ::= Eq | NEq | LessThan | GreaterThan
<rank5_op> ::= Range <rank5_op> ::= Range
<rank6_op> ::= Add | Sub <rank6_op> ::= Add | Sub

View file

@ -1,7 +1,8 @@
type MyType MyType: Type
= Variant1 MyType = type
| Variant2 int[usize] Variant1
| Variant3 str int[0..21] | Variant2 (int 0..usize)
| Variant3 str (int 0..21)
myFunc: str, int -> MyType myFunc: str, int -> MyType
myFunc strval intval = myFunc strval intval =
@ -9,8 +10,9 @@ myFunc strval intval =
then Variant3 strval intval then Variant3 strval intval
else Variant1 else Variant1
type ComplexType ComplexType: Type
= ComplexVariant (str, int -> MyType) str ComplexType = type
ComplexVariant (str, int -> MyType) str
myVal : ComplexType myVal : ComplexType
myVal = ComplexVariant myFunc "uwu~" myVal = ComplexVariant myFunc "uwu~"

View file

@ -2,73 +2,36 @@ use std::mem::{Discriminant, discriminant};
use crate::token::Token; use crate::token::Token;
use super::{Parsable, WrappedLexer, ParseError, absorb_token_or_error, types::{TightType, FullType}, expr::Expr}; use super::{Parsable, WrappedLexer, ParseError, absorb_token_or_error, expr::Expr};
#[derive(Debug)] #[derive(Debug)]
pub enum Declaration { pub struct Declaration {
Type { name: String,
name: String, type_: Expr,
first_variants: Variant, name2: String,
other_variants: Vec<Variant>, args: Vec<String>,
}, value: Expr,
/*Struct {
name: String,
typeargs: Vec<TypeArg>,
first_fields: Field,
other_fields: Vec<Field>,
},
Trait {
name: String,
typeargs: Vec<TypeArg>,
on: Option<CompositeType>,
},
Impl(ImplDecl),*/
Symbol {
name: String,
type_: FullType,
name2: String,
args: Vec<String>,
value: Expr,
},
Empty,
} }
impl Parsable for Declaration { impl Parsable for Declaration {
fn expected() -> (Vec<Discriminant<Token>>, bool) { fn expected() -> (Vec<Discriminant<Token>>, bool) {
(vec![ (vec![
discriminant(&Token::Symbol("".to_string())), discriminant(&Token::Symbol("".to_string())),
discriminant(&Token::Type),
], false) ], false)
} }
fn parse(l: &mut WrappedLexer) -> Result<Self, ParseError> { fn parse(l: &mut WrappedLexer) -> Result<Self, ParseError> {
let span = l.span(); let span = l.span();
match l.monch() { match l.monch() {
// Type Symbol Assign <variant_decl> { VBar <variant_decl> }
Token::Type => {
let name = String::parse(l)?;
absorb_token_or_error(l, discriminant(&Token::Assign))?;
let first_variants = Variant::parse(l)?;
let mut other_variants = Vec::new();
while *l.curtok() == Token::VBar {
l.monch();
other_variants.push(Variant::parse(l)?);
}
Ok(Self::Type {
first_variants,
other_variants,
name,
})
},
// Symbol Colon <full_type> DeclarationStart Symbol { Symbol } Assign <expr> // Symbol Colon <full_type> DeclarationStart Symbol { Symbol } Assign <expr>
Token::Symbol(name) => { Token::Symbol(name) => {
absorb_token_or_error(l, discriminant(&Token::Colon))?; absorb_token_or_error(l, discriminant(&Token::Colon))?;
let type_ = FullType::parse(l)?; let type_ = Expr::parse(l)?;
absorb_token_or_error(l, discriminant(&Token::DeclarationStart))?; absorb_token_or_error(l, discriminant(&Token::DeclarationStart))?;
let name2 = String::parse(l)?; let name2 = String::parse(l)?;
let args = Vec::parse(l)?; let args = Vec::parse(l)?;
absorb_token_or_error(l, discriminant(&Token::Assign))?; absorb_token_or_error(l, discriminant(&Token::Assign))?;
let value = Expr::parse(l)?; let value = Expr::parse(l)?;
Ok(Self::Symbol { Ok(Self {
name, name,
name2, name2,
args, args,
@ -76,9 +39,6 @@ impl Parsable for Declaration {
value, value,
}) })
}, },
Token::EOF | Token::DeclarationStart => {
Ok(Self::Empty)
},
_ => { _ => {
Err(ParseError { Err(ParseError {
expected: Self::expected().0, expected: Self::expected().0,
@ -88,25 +48,3 @@ impl Parsable for Declaration {
} }
} }
} }
#[derive(Debug)]
pub struct Variant {
name: String,
elements: Vec<TightType>,
}
impl Parsable for Variant {
fn expected() -> (Vec<Discriminant<Token>>, bool) {
(
vec![discriminant(&Token::Symbol(String::new()))],
false,
)
}
fn parse(l: &mut WrappedLexer) -> Result<Self, ParseError> {
// <variant_declaration> ::= Symbol { <grouped_type> }
Ok(Variant {
name: String::parse(l)?,
elements: Vec::parse(l)?,
})
}
}

View file

@ -1,7 +1,7 @@
// <expr> ::= <tiGHtexpr> { <tightexpr> } // <expr> ::= If <expr> <ifblock>
// | If <expr> <ifblock>
// | <let> { <let> } In <expr> // | <let> { <let> } In <expr>
// | <infix_expr> // | <infix_expr>
// | TypeOp <expr>
// //
// <tightexpr> ::= Literal // <tightexpr> ::= Literal
// | OpenParen <expr> CloseParen // | OpenParen <expr> CloseParen
@ -24,6 +24,7 @@ pub enum Expr {
If(Box<Expr>, Box<IfBlock>), If(Box<Expr>, Box<IfBlock>),
Let(Box<LetStmt>, Vec<LetStmt>, Box<Expr>), Let(Box<LetStmt>, Vec<LetStmt>, Box<Expr>),
Infix(Rank1Exp), Infix(Rank1Exp),
Type(Box<Expr>),
} }
impl Parsable for Expr { impl Parsable for Expr {
@ -32,6 +33,7 @@ impl Parsable for Expr {
expected.extend_from_slice(&[ expected.extend_from_slice(&[
discriminant(&Token::Let), discriminant(&Token::Let),
discriminant(&Token::If), discriminant(&Token::If),
discriminant(&Token::TypeOp),
]); ]);
(expected, false) (expected, false)
} }
@ -49,6 +51,11 @@ impl Parsable for Expr {
let in_expr = Expr::parse(l)?; let in_expr = Expr::parse(l)?;
Ok(Expr::Let(Box::new(initial_let), subsequent_lets, Box::new(in_expr))) Ok(Expr::Let(Box::new(initial_let), subsequent_lets, Box::new(in_expr)))
} }
Token::TypeOp => {
l.monch();
let value = Expr::parse(l)?;
Ok(Expr::Type(Box::new(value)))
}
Token::If => { Token::If => {
l.monch(); l.monch();
let cond = Expr::parse(l)?; let cond = Expr::parse(l)?;
@ -172,7 +179,7 @@ impl Parsable for Case {
fn parse(l: &mut WrappedLexer) -> Result<Self, ParseError> { fn parse(l: &mut WrappedLexer) -> Result<Self, ParseError> {
let pattern = BindingPattern::parse(l)?; let pattern = BindingPattern::parse(l)?;
absorb_token_or_error(l, discriminant(&Token::Aro))?; absorb_token_or_error(l, discriminant(&Token::DubAro))?;
let expr = Expr::parse(l)?; let expr = Expr::parse(l)?;
Ok(Case(pattern, expr)) Ok(Case(pattern, expr))
} }

View file

@ -19,7 +19,6 @@ use std::mem::discriminant;
pub mod program; pub mod program;
pub mod declaration; pub mod declaration;
pub mod types;
pub mod expr; pub mod expr;
pub mod pattern; pub mod pattern;
pub mod infix; pub mod infix;

View file

@ -1,85 +0,0 @@
// <full_type> ::= <tight_type> [ Comma [ <additional_type> ] [ Aro <full_type> ] ]
// <additional_type> ::= <tight_type> [ Comma [ <additional_type> ] ]
// <tight_type> ::= OpenParen <full_type> CloseParen
// | Symbol [ OpenSquareBracket <domain> CloseSquareBracket ]
// <domain> ::= <expr> [ Comma [ <domain> ] ]
use std::mem::{Discriminant, discriminant};
use crate::token::Token;
use super::{Parsable, WrappedLexer, parse_delineated_vec, absorb_token_or_error, ParseError, expr::Expr};
#[derive(Debug)]
pub enum TightType {
Grouped(Box<FullType>),
Simple {
name: String,
domains: Vec<Expr>,
},
}
impl Parsable for TightType {
fn expected() -> (Vec<Discriminant<Token>>, bool) {
(
vec![
discriminant(&Token::OpenParen),
discriminant(&Token::Symbol(String::new())),
],
false,
)
}
fn parse(l: &mut WrappedLexer) -> Result<Self, ParseError> {
let span = l.span();
match l.monch() {
Token::Symbol(name) => {
let domains: Vec<Expr> = if *l.curtok() == Token::OpenSquareBracket {
l.monch();
let d = parse_delineated_vec(l, discriminant(&Token::Comma))?;
absorb_token_or_error(l, discriminant(&Token::CloseSquareBracket))?;
d
} else {
Vec::new()
};
Ok(TightType::Simple { name, domains })
}
Token::OpenParen => {
let full_type = Box::new(FullType::parse(l)?);
absorb_token_or_error(l, discriminant(&Token::CloseParen))?;
Ok(TightType::Grouped(full_type))
}
_ => Err(ParseError {
location: span,
expected: Self::expected().0,
})
}
}
}
#[derive(Debug)]
pub enum FullType {
Simple(TightType),
Function(Vec<TightType>, Box<FullType>),
}
impl Parsable for FullType {
fn expected() -> (Vec<Discriminant<Token>>, bool) {
TightType::expected()
}
// <full_type> ::= <tight_type> [ Comma [ <additional_type> ] [ Aro <full_type> ] ]
fn parse(l: &mut WrappedLexer) -> Result<Self, ParseError> {
let first_type = TightType::parse(l)?;
if let Token::Comma = l.curtok() {
l.monch();
let other_args = parse_delineated_vec(l, discriminant(&Token::Comma))?;
absorb_token_or_error(l, discriminant(&Token::Aro))?;
let result = Box::new(FullType::parse(l)?);
let mut args = Vec::with_capacity(1 + other_args.len());
args.push(first_type);
args.extend(other_args);
Ok(FullType::Function(args, result))
} else {
Ok(FullType::Simple(first_type))
}
}
}

View file

@ -103,11 +103,11 @@ pub enum Token {
#[token("in")] #[token("in")]
In, //d12 In, //d12
/// An `->` arrow /// An `=>` arrow
/// ///
/// Used as part of function type annotations as well as in the cases of If-Is blocks /// Used as part of function type annotations as well as in the cases of If-Is blocks
#[token("->")] #[token("=>")]
Aro, //d13 DubAro, //d13
/// An `=` assignment operator /// An `=` assignment operator
/// ///
@ -115,11 +115,9 @@ pub enum Token {
#[token("=")] #[token("=")]
Assign, //d14 Assign, //d14
/// The `|` keyword (or punctuation? idk what it's called) /// Type Operator
/// #[token("type", priority = 9)]
/// Used in deliniating variants of a type TypeOp, //d15
#[token("|")]
VBar, //d15
/// The `_` symbol /// The `_` symbol
/// ///
@ -137,12 +135,15 @@ pub enum Token {
/// ///
/// i.e. Logical Or /// i.e. Logical Or
#[token("||", |_| InfixRank1::LOr)] #[token("||", |_| InfixRank1::LOr)]
#[token("|", |_| InfixRank1::VBar)]
R1Infix(InfixRank1), //d18 R1Infix(InfixRank1), //d18
/// A rank 2 infix binop (binary operator) /// A rank 2 infix binop (binary operator)
/// ///
/// i.e. Logical And /// i.e. Logical And
#[token("&&", |_| InfixRank2::LAnd)] #[token("&&", |_| InfixRank2::LAnd)]
#[token("->", |_| InfixRank2::Aro)]
#[token(",", |_| InfixRank2::Aro)]
R2Infix(InfixRank2), //d19 R2Infix(InfixRank2), //d19
/// A rank 3 infix binop (binary operator) /// A rank 3 infix binop (binary operator)
@ -225,7 +226,7 @@ pub enum Token {
/// A `,` comma /// A `,` comma
/// ///
/// The age-old and timeless delineator /// The age-old and timeless delineator
#[token(",")] #[token(";")]
Comma, //d31 Comma, //d31
/// A newline NOT followed by whitespace /// A newline NOT followed by whitespace
@ -245,7 +246,7 @@ pub enum Token {
/// ///
/// simply lexes to `Symbol(variable)`, `Assign`, `Symbol(value)`. This makes it easy /// simply lexes to `Symbol(variable)`, `Assign`, `Symbol(value)`. This makes it easy
/// to identify declarations. /// to identify declarations.
#[regex(r"\s*\n")] #[regex(r"(\s*\n)+")]
DeclarationStart, //d32 DeclarationStart, //d32
/// Denotes that the parser has reached the end of the input /// Denotes that the parser has reached the end of the input
@ -261,6 +262,11 @@ pub enum InfixRank1 {
/// ///
/// Takes two boolean values and returns true if either is true /// Takes two boolean values and returns true if either is true
LOr, LOr,
/// The VBar operator
///
/// Takes two variant sets and returns sum
VBar,
} }
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@ -270,6 +276,9 @@ pub enum InfixRank2 {
/// Takes two boolean values and returns true iff both values are true. Otherwise, /// Takes two boolean values and returns true iff both values are true. Otherwise,
/// returns false. /// returns false.
LAnd, LAnd,
/// The Aro operator
Aro,
} }
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]