Add multiplication to the expression parser

Now it also handles expression such as 11 * 13, 11 + 13 * 17, and even
11 + 13 * 17 + 19. We also introduced new function, is_binary_operator,
which allows for simpler checking of operators in the expression parser.

We fixed the issue that parser didn't parse its input exhaustively, i.e.
when given '+17 + 23 + +21' it would return only +17 + 23, which is far
from what we would expect.

src/main.rs

@@ -91,7 +91,8 @@ fn main() {
 	let path = path.unwrap();
 	eprintln!("compiling `{}`", path);
 
-	#[allow(non_upper_case_globals)] const source: &'static str = "+17 + 23 + +21;";
+	// #[allow(non_upper_case_globals)] const source: &'static str = "+17 + 23 + +21 + 11;";
+	#[allow(non_upper_case_globals)] const source: &'static str = "11 + 13 * 17 + 19;";
 	let mut tokens = TokenStream::from(source);
 	let expr = parse_expression(&mut tokens, 0);
 	eprintln!("{:?}", expr);
@@ -203,20 +204,28 @@ impl Expression {
 
 fn unary_precedence(token: Token) -> Option<usize> {
 	return match token {
-		Token::Plus => Some(2),
+		Token::Plus => Some(3),
 		_ => None,
 	};
 }
 
 fn binary_precedence(token: Token) -> Option<usize> {
 	return match token {
+		Token::Asterisk => Some(2),
 		Token::Plus => Some(1),
 		_ => None,
 	};
 }
 
+fn is_binary_operator(token: Token) -> bool {
+	return match token {
+		Token::Plus | Token::Asterisk => true,
+		_ => false,
+	};
+}
+
 fn parse_expression<'a, 'b: 'a>(tokens: &'a mut TokenStream<'b>, highest_precedence: usize) -> Option<Expression> {
-	let lhs = match tokens.next()? {
+	let mut lhs = match tokens.next()? {
 		token @ Token::IntegerLiteral(_) => Expression::Literal(token),
 		token => {
 			if let Some(precedence) = unary_precedence(token) {
@@ -230,8 +239,8 @@ fn parse_expression<'a, 'b: 'a>(tokens: &'a mut TokenStream<'b>, highest_precede
 
 	loop {
 		let operator = match tokens.peek()? {
-			operator @ Token::Plus => operator,
+			operator if is_binary_operator(operator) => operator,
-			_ => return None,
+			_ => return Some(lhs),
 		};
 
 		let precedence =  binary_precedence(operator)?;
@@ -240,17 +249,17 @@ fn parse_expression<'a, 'b: 'a>(tokens: &'a mut TokenStream<'b>, highest_precede
 			return Some(lhs);
 		}
 
-		return match operator {
+		if is_binary_operator(operator) {
-			Token::Plus => {
 			tokens.next();
 			let rhs = parse_expression(tokens, precedence)?;
+			lhs = Expression::Binary(operator, box lhs, box rhs);
 
-				Some(Expression::Binary(operator, box lhs, box rhs))
+			if tokens.peek().map(is_binary_operator).unwrap_or(false) {
-			},
+				continue;
-			// If it's not a valid operator, then caller can get rest of the input in the token stream
+			}
-			// it has provided to us.
+		}
-			_ => Some(lhs),
+
-		};
+		return Some(lhs);
 	}
 }
 
@@ -298,6 +307,7 @@ impl<'a> TokenStream<'a> {
 		let mut chars = self.chars();
 		
 		let token = match chars.next()? {
+			'*' => Token::Asterisk,
 			'+' => Token::Plus,
 			';' => Token::Semicolon,
 			c if c.is_numeric() => {
@@ -378,6 +388,7 @@ impl OffsetStr {
 #[derive(Debug, Copy, Clone)]
 enum Token {
 	Plus,
+	Asterisk,
 	Semicolon,
 	IntegerLiteral(OffsetStr),
 }
@@ -388,6 +399,7 @@ impl std::fmt::Display for Token {
 			Token::IntegerLiteral(s) => write!(f, "{}", s),
 			token => write!(f, "{}", match token {
 				Token::Plus => "+",
+				Token::Asterisk => "*",
 				Token::Semicolon => ";",
 				_ => unreachable!(),
 			}),
@@ -398,7 +410,7 @@ impl std::fmt::Display for Token {
 impl Token {
 	pub fn len(&self) -> usize {
 		return match self {
-			Token::Plus | Token::Semicolon => 1,
+			Token::Plus | Token::Asterisk | Token::Semicolon => 1,
 			Token::IntegerLiteral(i) => i.length,
 		};
 	}