Add comment parsing to the lexer

Now lexer can properly parse and skip comments, as well as any
other combination of whitespace and comments.
This commit is contained in:
Aodhnait Étaín 2021-05-28 12:59:55 +00:00
parent 48f47671cd
commit 569391612a
Signed by: aodhneine
GPG Key ID: ECE91C73AD45F245
1 changed files with 33 additions and 6 deletions

View File

@ -48,21 +48,47 @@ impl<'a> Source<'a> {
};
}
fn skip_whitespace(&mut self) {
fn skip_whitespace(&mut self) -> bool {
let mut chars = self.source[self.cursor..].chars();
let mut skipped = false;
while let Some(c) = chars.next() {
if c.is_whitespace() {
self.cursor += c.len_utf8();
skipped = true;
} else {
return;
return skipped;
}
}
};
return skipped;
}
fn skip_comments(&mut self) -> bool {
let mut chars = self.source[self.cursor..].chars();
if let Some('/') = chars.next() {
if let Some('/') = chars.next() {
self.cursor += 2;
while let Some(c) = chars.next() {
if c == '\n' {
self.cursor += 1;
return true;
};
self.cursor += c.len_utf8();
};
};
};
return false;
}
fn get_next(&mut self) -> Option<Token<'a>> {
self.skip_whitespace();
let mut chars = self.source[self.cursor..].chars();
// Skip all possible comments and whitespace.
while self.skip_comments() || self.skip_whitespace() { };
let mut chars = self.source[self.cursor..].chars().peekable();
let token = match chars.next()? {
'+' => Token::Plus,
@ -215,7 +241,8 @@ impl<'a> Parser<'a> {
}
fn main() {
let inline_source = "3 + 2 - -5;";
let inline_source = " // hello, world\n 3;";
// let inline_source = "3 + 2 - -5;";
// let inline_source = "3 + +5 * +7;";
// let inline_source = "3 + 5 * 7;";
// let inline_source = "(3 + 5) + 7;";