JSON-Lang/comb_parse.py

from emis_funky_funktions import *

from dataclasses import dataclass
from enum import auto, IntEnum
from functools import reduce
from re import compile, Pattern

from lex import Lexeme, tokenize

from typing import Any, Callable, Collection, Mapping, Sequence, Tuple, TypeAlias

# Grammar
# S := <P> Eof
# P := <M> + <P>
# P := <M>
# M := <T> * <M>
# M := <T>
# T := Letter
# T := Number
# T := ( <P> )

class TrivialTok(IntEnum):
	"A set of tokens for a trivial grammar used for testing"

	EOF = auto()
	"Special token: End of file"

	ADD = auto()
	"Addition (+)"

	MUL = auto()
	"Multiplication (*)"

	LTR = auto()
	"A single letter (x)"

	NUM = auto()
	"A number (3)"

	OPN = auto()
	"An open paren, i.e. '('"

	CLS = auto()
	"An close paren, i.e. ')'"

	NUL = auto()
	"Whitespace ( )"

	def __repr__(self):
		return self._name_

TRIVIAL_LEX_TABLE: Collection[Tuple[Pattern[str], TrivialTok]] = [
	(compile(r"\+"      ), TrivialTok.ADD),
	(compile(r"\*"      ), TrivialTok.MUL),
	(compile(r"[a-zA-Z]"), TrivialTok.LTR),
	(compile("\d+"      ), TrivialTok.NUM),
	(compile("\("       ), TrivialTok.OPN),
	(compile("\)"       ), TrivialTok.CLS),
	(compile("\s+"      ), TrivialTok.NUL),
]
"""
A mapping of regexs to the tokens the identify in a trivial grammar for testing

Tokens earlier on in the list should be regarded as higher priority, even if a match lower
on the list also matches.  All unicode strings should be matched by at least one token.
"""

lex_trivial: Callable[[str], List[Lexeme[TrivialTok]]] =\
	c(unwrap_r, p(tokenize, TRIVIAL_LEX_TABLE, [TrivialTok.NUL], TrivialTok.EOF)) #type: ignore
"""
A lexer for the trivial grammar defined above

Throws an error if the lex fails.

>>> lex_trivial("1 + 3") #doctest: +NORMALIZE_WHITESPACE
[[NUM: '1']@(1, 1-1),
 [ADD: '+']@(1, 3-3),
 [NUM: '3']@(1, 5-5),
 [EOF: '']@(1, 6-6)]
"""

############# Combinator Parsing ################

Out = TypeVar('Out')
Out1 = TypeVar('Out1')
Out2 = TypeVar('Out2')
TokN = TypeVar('TokN')
LexStream: TypeAlias = Sequence[Lexeme[TokN]]
ParserResult: TypeAlias = Result[Collection[Tuple[Out, LexStream[TokN]]], Collection[Tuple[Lexeme[TokN], TokN]]]

@dataclass(frozen=True)
class Parser(Generic[Out, TokN]):
	"""
	A parser which consumes a token stream and produces a series of possible parses

	Each possible parse consists of the subsection of the input stream left over after the
	parse, along with the output of that parse.

	If the parse fails, the error returned will be a series of two-tuples, each containing
	the lexeme at which the error occurred and the token which was expected in its stead.

	By the nature of combinator parsing, parsers can be built and tested in small pieces.
	To this end, each combinator method on this class comes with a small example
	demonstrating how to build a trivial parser using that method.  However, in the
	following section, we also provide an example for a grammar that is still trivial, but
	slightly less so.

	### Example:  Arithmetic Grammar

	Let us define the following grammar for basic additive and multiplicative arithmetic

	```
	S := <P> Eof
	P := <M> + <P>
	P := <M>
	M := <T> * <M>
	M := <T>
	T := Number
	T := ( <P> )
	```

	As with the rest of the examples in this class, we will use the `TrivialTok` token
	class to build this parser.

	Working our way from the bottom up, we start with defining a parser for T.  For the
	parenthetical production, we use a lazy parser to refer to the parser for P, which
	hasn't been constructed yet.

	>>> parse_parens = Parser.token(TrivialTok.OPN)\\
	...                      .bind(k(Parser.lazy(lambda: parse_p)))\\
	...                      .seq_ignore_tok(TrivialTok.CLS)
	>>> parse_num = Parser.token(TrivialTok.NUM)\\
	...                   .map(c(int, Lexeme.get_match))
	>>> parse_t = parse_parens.or_(parse_num)

	For multiplication, we use `Parser.many()` to represent any number of "* <T>" matches
	and combine them together.  This is a slight departure from how our grammar is
	written above.  The set of accepted inputs will be the same, but our implementation
	will be left-associative.  It is possible to implement this grammar as it is written,
	but this will result in a right-associative structure.  Of course, multiplication and
	addition are associative, so in this case it doesn't matter.

	>>> parse_times = Parser.token(TrivialTok.MUL)\\
	...                     .map(k(lambda x: lambda y: x * y))\\
	...                     .fapply(parse_t)\\
	...                     .many(lambda l: lambda r: c(l, r), lambda x: x)
	>>> parse_m = parse_t.fapply_r(parse_times)

	Addition is largely the same as multiplication:

	>>> parse_plus = Parser.token(TrivialTok.ADD)\\
	...                    .map(k(lambda x: lambda y: x + y))\\
	...                    .fapply(parse_m)\\
	...                    .many(lambda l: lambda r: c(l, r), lambda x: x)
	>>> parse_p = parse_m.fapply_r(parse_plus)

	And finally, we expect and EOF after the end of the term, to ensure that we've reached
	the end of the input.

	>>> parse_s = parse_p.seq_ignore_tok(TrivialTok.EOF)

	And now, for a few sample runs:

	>>> parse_s.parse_(lex_trivial('1'))
	Ok(1)

	>>> parse_s.parse_(lex_trivial('1 + 2'))
	Ok(3)

	>>> parse_s.parse_(lex_trivial('1 + 2 * 3 + 4'))
	Ok(11)

	>>> parse_s.parse_(lex_trivial('(1 + 2) * (3 + 4)'))
	Ok(21)

	>>> parse_s.parse_(lex_trivial('(1 + 2 * 3) * (4 + 5) + 6'))
	Ok(69)

	And an example of a bad parse:

	# TODO fix this
	>>> parse_s.parse_(lex_trivial('1 * * 2')) #doctest: +ELLIPSIS
	Err(...)
	"""

	parse: Callable[[LexStream[TokN]], ParserResult[Out, TokN]]
	"""
	Run this parser

	Accepts a stream of tokens to parse, and returns either a list of possible successful
	parses, each with a value and a remainder of the token stream, or a list of all
	failures which lead to the failure of this parser, each with a lexeme they failed at
	and the token the expected.

	This is meant for use in constructing parsers, and it's probably more useful to call
	`parse_()`.
	"""

	def parse_(self, input: LexStream[TokN]) -> Result[Out, Mapping[Lexeme[TokN], Collection[TokN]]]:
		"""
		Run this parser, expecting at most one result which consumes the full input stream

		This is a wrapper around `parse()` with a few features for ease of use, including:
			- expects that at most one single successful parse will be produced
			- asserts that the parse consumed the whole input stream
			- aggregates failures into a multi-set of `Lexeme` -> expected tokens

		Normal failures are passed through as an `Err`, but if the results returned by a
		successful parse violate the above conditions, an assertion error will be raised.
		However, by carefully constructing your grammar, you can ensure that this will not
		happen for any input.

		Because of these assertions, the successful return value of this function is
		*just* the output type of this parse, which is much easier to use.

		### Example

		A parser which parses a single number.  Notice that we also require that it parses
		an EOF.  Without this, it would be unable to consume the entire input stream, and
		thus would fail.

		>>> parse_num = Parser.token(TrivialTok.NUM).seq_ignore_tok(TrivialTok.EOF)
		>>> parse_num.parse_(lex_trivial('1312'))
		Ok([NUM: '1312']@(1, 1-4))
		"""
		match self.parse(input):
			case Err(errors):
				# The input is bad
				failure_locations = FSet(lex for (lex, expected) in errors)
				return Err({
					location: FSet(expected for (lex, expected) in errors if lex == location)
					for location in failure_locations
				})
			case Ok([result1, result2, *rest] as possible_parses):
				# The grammar is bad
				raise AssertionError("Parse returned multiple possible parses", possible_parses)
			case Ok([(value, [non_empty, *rest] as remainder)]):
				# The grammar is bad
				raise AssertionError("Parse failed to consume the whole input, and left remainder", remainder)
			case Ok([]):
				# The parser code is bad
				raise AssertionError('"Successful" parse returned no possible parses')
			case Ok([(value, [])]):
				return Ok(value)
		# The code in this function is bad
		raise AssertionError('Unreachable')

	@staticmethod
	def epsilon(ret: Out) -> 'Parser[Out, TokN]':
		"""
		Parse an empty string, then return a constant

		Always succeeds, and always produces exactly one possibility

		>>> Parser.epsilon(100).parse(lex_trivial("+"))
		Ok(((100, [[ADD: '+']@(1, 1-1), [EOF: '']@(1, 2-2)]),))
		"""
		return Parser(lambda s: Ok(((ret, s),)))

	@staticmethod
	def token(t: TokN)  -> 'Parser[Lexeme[TokN], TokN]':
		"""
		Parser that only accepts a single token, and returns the parsed lexeme

		The first argument is a function which, given a lexeme, returns the token that
		that lexeme instantiates.  The second argument is a token which this parser should
		accept

		>>> parse_num = Parser.token(TrivialTok.NUM)

		>>> parse_num.parse(lex_trivial('3'))
		Ok((([NUM: '3']@(1, 1-1), [[EOF: '']@(1, 2-2)]),))

		>>> parse_num.parse(lex_trivial('x'))
		Err((([LTR: 'x']@(1, 1-1), NUM),))
		"""
		def parse_single(input: LexStream) -> ParserResult[Lexeme[TokN], TokN]:
			match input:
				case [lexeme, *rest] if lexeme.token == t:
					return Ok(((lexeme, rest),))
				case [bad_lexeme, *rest]:
					return Err(((bad_lexeme, t),))
				case []:
					raise Exception('Bad grammar!  Reached an empty input')
			raise Exception('Unreachable')
		return Parser(parse_single)

	@staticmethod
	def lazy(
		gen: 'Callable[[], Parser[Out, TokN]]',
	) -> 'Parser[Out, TokN]':
		"""
		A stand-in parser which will only be actually computed when called
		"""
		return Parser(lambda s: gen().parse(s))

	def bind(self, f: 'Callable[[Out], Parser[Out2, TokN]]') -> 'Parser[Out2, TokN]':
		"""
		A monadic bind operator - allows a parser to be generated from its precenent

		### Example

		We generate a parser which reads a number then accepts exactly that many plus
		tokens, returning the last one.

		>>> some_n = Parser.token(TrivialTok.NUM)
		>>> n_plus = some_n.bind(lambda prev_result:
		...     reduce(
		...         lambda a, p: a.bind(k(p)),
		...         [
		...             Parser.token(TrivialTok.ADD)
		...             for i in range(int(prev_result.matched_string))
		...         ]
		...     )
		... )

		**Sample Run 1**: We parse the string `3+++`.  Since this is a three followed by
		exactly three plus signs, this should parse successfully.  Sure enough, the result
		contains exactly one possibility, where the `3+++` has been consumed, leaving only
		the EOF, and returning the value of the last plus sign.

		>>> n_plus.parse(lex_trivial('3+++'))
		Ok([([ADD: '+']@(1, 4-4), [[EOF: '']@(1, 5-5)])])

		**Simple Run 2**: We parse the string `3++`.  This only has two of the three plus
		signs, so we should expect it to fail.  As expected, at does, correctly
		identifying that it saw an EOF while expecting an ADD.

		>>> n_plus.parse(lex_trivial('3++'))
		Err([([EOF: '']@(1, 4-4), ADD)])
		"""
		def handle_results(results: Collection[Tuple[Out, LexStream]]) -> ParserResult[Out2, TokN]:
			successes, errors = partition([
				f(out1).parse(stream)
				for (out1, stream) in results
			])
			if len(successes):
				return Ok([p for s in successes for p in s])
			else:
				return Err([e for errs in errors for e in errs])
		def inner(input: LexStream) -> ParserResult[Out2, TokN]:
			return self.parse(input) << handle_results
		return Parser(inner)

	def map(self, f: Callable[[Out], B]) -> 'Parser[B, TokN]':
		"""
		Transform the output of some parser with a function

		This is a particularly useful method, because it allows converting parsers which
		return lexemes (e.g. `Parser.token()`) into parsers that return other thing.

		As an example, here's a parser which parses a number, and returns it as a number.

		>>> parse_num = Parser.token(TrivialTok.NUM)\\
		...                   .map(lambda l: int(l.matched_string))
		>>> parse_num.parse(lex_trivial('3'))
		Ok([(3, [[EOF: '']@(1, 2-2)])])
		"""
		return self.bind(c(Parser.epsilon, f)) #type: ignore

	def fapply(self: 'Parser[Callable[[Out1], Out2], TokN]', arg: 'Parser[Out1, TokN]') -> 'Parser[Out2, TokN]':
		"""
		Apply the function which this returns to the value produced by another parser

		Equivalent to the fapply method of an applicative functor.
		"""
		return self.bind(p(Parser.map, arg)) #type: ignore

	def fapply_r(self: 'Parser[Out, TokN]', arg: 'Parser[Callable[[Out], Out2], TokN]') -> 'Parser[Out2, TokN]':
		"""
		A reversed version of `fapply()`

		Applies the function returned by the argument to the value returned by this.
		"""
		return self.bind(lambda v: arg.map(lambda f: f(v)))

	def seq_ignore(self, subsequent: 'Parser[Any, TokN]') -> 'Parser[Out, TokN]':
		"""
		Parses two things in series, ignoring the output of the second parser

		Example:  Parse a number followed by any letter (ignored)

		>>> parse_num = Parser.token(TrivialTok.NUM)\\
		...                   .map(lambda l: int(l.matched_string))
		>>> parse_numlet = parse_num.seq_ignore(Parser.token(TrivialTok.LTR))
		>>> parse_numlet.parse(lex_trivial('4a'))
		Ok([(4, [[EOF: '']@(1, 3-3)])])
		"""
		return self.map(k).fapply(subsequent) #type:ignore

	def seq_ignore_tok(self, subsequent: TokN) -> 'Parser[Out, TokN]':
		"""
		A shorthand for calling `seq_ignore()` with `Parser.token`

		Example:  Parse a number followed by any letter (ignored)

		>>> parse_num = Parser.token(TrivialTok.NUM)\\
		...                   .map(lambda l: int(l.matched_string))
		>>> parse_numlet = parse_num.seq_ignore_tok(TrivialTok.LTR)
		>>> parse_numlet.parse(lex_trivial('4a'))
		Ok([(4, [[EOF: '']@(1, 3-3)])])
		"""
		return self.seq_ignore(Parser.token(subsequent))

	def or_(self: 'Parser[Out, TokN]', *parsers: 'Parser[Out, TokN]') -> 'Parser[Out, TokN]':
		"""
		Returns a parser which succeeds if this or any arguments succeed

		**Example:**  A parser which parses a letter or a number

		>>> parse_or = Parser.token(TrivialTok.NUM).or_(Parser.token(TrivialTok.LTR))
		>>> parse_or.parse(lex_trivial('a')) #doctest: +ELLIPSIS
		Ok(...)
		>>> parse_or.parse(lex_trivial('1')) #doctest: +ELLIPSIS
		Ok(...)

		Notice that this can produce multiple successes.  A simple example would be a
		parser which parses either a single number or an empty string (epsilon).  When
		faced with an input stream starting with a number, it could either parse that
		number, returning the rest of the input stream, or parse the empty string,
		returning the input unchanged.

		>>> parse_maybe_num = Parser.token(TrivialTok.NUM)\\
		...                   .map(lambda l: int(l.matched_string))\\
		...                   .or_(Parser.epsilon(-1))
		>>> parse_maybe_num.parse(lex_trivial('3')) #doctest: +NORMALIZE_WHITESPACE
		Ok([(3,  [[EOF: '']@(1, 2-2)]),
		    (-1, [[NUM: '3']@(1, 1-1), [EOF: '']@(1, 2-2)])])

		Of course, this can also produce multiple failures as well:

		>>> parse_or.parse(lex_trivial('+'))
		Err([([ADD: '+']@(1, 1-1), NUM), ([ADD: '+']@(1, 1-1), LTR)])
		"""
		all_parsers = (self, *parsers)
		def inner(input: LexStream) -> ParserResult[Out, TokN]:
			successes, failures = partition([p.parse(input) for p in all_parsers])
			if len(successes):
				return Ok([successful_path for success in successes for successful_path in success])
			else:
				return Err([expectation for failure in failures for expectation in failure])
		return Parser(inner)

	def opt(self, fallback: Out) -> 'Parser[Out, TokN]':
		"""
		Parse one or zero of this thing, returning fallback if not parsing it

		We can use this to write a simpler example of parse_maybe_num from the `or_()`
		example:

		>>> parse_maybe_num = Parser.token(TrivialTok.NUM)\\
		...                   .map(lambda l: int(l.matched_string))\\
		...                   .opt(-1)
		>>> parse_maybe_num.parse(lex_trivial('3')) #doctest: +NORMALIZE_WHITESPACE
		Ok([(3,  [[EOF: '']@(1, 2-2)]),
		    (-1, [[NUM: '3']@(1, 1-1), [EOF: '']@(1, 2-2)])])
		"""
		return self.or_(Parser.epsilon(fallback))

	def many(self, combine: Callable[[Out], Callable[[Out2], Out2]], default: Out2) -> 'Parser[Out2, TokN]':
		"""
		Create a new parser which accepts any number of instances of this parser

		The combine argument is a function which joins the result of this parser with the
		result of parsing the 0 or more identical parsers on the right.  Think of this
		like a reduce.

		As an example, here's a parser which parses any number of numbers, and sums them
		all together.

		We start with the single number parser from the `map()` example.

		>>> parse_num = Parser.token(TrivialTok.NUM)\\
		...                   .map(lambda l: int(l.matched_string))

		Then we call `many()` on it.  We also add the EOF parser to the end to force it to
		parse the whole input.

		>>> parse_nums = parse_num.many(lambda num: lambda sum: num + sum, 0)\\
		...                       .seq_ignore_tok(TrivialTok.EOF)

		**Sample Run 1:**  Sum of the numbers 1, 2, and 3.  This produces the operation (1
		+ (2 + (3 + 0))

		>>> parse_nums.parse(lex_trivial('1 2 3'))
		Ok([(6, [])])

		**Sample Run 2:**  If attempting to sum no numbers, we get back the default
		argument, zero in this case.

		>>> parse_nums.parse(lex_trivial(''))
		Ok([(0, [])])
		"""
		return (self.map(combine)
		            .fapply(Parser.lazy(p(self.many, combine, default))) #type: ignore
		            .opt(default))

if __name__ == '__main__':
	import doctest
	doctest.testmod()