Organize the module tree a little

compile.py

@@ -2,9 +2,9 @@ from emis_funky_funktions import *
 from typing import Collection, Sequence, TypeAlias
 
 from ir import BUILTIN_SUBSTITUTIONS, Expression, ReplHole, subst_all
-from genir import json_to_ir, PatternParseProblem, BranchTypesDiffer, UndefinedVariable
+from ir.genir import json_to_ir, PatternParseProblem, BranchTypesDiffer, UndefinedVariable
-from types_ import BUILTINS_CONTEXT, UnificationError
+from ir.types_ import BUILTINS_CONTEXT, UnificationError
-from silly_thing import evaluate
+from repl import evaluate
 from opt import optimize_to_fixpoint, all_optimizations
 
 import json

grammar.py

@@ -1,39 +0,0 @@
-from enum import auto, IntEnum
-from typing import Collection, Tuple
-from re import compile, Pattern
-
-class Tok(IntEnum):
-	"""
-	All possible tokens used in the grammar
-	"""
-	Whitespace = auto()
-	OpenCurly = auto()
-	CloseCurly = auto()
-	OpenSquare = auto()
-	CloseSquare = auto()
-	Comma = auto()
-	Colon = auto()
-	String = auto()
-	Number = auto()
-	Eof = auto()
-
-	def __repr__(self):
-		return self._name_
-
-LEX_TABLE: Collection[Tuple[Pattern[str], Tok]] = [
-	(compile(r"[\s\n]+"), Tok.Whitespace),
-	(compile(r"{"), Tok.OpenCurly),
-	(compile(r"}"), Tok.CloseCurly),
-	(compile(r"\["), Tok.OpenSquare),
-	(compile(r"\]"), Tok.CloseSquare),
-	(compile(r","), Tok.Comma),
-	(compile(r":"), Tok.Colon),
-	(compile(r'"[^"]*"'), Tok.String),
-	(compile(r'\d+'), Tok.Number),
-]
-"""
-A mapping of regexs to the tokens the identify
-
-Tokens earlier on in the list should be regarded as higher priority, even if a match lower
-on the list also matches.  All unicode strings should be matched by at least one token.
-"""

ir/__init__.py

@@ -2,10 +2,10 @@ from emis_funky_funktions import *
 
 from typing import Collection, Mapping, Sequence, Tuple, TypeAlias
 from functools import reduce
-from match_tree import MatchTree, MatchException, StructurePath, LeafNode, merge_all_trees, IntNode, EMPTY_STRUCT_PATH, FAIL_NODE
+from ir.match_tree import MatchTree, MatchException, StructurePath, LeafNode, merge_all_trees, IntNode, EMPTY_STRUCT_PATH, FAIL_NODE
-from patterns import Pattern
+from ir.patterns import Pattern
 
-import types_
+from ir import types_
 
 
 Expression: TypeAlias = 'MonoFunc | Application | Int | Variable | Builtin | LetBinding | ReplHole | Switch'

ir/genir.py

@@ -1,11 +1,11 @@
 from emis_funky_funktions import *
 from typing import Sequence, Mapping
 
-from pattern import lex_and_parse_pattern
+from parse.pattern import lex_and_parse_pattern
 from ir import Expression, MonoFunc, Application, Int, Variable, LetBinding, ReplHole
-from patterns import Pattern, NamePattern, IgnorePattern, IntPattern, SPattern
+from ir.patterns import Pattern, NamePattern, IgnorePattern, IntPattern, SPattern
-from match_tree import MatchException
+from ir.match_tree import MatchException
-from types_ import *
+from ir.types_ import *
 
 from functools import reduce
 import json

ir/patterns.py

@@ -2,8 +2,8 @@ from emis_funky_funktions import *
 
 from typing import Collection, Mapping, Sequence, Tuple, TypeAlias
 
-import types_
+import ir.types_
-from match_tree import LeafNode, IntNode, MatchTree, StructurePath, FAIL_NODE, EMPTY_STRUCT_PATH
+from ir.match_tree import LeafNode, IntNode, MatchTree, StructurePath, FAIL_NODE, EMPTY_STRUCT_PATH
 
 Pattern: TypeAlias = 'NamePattern | IntPattern | SPattern | IgnorePattern'
 

main.py

@@ -1,8 +1,8 @@
 from emis_funky_funktions import *
 
-from genir import json_to_ir
+from ir.genir import json_to_ir
-from types_ import BUILTINS_CONTEXT
+from ir.types_ import BUILTINS_CONTEXT
-from silly_thing import repl, repl_expr
+from repl import repl, repl_expr
 
 import json, sys
 

parse/comb_parse.py

@@ -5,7 +5,7 @@ from enum import auto, IntEnum
 from functools import reduce
 from re import compile, Pattern
 
-from lex import Lexeme, tokenize
+from parse.lex import Lexeme, tokenize
 
 from typing import Any, Callable, Collection, Mapping, Sequence, Tuple, TypeAlias
 

parse/lex.py

@@ -99,5 +99,4 @@ if __name__ == '__main__':
 	# print(tokenize(open('sample.cnf').read()))
 	import doctest
 	from re import compile
-	from grammar import Tok, LEX_TABLE
 	doctest.testmod()

parse/pattern.py

@@ -1,9 +1,9 @@
 from emis_funky_funktions import *
 from typing import Collection, Mapping, Sequence, Tuple, TypeAlias
 
-from comb_parse import Parser
+from parse.comb_parse import Parser
-from patterns import Pattern, NamePattern, IgnorePattern, IntPattern, SPattern
+from ir.patterns import Pattern, NamePattern, IgnorePattern, IntPattern, SPattern
-from lex import Lexeme, tokenize
+from parse.lex import Lexeme, tokenize
 
 from enum import auto, IntEnum
 import re

parse2.py

@@ -1,58 +0,0 @@
-from emis_funky_funktions import *
-
-from typing import AbstractSet, FrozenSet, TypeAlias, TypeGuard, TypeVar
-
-Lexeme = TypeVar('Lexeme')
-Token = TypeVar('Token')
-Variable = TypeVar('Variable')
-
-Handle: TypeAlias = Sequence[Variable | Token]
-Production: TypeAlias = Tuple[Variable, Handle[Variable, Token]]
-Grammar: TypeAlias = Sequence[Production[Variable, Token]]
-
-NfaState: TypeAlias = Tuple[int, int]
-Nfa: TypeAlias = Callable[[NfaState, Variable | Token], FrozenSet[NfaState]]
-
-DfaState: TypeAlias = FrozenSet(Tuple[int, int])
-Dfa: TypeAlias =  Callable[[DfaState, Variable | Token], FrozenSet[NfaState]]
-
-def build_nfa(
-	is_var: Callable[[Variable | Token], TypeGuard[Variable]],
-	grammar: Grammar[Variable, Token],
-) -> Nfa[Variable, Token]:
-
-	def epsilon_closure_step(state: NfaState) -> FrozenSet[NfaState]:
-		production_no, symbol_no = state
-		_, production = grammar[production_no]
-		next_symbol = production[symbol_no]
-
-		if is_var(next_symbol):
-			possible_productions: Iterator[NfaState] = ((i, 0) for i, (variable, handle) in enumerate(grammar) if variable == next_symbol)
-			return fset(state, *possible_productions)
-		else:
-			return fset(state,)
-
-	def epsilon_closure(states: FrozenSet[NfaState], previous_states: FrozenSet[NfaState] = fset()) -> FrozenSet[NfaState]:
-		new_states = FSet(new_state for old_state in states for new_state in epsilon_closure_step(old_state)) - previous_states - states
-		if len(new_states) == 0:
-			return states | previous_states
-		else:
-			return epsilon_closure(new_states, states | previous_states)
-
-	def nfa(state: Tuple[int, int], symbol: Variable | Token) -> FrozenSet[NfaState]:
-		production_no, symbol_no = state
-		production = grammar[production_no]
-		next_symbol = production[symbol_no]
-		if next_symbol == symbol:
-			return epsilon_closure(fset((production_no, symbol_no + 1)))
-		else:
-			return fset()
-
-	def dfa(dstate: DfaState, symbol: Variable | Token) -> DfaState:
-		return FSet(
-			new_nstate
-			for nstate in dstate
-			for new_nstate in nfa(nstate, symbol)
-		)
-
-	return nfa

repl.py

@@ -2,8 +2,8 @@ from emis_funky_funktions import *
 from typing import Collection, Sequence, TypeAlias
 
 from ir import Expression, ReplHole, subst_all
-from genir import json_to_ir, PatternParseProblem, BranchTypesDiffer, UndefinedVariable
+from ir.genir import json_to_ir, PatternParseProblem, BranchTypesDiffer, UndefinedVariable
-from types_ import BUILTINS_CONTEXT, UnificationError
+from ir.types_ import BUILTINS_CONTEXT, UnificationError
 
 import json
 from dataclasses import dataclass