anglais/core/parser.go
2026-09-07 17:41:04 +02:00

1242 lines
22 KiB
Go

package core
import (
"errors"
"fmt"
"log"
"math/big"
"strconv"
"strings"
)
type FormatedError interface {
Error() string
Format() string
}
type ParsingError struct {
Description string
Causer *Token
Source string
Trace []string
}
func (p ParsingError) Error() string {
return p.Description
}
// Format Print a rich and informative error
func (p ParsingError) Format() string {
src := []rune(p.Source)
b := strings.Builder{}
lineNumber := 1
lineBeginning := 0
for i := 0; i < int(p.Causer.Start); i++ {
if src[i] == '\n' {
lineBeginning = i + 1
lineNumber++
}
}
lineEnd := len(src)
for i := lineBeginning; i < len(src); i++ {
if src[i] == '\n' {
lineEnd = i
break
}
}
descriptor := fmt.Sprintf("%d:%d", lineNumber, int(p.Causer.Start)-lineBeginning+1)
b.WriteString(p.Description)
b.WriteRune('\n')
b.WriteString(descriptor)
b.WriteString(" | ")
b.WriteString(string(src[lineBeginning:lineEnd]))
b.WriteString("\n")
b.WriteString(strings.Repeat(" ", len(descriptor)))
b.WriteString(" ")
for i := lineBeginning; i <= int(p.Causer.Start); i++ {
b.WriteRune(' ')
}
for i := 0; i < len(p.Causer.Lexeme); i++ {
b.WriteRune('^')
}
b.WriteRune('\n')
b.WriteRune('\n')
for i := len(p.Trace) - 1; i >= 0; i-- {
b.WriteString(fmt.Sprintf("[%d] %s\n", i, p.Trace[i]))
}
return b.String()
}
type Parser struct {
source string
trace []string
tokens []Token
state ParserState
}
type ParserState struct {
prev *Token
curr *Token
pos Pos
ignoreNewLine bool
}
func NewParser(source string, trace []string, tokens []Token) *Parser {
return &Parser{
source: source,
trace: trace,
tokens: tokens,
state: ParserState{
pos: 0,
},
}
}
type Program struct {
Block *BlockNode
Path string
}
func (p *Program) String() string {
sb := strings.Builder{}
sb.WriteString(fmt.Sprintf("=v= program %s =v=\n", p.Path))
sb.WriteString(p.Block.String())
sb.WriteString(fmt.Sprintf("=^= program %s =^=\n", p.Path))
return sb.String()
}
func (p *Parser) Parse(path string) (*Program, error) {
// top level statements
statements := make([]Node, 0)
// initialize current
p.advance()
for int(p.state.pos) < len(p.tokens) && p.state.curr.Kind != TokenEOF {
for p.accept(TokenNewLine) {
}
if p.state.curr.Kind == TokenEOF {
break
}
b, err := p.expression(false)
if err != nil {
return nil, err
}
if b != nil {
statements = append(statements, b)
}
}
return &Program{
&BlockNode{
statements,
0,
p.state.curr.End,
},
path,
}, nil
}
func (p *Parser) accept(tokenType TokenKind) bool {
if p.state.curr == nil {
log.Fatal("unexpected current token nil")
return false
}
if p.state.ignoreNewLine && tokenType != TokenNewLine {
for p.state.curr.Kind == TokenNewLine {
p.advance()
}
}
if (*p.state.curr).Kind == tokenType {
p.advance()
return true
}
return false
}
func (p *Parser) getState() ParserState {
return p.state
}
func (p *Parser) restoreState(state ParserState) {
p.state = state
}
func (p *Parser) acceptSeq(tokenTypes ...TokenKind) bool {
state := p.getState()
for _, t := range tokenTypes {
if !p.accept(t) {
p.restoreState(state)
return false
}
}
return true
}
func (p *Parser) expect(tokenType TokenKind, reason string) error {
if !p.accept(tokenType) {
return p.error(fmt.Sprintf("Expected token %s, got %s; %s", tokenType, p.state.curr.Kind, reason), p.state.curr)
}
return nil
}
func (p *Parser) peek() (Token, error) {
if p.state.pos >= Pos(len(p.tokens)) {
return Token{}, errors.New("cannot peek beyond tokens")
}
return p.tokens[p.state.pos], nil
}
func (p *Parser) advance() {
p.state.prev = p.state.curr
if p.state.pos < Pos(len(p.tokens)) {
p.state.curr = &p.tokens[p.state.pos]
} else {
p.state.curr = nil
}
p.state.pos++
}
func (p *Parser) error(error string, causer *Token) error {
return ParsingError{
Description: error,
Causer: causer,
Source: p.source,
Trace: p.trace,
}
}
func (p *Parser) expression(mustBeBlock bool) (Node, error) {
if mustBeBlock || p.accept(TokenOpenBrace) {
if mustBeBlock {
if err := p.expect(TokenOpenBrace, "expected block"); err != nil {
return nil, err
}
}
oldIgnoreNewline := p.state.ignoreNewLine
p.state.ignoreNewLine = false
start := p.state.prev.Start
var statements []Node
for !p.accept(TokenCloseBrace) {
if p.accept(TokenNewLine) {
continue
}
s, err := p.expression(false)
if err != nil {
return nil, err
}
statements = append(statements, s)
if !p.accept(TokenNewLine) {
if err := p.expect(TokenCloseBrace, "blocks must be closed"); err != nil {
return nil, err
}
break
}
}
p.state.ignoreNewLine = oldIgnoreNewline
return &BlockNode{statements, start, p.state.prev.End}, nil
}
t := p.state.curr
switch t.Kind {
case TokenType:
p.advance()
start := p.state.prev.Start
if err := p.expect(TokenName, "types must have a name"); err != nil {
return nil, err
}
name := p.state.prev
if err := p.expect(TokenAssign, "type aliases must be defined with an assign"); err != nil {
return nil, err
}
sig, err := p.parseSignature()
if err != nil {
return nil, err
}
return &AliasNode{
name,
sig,
start,
p.state.prev.End,
}, nil
case TokenIf:
p.advance()
cond, err := p.expression(false)
if err != nil {
return nil, err
}
do, err := p.expression(true)
if err != nil {
return nil, err
}
var otherwise Node
if p.accept(TokenElse) {
otherwise, err = p.expression(p.state.curr.Kind != TokenIf)
if err != nil {
return nil, err
}
}
return &ConditionalNode{
cond,
do,
otherwise,
t.Start,
t.End,
}, nil
case TokenReturn:
p.advance()
start := p.state.prev.Start
v, err := p.expression(false)
if err != nil {
return nil, err
}
return &ReturnNode{
v,
start,
p.state.prev.End,
}, nil
case TokenWhile:
p.advance()
start := p.state.prev.Start
cond, err := p.expression(false)
if err != nil {
return nil, err
}
logic, err := p.expression(true)
if err != nil {
return nil, err
}
return &LoopNode{
cond,
logic,
start,
p.state.prev.End,
}, nil
case TokenFor:
p.advance()
start := p.state.prev.Start
counter, err := p.expression(false)
if err != nil {
return nil, err
}
if err := p.expect(TokenIn, "for-loops must be for each item in an iterator"); err != nil {
return nil, err
}
iterator, err := p.expression(false)
if err != nil {
return nil, err
}
logic, err := p.expression(true)
if err != nil {
return nil, err
}
return &ForNode{
counter,
iterator,
logic,
start,
p.state.prev.End,
}, nil
case TokenInclude:
p.advance()
start := p.state.prev.Start
if err := p.expect(TokenString, "import requires a path/name to include"); err != nil {
return nil, err
}
return &IncludeNode{
&StringNode{
p.state.prev.Lexeme[1 : len(p.state.prev.Lexeme)-1],
p.state.prev.Lexeme,
p.state.prev.Start,
p.state.prev.End,
},
start,
p.state.prev.End,
}, nil
default:
s, err := p.binary()
if err != nil {
return nil, err
}
if p.accept(TokenDeclare) || p.accept(TokenAssign) {
isDeclaration := p.state.prev.Kind == TokenDeclare
// possibly assign tuples; not implemented yet
v, err := p.expression(false)
if err != nil {
return nil, err
}
start, _ := s.Bounds()
_, end := v.Bounds()
return &AssignNode{
s,
v,
isDeclaration,
start,
end,
}, nil
}
return s, nil
}
}
func isBinaryOperator(tokenType TokenKind) bool {
switch tokenType {
case TokenPlus, TokenMinus, TokenStar, TokenSlash, TokenPercent, TokenPipe, TokenDoubleAmpersand, TokenDoublePipe, TokenEquals, TokenBangEquals, TokenLessThan, TokenLessThanOrEqual, TokenGreaterThan, TokenGreaterThanOrEqual:
return true
default:
return false
}
}
func binaryPrecedence(op TokenKind) int {
switch op {
case TokenDoubleAmpersand, TokenDoublePipe:
return 1
case TokenEquals, TokenBangEquals, TokenLessThan, TokenGreaterThan, TokenLessThanOrEqual, TokenGreaterThanOrEqual:
return 2
case TokenPercent:
return 3
case TokenPlus, TokenMinus, TokenPipe:
return 5
case TokenStar, TokenSlash:
return 10
default:
panic("unimplemented")
}
}
func tokenToBinaryOperation(tokenType TokenKind) BinaryOperation {
switch tokenType {
case TokenPlus:
return BinaryAddition
case TokenMinus:
return BinarySubtraction
case TokenStar:
return BinaryMultiplication
case TokenSlash:
return BinaryDivision
case TokenPercent:
return BinaryModulo
case TokenPipe:
panic("unimplemented bitwise ops")
case TokenDoubleAmpersand:
return BinaryBooleanAnd
case TokenDoublePipe:
return BinaryBooleanOr
case TokenEquals:
return BinaryEquality
case TokenBangEquals:
return BinaryInequality
case TokenLessThan:
return BinaryLess
case TokenLessThanOrEqual:
return BinaryLessEqual
case TokenGreaterThan:
return BinaryGreater
case TokenGreaterThanOrEqual:
return BinaryGreaterEqual
default:
panic("unimplemented")
}
}
func (p *Parser) binary() (Node, error) {
t, err := p.chain()
if err != nil {
return nil, err
}
ops := NewStack[*Token](128)
values := NewStack[Node](256)
values.pushItem(t)
reduce := func() {
r := values.Pop()
l := values.Pop()
opToken := ops.Pop()
op := tokenToBinaryOperation(opToken.Kind)
start, _ := l.Bounds()
_, end := r.Bounds()
values.Push(&BinaryNode{
op,
l,
r,
opToken,
start,
end,
})
}
for isBinaryOperator(p.state.curr.Kind) {
for ops.Current > 0 && binaryPrecedence(p.state.curr.Kind) <= binaryPrecedence(ops.Peek().Kind) {
reduce()
}
ops.Push(p.state.curr)
p.advance()
v, err := p.chain()
if err != nil {
return nil, err
}
values.Push(v)
}
for ops.Current > 0 {
reduce()
}
return values.Pop(), nil
}
func (p *Parser) chain() (Node, error) {
f, err := p.factor()
if err != nil {
return nil, err
}
for {
if p.accept(TokenDot) {
if err = p.expect(TokenName, "can only access properties by name"); err != nil {
return nil, err
}
name := p.state.prev
f = &AccessNode{
f,
p.state.prev,
name.Start,
name.End,
}
if p.state.curr.Kind == TokenOpenParenthesis {
args, err := p.parseArgs()
if err != nil {
return nil, err
}
f = &InvokeNode{
f,
args,
name.Start,
p.state.prev.End,
}
}
} else if p.state.curr.Kind == TokenOpenParenthesis {
start := p.state.curr.Start
args, err := p.parseArgs()
if err != nil {
return nil, err
}
f = &InvokeNode{
f,
args,
start,
p.state.prev.End,
}
} else if p.accept(TokenOpenBracket) {
start := p.state.prev.Start
index, err := p.expression(false)
if err != nil {
return nil, err
}
if err := p.expect(TokenCloseBracket, "opening bracket must be closed"); err != nil {
return nil, err
}
f = &IndexNode{
f,
index,
start,
p.state.prev.End,
}
} else {
break
}
}
return f, nil
}
func (p *Parser) factor() (Node, error) {
switch (*p.state.curr).Kind {
case TokenString:
p.advance()
return &StringNode{
(*p.state.prev).Lexeme[1 : len((*p.state.prev).Lexeme)-1],
(*p.state.prev).Lexeme,
p.state.prev.Start,
p.state.prev.End,
}, nil
case TokenInteger:
p.advance()
num, success := new(big.Int).SetString(p.state.prev.Lexeme, 10)
if !success {
return nil, p.error(fmt.Sprintf("cannot parse integer base 10: %s", p.state.prev.Lexeme), p.state.prev)
}
return &IntegerNode{
num,
p.state.prev.Start,
p.state.prev.End,
}, nil
case TokenFloat:
p.advance()
num, err := strconv.ParseFloat((*p.state.prev).Lexeme, FloatSize)
if err != nil {
return nil, p.error(fmt.Sprintf("Error parsing number: %v", err), p.state.prev)
}
return &FloatNode{
num,
p.state.prev.Start,
p.state.prev.End,
}, nil
case TokenHexadecimal:
p.advance()
start := (*p.state.prev).Start
num, ok := new(big.Int).SetString(p.state.prev.Lexeme[2:], 16)
if !ok {
return nil, p.error(fmt.Sprintf("cannot parse hexadecimal: %v", p.state.prev.Lexeme), p.state.prev)
}
return &IntegerNode{
num,
start,
p.state.prev.End,
}, nil
case TokenTrue:
p.advance()
return &BooleanNode{
true,
p.state.prev.Start,
p.state.prev.End,
}, nil
case TokenFalse:
p.advance()
return &BooleanNode{
false,
p.state.prev.Start,
p.state.prev.End,
}, nil
case TokenNil:
p.advance()
return &NilNode{}, nil
case TokenOpenBracket:
p.advance()
start := p.state.prev.Start
// TODO: find better solution; current one is messy
// Maybe perform better analysis to determine the kind of the list...
if p.accept(TokenCloseBracket) {
s, err := p.parseSignature()
if err != nil {
return nil, err
}
return &ListNode{
[]Node{},
s,
start,
p.state.prev.End,
}, nil
}
oldIgnoreNewline := p.state.ignoreNewLine
p.state.ignoreNewLine = true
var values []Node
for !p.accept(TokenCloseBracket) {
if len(values) > 0 {
if err := p.expect(TokenComma, "list values must be separated by a comma"); err != nil {
return nil, err
}
}
value, err := p.expression(false)
if err != nil {
return nil, err
}
values = append(values, value)
}
p.state.ignoreNewLine = oldIgnoreNewline
return &ListNode{
values,
nil,
start,
p.state.prev.End,
}, nil
// unary minus
case TokenMinus:
p.advance()
op := p.state.prev
f, err := p.factor()
if err != nil {
return nil, err
}
return &UnaryNode{
UnaryNegate,
f,
op,
op.Start,
p.state.prev.End,
}, nil
case TokenBang:
p.advance()
op := p.state.prev
v, err := p.factor()
if err != nil {
return nil, err
}
return &UnaryNode{
UnaryNot,
v,
op,
op.Start,
p.state.prev.End,
}, nil
case TokenName:
p.advance()
name := (*p.state.prev).Lexeme
start := p.state.prev.Start
nameEnd := p.state.prev.End
if p.state.curr.Kind == TokenOpenParenthesis {
args, err := p.parseArgs()
if err != nil {
return nil, err
}
return &InvokeNode{
&ReferenceNode{
name,
start,
nameEnd,
},
args,
start,
p.state.prev.End,
}, nil
}
return &ReferenceNode{
name,
start,
nameEnd,
}, nil
case TokenFunc:
p.advance()
start := p.state.prev.Start
var name *Token
if p.accept(TokenName) { // can be unnamed, but accept name if it is named
name = p.state.prev
}
params, err := p.parseParams()
if err != nil {
return nil, err
}
var yield TypeSignature
if p.accept(TokenArrow) {
yield, err = p.parseSignature()
if err != nil {
return nil, err
}
}
logic, err := p.expression(true)
if err != nil {
return nil, err
}
names := "*"
if name != nil {
names = name.Lexeme
}
fn := &FunctionNode{
names,
params,
yield,
logic,
start,
p.state.prev.End,
}
if name != nil {
return &AssignNode{
&ReferenceNode{name.Lexeme, name.Start, name.End},
fn,
true,
start,
p.state.prev.End,
}, nil
}
return fn, nil
case TokenOpenParenthesis:
p.advance()
start := p.state.prev.Start
oldCare := p.state.ignoreNewLine
p.state.ignoreNewLine = true
p.skipNewLines()
// we're inside an object
key := p.state.curr
if p.acceptSeq(TokenName, TokenColon) {
entries := map[string]Node{}
for len(entries) == 0 || !p.accept(TokenCloseParenthesis) {
if len(entries) != 0 {
p.skipNewLines()
key = p.state.curr
if !p.acceptSeq(TokenName, TokenColon) {
return nil, p.error("expected a record name", key)
}
}
name := key.Lexeme
if _, ok := entries[name]; ok {
return nil, p.error("duplicate key; already defined.", key)
}
if p.accept(TokenComma) || p.accept(TokenCloseParenthesis) {
entries[name] = &ReferenceNode{
name,
key.Start,
key.End,
}
if p.state.prev.Kind == TokenCloseParenthesis {
break
}
continue
} else {
x, err := p.expression(false)
if err != nil {
return nil, err
}
entries[name] = x
}
if !p.accept(TokenComma) {
if err := p.expect(TokenCloseParenthesis, "record must be closed"); err != nil {
return nil, err
}
break
}
}
p.state.ignoreNewLine = oldCare
return &RecordNode{
entries,
start,
p.state.prev.End,
}, nil
}
v, err := p.expression(false)
if err != nil {
return nil, err
}
if !p.accept(TokenComma) {
if err := p.expect(TokenCloseParenthesis, "parenthesis must be closed"); err != nil {
return nil, err
}
p.state.ignoreNewLine = oldCare
return v, nil
}
items := []Node{v}
for !p.accept(TokenCloseParenthesis) {
i, err := p.expression(false)
if err != nil {
return nil, err
}
items = append(items, i)
if !p.accept(TokenComma) {
if err := p.expect(TokenCloseParenthesis, "tuples must be closed"); err != nil {
return nil, err
}
break
}
}
p.state.ignoreNewLine = oldCare
return &TupleNode{
items,
start,
p.state.prev.End,
}, nil
case TokenBreakpoint:
p.advance()
return &BreakpointNode{
p.state.prev.Start,
p.state.prev.End,
}, nil
case TokenOpenBrace:
return p.expression(true)
default:
return nil, p.error(fmt.Sprintf("invalid factor %s", p.state.curr), p.state.curr)
}
}
func (p *Parser) parseArgs() ([]Node, error) {
args := make([]Node, 0)
if err := p.expect(TokenOpenParenthesis, "arguments must be contained in parenthesis"); err != nil {
return nil, err
}
if !p.accept(TokenCloseParenthesis) {
c, err := p.expression(false)
if err != nil {
return nil, err
}
args = append(args, c)
for !p.accept(TokenCloseParenthesis) {
if err := p.expect(TokenComma, "arguments must be separated by comma"); err != nil {
return nil, err
}
c, err = p.expression(false)
if err != nil {
return nil, err
}
args = append(args, c)
}
}
return args, nil
}
// parseParams parse parameters and parentheses
func (p *Parser) parseParams() ([]FunctionParameter, error) {
if err := p.expect(TokenOpenParenthesis, "parameters must be in parentheses"); err != nil {
return nil, err
}
params := make([]FunctionParameter, 0)
if p.accept(TokenName) {
name := (*p.state.prev).Lexeme
if err := p.expect(TokenColon, "parameters must have a type separated by a colon"); err != nil {
return nil, err
}
t, err := p.parseSignature()
if err != nil {
return nil, err
}
params = append(params, FunctionParameter{
name,
t,
})
for !p.accept(TokenCloseParenthesis) {
if err := p.expect(TokenComma, "parameters must be separated by comma"); err != nil {
return nil, err
}
if err := p.expect(TokenName, "parameters must have a name (cannot have trailing comma)"); err != nil {
return nil, err
}
name = (*p.state.prev).Lexeme
if err := p.expect(TokenColon, "parameters must have a type separated by a colon"); err != nil {
return nil, err
}
t, err := p.parseSignature()
if err != nil {
return nil, err
}
params = append(params, FunctionParameter{
name,
t,
})
}
} else {
if err := p.expect(TokenCloseParenthesis, "must close parameter list"); err != nil {
return nil, err
}
}
return params, nil
}
func (p *Parser) parseSignature() (TypeSignature, error) {
var s TypeSignature
if p.accept(TokenOpenParenthesis) {
oldCare := p.state.ignoreNewLine
p.state.ignoreNewLine = true
p.skipNewLines()
// we're inside an object
name := p.state.curr
if p.acceptSeq(TokenName, TokenColon) {
entries := map[string]TypeSignature{}
for len(entries) == 0 || !p.accept(TokenCloseParenthesis) {
if len(entries) != 0 {
p.skipNewLines()
name = p.state.curr
if !p.acceptSeq(TokenName, TokenColon) {
return nil, p.error("expected record member", p.state.curr)
}
}
sig, err := p.parseSignature()
if err != nil {
return nil, err
}
entries[name.Lexeme] = sig
if !p.accept(TokenComma) {
if err := p.expect(TokenCloseParenthesis, "record must be closed"); err != nil {
return nil, err
}
break
}
}
return &RecordSignature{
entries,
}, nil
}
v, err := p.parseSignature()
if err != nil {
return nil, err
}
if !p.accept(TokenComma) {
if err := p.expect(TokenCloseParenthesis, "parenthesis must be closed"); err != nil {
return nil, err
}
s = v
} else {
items := []TypeSignature{v}
for !p.accept(TokenCloseParenthesis) {
i, err := p.parseSignature()
if err != nil {
return nil, err
}
items = append(items, i)
if !p.accept(TokenComma) {
if err := p.expect(TokenCloseParenthesis, "tuples must be closed"); err != nil {
return nil, err
}
break
}
}
s = &TupleSignature{
items,
}
}
p.state.ignoreNewLine = oldCare
} else if p.accept(TokenFunc) {
if err := p.expect(TokenOpenParenthesis, "func signature must have parentheses for parameters"); err != nil {
return nil, err
}
var in []TypeSignature
for !p.accept(TokenCloseParenthesis) {
if len(in) > 0 {
if err := p.expect(TokenComma, "parameter types must be separated by a comma"); err != nil {
return nil, err
}
}
sig, err := p.parseSignature()
if err != nil {
return nil, err
}
in = append(in, sig)
}
var out TypeSignature
var err error
if p.accept(TokenArrow) {
out, err = p.parseSignature()
if err != nil {
return nil, err
}
} else {
out = &NilSignature{}
}
s = &FunctionSignature{
in,
out,
}
} else if p.accept(TokenOpenBracket) {
inner, err := p.parseSignature()
if err != nil {
return nil, err
}
if err := p.expect(TokenCloseBracket, "list type must be enclosed in brackets"); err != nil {
return nil, err
}
return &ListSignature{inner}, nil
} else {
if err := p.expect(TokenName, "type must be a name"); err != nil {
return nil, err
}
name := (*p.state.prev).Lexeme
switch name {
case "str":
s = &StringSignature{}
case "int":
s = &IntegerSignature{}
case "float":
s = &FloatSignature{}
case "bool":
s = &BooleanSignature{}
case "any":
s = &AnySignature{}
default:
s = &NamedSignature{
name,
}
}
}
if p.accept(TokenPipe) {
other, err := p.parseSignature()
if err != nil {
return nil, err
}
return &CompositeSignature{
s,
other,
}, nil
}
return s, nil
}
func (p *Parser) skipNewLines() {
for p.accept(TokenNewLine) {
}
}