package core import ( "errors" "fmt" "log" "math/big" "strconv" "strings" ) type FormatedError interface { Error() string Format() string } type ParsingError struct { Description string Causer *Token Source string Trace []string } func (p ParsingError) Error() string { return p.Description } // Format Print a rich and informative error func (p ParsingError) Format() string { src := []rune(p.Source) b := strings.Builder{} lineNumber := 1 lineBeginning := 0 for i := 0; i < int(p.Causer.Start); i++ { if src[i] == '\n' { lineBeginning = i + 1 lineNumber++ } } lineEnd := len(src) for i := lineBeginning; i < len(src); i++ { if src[i] == '\n' { lineEnd = i break } } descriptor := fmt.Sprintf("%d:%d", lineNumber, int(p.Causer.Start)-lineBeginning+1) b.WriteString(p.Description) b.WriteRune('\n') b.WriteString(descriptor) b.WriteString(" | ") b.WriteString(string(src[lineBeginning:lineEnd])) b.WriteString("\n") b.WriteString(strings.Repeat(" ", len(descriptor))) b.WriteString(" ") for i := lineBeginning; i <= int(p.Causer.Start); i++ { b.WriteRune(' ') } for i := 0; i < len(p.Causer.Lexeme); i++ { b.WriteRune('^') } b.WriteRune('\n') b.WriteRune('\n') for i := len(p.Trace) - 1; i >= 0; i-- { b.WriteString(fmt.Sprintf("[%d] %s\n", i, p.Trace[i])) } return b.String() } type Parser struct { source string trace []string tokens []Token prev *Token curr *Token pos Pos ignoreNewLine bool } func NewParser(source string, trace []string, tokens []Token) *Parser { return &Parser{ source: source, trace: trace, tokens: tokens, pos: 0, } } type Program struct { Block *BlockNode Path string } func (p *Program) String() string { sb := strings.Builder{} sb.WriteString(fmt.Sprintf("=v= program %s =v=\n", p.Path)) sb.WriteString(p.Block.String()) sb.WriteString(fmt.Sprintf("=^= program %s =^=\n", p.Path)) return sb.String() } func (p *Parser) Parse(path string) (*Program, error) { // top level statements statements := make([]Node, 0) // initialize current p.advance() for int(p.pos) < len(p.tokens) && p.curr.Type != TokenEOF { for p.accept(TokenNewLine) { } if p.curr.Type == TokenEOF { break } b, err := p.expression(false) if err != nil { return nil, err } if b != nil { statements = append(statements, b) } } return &Program{ &BlockNode{ statements, 0, p.curr.End, }, path, }, nil } func (p *Parser) accept(tokenType TokenKind) bool { if p.curr == nil { log.Fatal("unexpected current token nil") return false } if p.ignoreNewLine && tokenType != TokenNewLine { for p.curr.Type == TokenNewLine { p.advance() } } if (*p.curr).Type == tokenType { p.advance() return true } return false } func (p *Parser) acceptAll(tokenTypes ...TokenKind) bool { if int(p.pos)+len(tokenTypes) > len(p.tokens) { return false } for i, tokenType := range tokenTypes { if p.tokens[int(p.pos)+i].Type != tokenType { return false } } p.pos += Pos(len(tokenTypes)) return true } func (p *Parser) expect(tokenType TokenKind, reason string) error { if !p.accept(tokenType) { return p.error(fmt.Sprintf("Expected token %s, got %s; %s", tokenType, p.curr.Type, reason), p.curr) } return nil } func (p *Parser) peek() (Token, error) { if p.pos >= Pos(len(p.tokens)) { return Token{}, errors.New("cannot peek beyond tokens") } return p.tokens[p.pos], nil } func (p *Parser) advance() { p.prev = p.curr if p.pos < Pos(len(p.tokens)) { p.curr = &p.tokens[p.pos] } else { p.curr = nil } p.pos++ } func (p *Parser) error(error string, causer *Token) error { return ParsingError{ Description: error, Causer: causer, Source: p.source, Trace: p.trace, } } func (p *Parser) expression(mustBeBlock bool) (Node, error) { if mustBeBlock || p.accept(TokenOpenBrace) { if mustBeBlock { if err := p.expect(TokenOpenBrace, "expected block"); err != nil { return nil, err } } oldIgnoreNewline := p.ignoreNewLine p.ignoreNewLine = false start := p.prev.Start var statements []Node for !p.accept(TokenCloseBrace) { if p.accept(TokenNewLine) { continue } s, err := p.expression(false) if err != nil { return nil, err } statements = append(statements, s) if !p.accept(TokenNewLine) { if err := p.expect(TokenCloseBrace, "blocks must be closed"); err != nil { return nil, err } break } } p.ignoreNewLine = oldIgnoreNewline return &BlockNode{statements, start, p.prev.End}, nil } t := p.curr switch t.Type { case TokenType: p.advance() start := p.prev.Start if err := p.expect(TokenName, "types must have a name"); err != nil { return nil, err } name := p.prev if err := p.expect(TokenAssign, "type aliases must be defined with an assign"); err != nil { return nil, err } sig, err := p.parseSignature() if err != nil { return nil, err } return &AliasNode{ name, sig, start, p.prev.End, }, nil case TokenIf: p.advance() cond, err := p.expression(false) if err != nil { return nil, err } do, err := p.expression(true) if err != nil { return nil, err } var otherwise Node if p.accept(TokenElse) { otherwise, err = p.expression(p.curr.Type != TokenIf) if err != nil { return nil, err } } return &ConditionalNode{ cond, do, otherwise, t.Start, t.End, }, nil case TokenReturn: p.advance() start := p.prev.Start v, err := p.expression(false) if err != nil { return nil, err } return &ReturnNode{ v, start, p.prev.End, }, nil case TokenWhile: p.advance() start := p.prev.Start cond, err := p.expression(false) if err != nil { return nil, err } logic, err := p.expression(true) if err != nil { return nil, err } return &LoopNode{ cond, logic, start, p.prev.End, }, nil case TokenFor: p.advance() start := p.prev.Start counter, err := p.expression(false) if err != nil { return nil, err } if err := p.expect(TokenIn, "for-loops must be for each item in an iterator"); err != nil { return nil, err } iterator, err := p.expression(false) if err != nil { return nil, err } logic, err := p.expression(true) if err != nil { return nil, err } return &ForNode{ counter, iterator, logic, start, p.prev.End, }, nil case TokenInclude: p.advance() start := p.prev.Start if err := p.expect(TokenString, "import requires a path/name to include"); err != nil { return nil, err } return &IncludeNode{ &StringNode{ p.prev.Lexeme[1 : len(p.prev.Lexeme)-1], p.prev.Lexeme, p.prev.Start, p.prev.End, }, start, p.prev.End, }, nil default: s, err := p.binary() if err != nil { return nil, err } if p.accept(TokenDeclare) || p.accept(TokenAssign) { isDeclaration := p.prev.Type == TokenDeclare // possibly assign tuples; not implemented yet v, err := p.expression(false) if err != nil { return nil, err } start, _ := s.Bounds() _, end := v.Bounds() return &AssignNode{ s, v, isDeclaration, start, end, }, nil } return s, nil } } func isBinaryOperator(tokenType TokenKind) bool { switch tokenType { case TokenPlus, TokenMinus, TokenStar, TokenSlash, TokenPercent, TokenPipe, TokenDoubleAmpersand, TokenDoublePipe, TokenEquals, TokenBangEquals, TokenLessThan, TokenLessThanOrEqual, TokenGreaterThan, TokenGreaterThanOrEqual: return true default: return false } } func binaryPrecedence(op TokenKind) int { switch op { case TokenDoubleAmpersand, TokenDoublePipe: return 1 case TokenEquals, TokenBangEquals, TokenLessThan, TokenGreaterThan, TokenLessThanOrEqual, TokenGreaterThanOrEqual: return 2 case TokenPercent: return 3 case TokenPlus, TokenMinus, TokenPipe: return 5 case TokenStar, TokenSlash: return 10 default: panic("unimplemented") } } func tokenToBinaryOperation(tokenType TokenKind) BinaryOperation { switch tokenType { case TokenPlus: return BinaryAddition case TokenMinus: return BinarySubtraction case TokenStar: return BinaryMultiplication case TokenSlash: return BinaryDivision case TokenPercent: return BinaryModulo case TokenPipe: panic("unimplemented bitwise ops") case TokenDoubleAmpersand: return BinaryBooleanAnd case TokenDoublePipe: return BinaryBooleanOr case TokenEquals: return BinaryEquality case TokenBangEquals: return BinaryInequality case TokenLessThan: return BinaryLess case TokenLessThanOrEqual: return BinaryLessEqual case TokenGreaterThan: return BinaryGreater case TokenGreaterThanOrEqual: return BinaryGreaterEqual default: panic("unimplemented") } } func (p *Parser) binary() (Node, error) { t, err := p.chain() if err != nil { return nil, err } ops := NewStack[*Token](128) values := NewStack[Node](256) values.pushItem(t) reduce := func() { r := values.Pop() l := values.Pop() opToken := ops.Pop() op := tokenToBinaryOperation(opToken.Type) start, _ := l.Bounds() _, end := r.Bounds() values.Push(&BinaryNode{ op, l, r, opToken, start, end, }) } for isBinaryOperator(p.curr.Type) { for ops.Current > 0 && binaryPrecedence(p.curr.Type) <= binaryPrecedence(ops.Peek().Type) { reduce() } ops.Push(p.curr) p.advance() v, err := p.chain() if err != nil { return nil, err } values.Push(v) } for ops.Current > 0 { reduce() } return values.Pop(), nil } func (p *Parser) chain() (Node, error) { f, err := p.factor() if err != nil { return nil, err } for { if p.accept(TokenDot) { if err = p.expect(TokenName, "can only access properties by name"); err != nil { return nil, err } name := p.prev f = &AccessNode{ f, p.prev, name.Start, name.End, } if p.curr.Type == TokenOpenParenthesis { args, err := p.parseArgs() if err != nil { return nil, err } f = &InvokeNode{ f, args, name.Start, p.prev.End, } } } else if p.curr.Type == TokenOpenParenthesis { start := p.curr.Start args, err := p.parseArgs() if err != nil { return nil, err } f = &InvokeNode{ f, args, start, p.prev.End, } } else if p.accept(TokenOpenBracket) { start := p.prev.Start index, err := p.expression(false) if err != nil { return nil, err } if err := p.expect(TokenCloseBracket, "opening bracket must be closed"); err != nil { return nil, err } f = &IndexNode{ f, index, start, p.prev.End, } } else { break } } return f, nil } func (p *Parser) factor() (Node, error) { switch (*p.curr).Type { case TokenString: p.advance() return &StringNode{ (*p.prev).Lexeme[1 : len((*p.prev).Lexeme)-1], (*p.prev).Lexeme, p.prev.Start, p.prev.End, }, nil case TokenInteger: p.advance() num, success := new(big.Int).SetString(p.prev.Lexeme, 10) if !success { return nil, p.error(fmt.Sprintf("cannot parse integer base 10: %s", p.prev.Lexeme), p.prev) } return &IntegerNode{ num, p.prev.Start, p.prev.End, }, nil case TokenFloat: p.advance() num, err := strconv.ParseFloat((*p.prev).Lexeme, FloatSize) if err != nil { return nil, p.error(fmt.Sprintf("Error parsing number: %v", err), p.prev) } return &FloatNode{ num, p.prev.Start, p.prev.End, }, nil case TokenHexadecimal: p.advance() start := (*p.prev).Start num, ok := new(big.Int).SetString(p.prev.Lexeme[2:], 16) if !ok { return nil, p.error(fmt.Sprintf("cannot parse hexadecimal: %v", p.prev.Lexeme), p.prev) } return &IntegerNode{ num, start, p.prev.End, }, nil case TokenTrue: p.advance() return &BooleanNode{ true, p.prev.Start, p.prev.End, }, nil case TokenFalse: p.advance() return &BooleanNode{ false, p.prev.Start, p.prev.End, }, nil case TokenNil: p.advance() return &NilNode{}, nil case TokenOpenBracket: p.advance() start := p.prev.Start // TODO: find better solution; current one is messy // Maybe perform better analysis to determine the kind of the list... if p.accept(TokenCloseBracket) { s, err := p.parseSignature() if err != nil { return nil, err } return &ListNode{ []Node{}, s, start, p.prev.End, }, nil } oldIgnoreNewline := p.ignoreNewLine p.ignoreNewLine = true var values []Node for !p.accept(TokenCloseBracket) { if len(values) > 0 { if err := p.expect(TokenComma, "list values must be separated by a comma"); err != nil { return nil, err } } value, err := p.expression(false) if err != nil { return nil, err } values = append(values, value) } p.ignoreNewLine = oldIgnoreNewline return &ListNode{ values, nil, start, p.prev.End, }, nil // unary minus case TokenMinus: p.advance() op := p.prev f, err := p.factor() if err != nil { return nil, err } return &UnaryNode{ UnaryNegate, f, op, op.Start, p.prev.End, }, nil case TokenBang: p.advance() op := p.prev v, err := p.factor() if err != nil { return nil, err } return &UnaryNode{ UnaryNot, v, op, op.Start, p.prev.End, }, nil case TokenName: p.advance() name := (*p.prev).Lexeme start := p.prev.Start nameEnd := p.prev.End if p.curr.Type == TokenOpenParenthesis { args, err := p.parseArgs() if err != nil { return nil, err } return &InvokeNode{ &ReferenceNode{ name, start, nameEnd, }, args, start, p.prev.End, }, nil } return &ReferenceNode{ name, start, nameEnd, }, nil case TokenFunc: p.advance() start := p.prev.Start var name *Token if p.accept(TokenName) { // can be unnamed, but accept name if it is named name = p.prev } params, err := p.parseParams() if err != nil { return nil, err } var yield TypeSignature if p.accept(TokenArrow) { yield, err = p.parseSignature() if err != nil { return nil, err } } logic, err := p.expression(true) if err != nil { return nil, err } names := "*" if name != nil { names = name.Lexeme } fn := &FunctionNode{ names, params, yield, logic, start, p.prev.End, } if name != nil { return &AssignNode{ &ReferenceNode{name.Lexeme, name.Start, name.End}, fn, true, start, p.prev.End, }, nil } return fn, nil case TokenOpenParenthesis: p.advance() start := p.prev.Start // we're inside an object if p.acceptAll(TokenName, TokenColon) { return nil, p.error("objects are not implemented yet (TBD)", p.prev) } v, err := p.expression(false) if err != nil { return nil, err } if !p.accept(TokenComma) { if err := p.expect(TokenCloseParenthesis, "parenthesis must be closed"); err != nil { return nil, err } return v, nil } items := []Node{v} for !p.accept(TokenCloseParenthesis) { i, err := p.expression(false) if err != nil { return nil, err } items = append(items, i) if !p.accept(TokenComma) { if err := p.expect(TokenCloseParenthesis, "tuples must be closed"); err != nil { return nil, err } break } } return &TupleNode{ items, start, p.prev.End, }, nil case TokenBreakpoint: p.advance() return &BreakpointNode{ p.prev.Start, p.prev.End, }, nil case TokenOpenBrace: return p.expression(true) default: return nil, p.error(fmt.Sprintf("invalid factor %s", p.curr), p.curr) } } func (p *Parser) parseArgs() ([]Node, error) { args := make([]Node, 0) if err := p.expect(TokenOpenParenthesis, "arguments must be contained in parenthesis"); err != nil { return nil, err } if !p.accept(TokenCloseParenthesis) { c, err := p.expression(false) if err != nil { return nil, err } args = append(args, c) for !p.accept(TokenCloseParenthesis) { if err := p.expect(TokenComma, "arguments must be separated by comma"); err != nil { return nil, err } c, err = p.expression(false) if err != nil { return nil, err } args = append(args, c) } } return args, nil } // parseParams parse parameters and parentheses func (p *Parser) parseParams() ([]FunctionParameter, error) { if err := p.expect(TokenOpenParenthesis, "parameters must be in parentheses"); err != nil { return nil, err } params := make([]FunctionParameter, 0) if p.accept(TokenName) { name := (*p.prev).Lexeme if err := p.expect(TokenColon, "parameters must have a type separated by a colon"); err != nil { return nil, err } t, err := p.parseSignature() if err != nil { return nil, err } params = append(params, FunctionParameter{ name, t, }) for !p.accept(TokenCloseParenthesis) { if err := p.expect(TokenComma, "parameters must be separated by comma"); err != nil { return nil, err } if err := p.expect(TokenName, "parameters must have a name (cannot have trailing comma)"); err != nil { return nil, err } name = (*p.prev).Lexeme if err := p.expect(TokenColon, "parameters must have a type separated by a colon"); err != nil { return nil, err } t, err := p.parseSignature() if err != nil { return nil, err } params = append(params, FunctionParameter{ name, t, }) } } else { if err := p.expect(TokenCloseParenthesis, "must close parameter list"); err != nil { return nil, err } } return params, nil } func (p *Parser) parseSignature() (TypeSignature, error) { var s TypeSignature if p.accept(TokenOpenParenthesis) { // we're inside an object if p.acceptAll(TokenName, TokenColon) { return nil, p.error("objects are not implemented yet (TBD)", p.prev) } v, err := p.parseSignature() if err != nil { return nil, err } if !p.accept(TokenComma) { if err := p.expect(TokenCloseParenthesis, "parenthesis must be closed"); err != nil { return nil, err } s = v } else { items := []TypeSignature{v} for !p.accept(TokenCloseParenthesis) { i, err := p.parseSignature() if err != nil { return nil, err } items = append(items, i) if !p.accept(TokenComma) { if err := p.expect(TokenCloseParenthesis, "tuples must be closed"); err != nil { return nil, err } break } } s = &TupleSignature{ items, } } } else if p.accept(TokenFunc) { if err := p.expect(TokenOpenParenthesis, "func signature must have parentheses for parameters"); err != nil { return nil, err } var in []TypeSignature for !p.accept(TokenCloseParenthesis) { if len(in) > 0 { if err := p.expect(TokenComma, "parameter types must be separated by a comma"); err != nil { return nil, err } } sig, err := p.parseSignature() if err != nil { return nil, err } in = append(in, sig) } var out TypeSignature var err error if p.accept(TokenArrow) { out, err = p.parseSignature() if err != nil { return nil, err } } else { out = &NilSignature{} } s = &FunctionSignature{ in, out, } } else if p.accept(TokenOpenBracket) { inner, err := p.parseSignature() if err != nil { return nil, err } if err := p.expect(TokenCloseBracket, "list type must be enclosed in brackets"); err != nil { return nil, err } return &ListSignature{inner}, nil } else { if err := p.expect(TokenName, "type must be a name"); err != nil { return nil, err } name := (*p.prev).Lexeme switch name { case "str": s = &StringSignature{} case "int": s = &IntegerSignature{} case "float": s = &FloatSignature{} case "bool": s = &BooleanSignature{} case "any": s = &AnySignature{} default: s = &NamedSignature{ name, } } } if p.accept(TokenPipe) { other, err := p.parseSignature() if err != nil { return nil, err } return &CompositeSignature{ s, other, }, nil } return s, nil }