package core import ( "errors" "fmt" "log" "math/big" "strconv" "strings" ) type FormatedError interface { Error() string Format() string } type ParsingError struct { Description string Causer *Token Source string Trace []string } func (p ParsingError) Error() string { return p.Description } // Format Print a rich and informative error func (p ParsingError) Format() string { src := []rune(p.Source) b := strings.Builder{} lineNumber := 1 lineBeginning := 0 for i := 0; i < int(p.Causer.Start); i++ { if src[i] == '\n' { lineBeginning = i + 1 lineNumber++ } } lineEnd := len(src) for i := lineBeginning; i < len(src); i++ { if src[i] == '\n' { lineEnd = i break } } descriptor := fmt.Sprintf("%d:%d", lineNumber, int(p.Causer.Start)-lineBeginning+1) b.WriteString(p.Description) b.WriteRune('\n') b.WriteString(descriptor) b.WriteString(" | ") b.WriteString(string(src[lineBeginning:lineEnd])) b.WriteString("\n") b.WriteString(strings.Repeat(" ", len(descriptor))) b.WriteString(" ") for i := lineBeginning; i <= int(p.Causer.Start); i++ { b.WriteRune(' ') } for i := 0; i < len(p.Causer.Lexeme); i++ { b.WriteRune('^') } b.WriteRune('\n') b.WriteRune('\n') for i := len(p.Trace) - 1; i >= 0; i-- { b.WriteString(fmt.Sprintf("[%d] %s\n", i, p.Trace[i])) } return b.String() } type Parser struct { source string trace []string tokens []Token state ParserState } type ParserState struct { prev *Token curr *Token pos Pos ignoreNewLine bool } func NewParser(source string, trace []string, tokens []Token) *Parser { return &Parser{ source: source, trace: trace, tokens: tokens, state: ParserState{ pos: 0, }, } } type Program struct { Block *BlockNode Path string } func (p *Program) String() string { sb := strings.Builder{} sb.WriteString(fmt.Sprintf("=v= program %s =v=\n", p.Path)) sb.WriteString(p.Block.String()) sb.WriteString(fmt.Sprintf("=^= program %s =^=\n", p.Path)) return sb.String() } func (p *Parser) Parse(path string) (*Program, error) { // top level statements statements := make([]Node, 0) // initialize current p.advance() for int(p.state.pos) < len(p.tokens) && p.state.curr.Kind != TokenEOF { for p.accept(TokenNewLine) { } if p.state.curr.Kind == TokenEOF { break } b, err := p.expression(false) if err != nil { return nil, err } if b != nil { statements = append(statements, b) } } return &Program{ &BlockNode{ statements, 0, p.state.curr.End, }, path, }, nil } func (p *Parser) accept(tokenType TokenKind) bool { if p.state.curr == nil { log.Fatal("unexpected current token nil") return false } if p.state.ignoreNewLine && tokenType != TokenNewLine { for p.state.curr.Kind == TokenNewLine { p.advance() } } if (*p.state.curr).Kind == tokenType { p.advance() return true } return false } func (p *Parser) getState() ParserState { return p.state } func (p *Parser) restoreState(state ParserState) { p.state = state } func (p *Parser) acceptSeq(tokenTypes ...TokenKind) bool { state := p.getState() for _, t := range tokenTypes { if !p.accept(t) { p.restoreState(state) return false } } return true } func (p *Parser) expect(tokenType TokenKind, reason string) error { if !p.accept(tokenType) { return p.error(fmt.Sprintf("Expected token %s, got %s; %s", tokenType, p.state.curr.Kind, reason), p.state.curr) } return nil } func (p *Parser) peek() (Token, error) { if p.state.pos >= Pos(len(p.tokens)) { return Token{}, errors.New("cannot peek beyond tokens") } return p.tokens[p.state.pos], nil } func (p *Parser) advance() { p.state.prev = p.state.curr if p.state.pos < Pos(len(p.tokens)) { p.state.curr = &p.tokens[p.state.pos] } else { p.state.curr = nil } p.state.pos++ } func (p *Parser) error(error string, causer *Token) error { return ParsingError{ Description: error, Causer: causer, Source: p.source, Trace: p.trace, } } func (p *Parser) expression(mustBeBlock bool) (Node, error) { if mustBeBlock || p.accept(TokenOpenBrace) { if mustBeBlock { if err := p.expect(TokenOpenBrace, "expected block"); err != nil { return nil, err } } oldIgnoreNewline := p.state.ignoreNewLine p.state.ignoreNewLine = false start := p.state.prev.Start var statements []Node for !p.accept(TokenCloseBrace) { if p.accept(TokenNewLine) { continue } s, err := p.expression(false) if err != nil { return nil, err } statements = append(statements, s) if !p.accept(TokenNewLine) { if err := p.expect(TokenCloseBrace, "blocks must be closed"); err != nil { return nil, err } break } } p.state.ignoreNewLine = oldIgnoreNewline return &BlockNode{statements, start, p.state.prev.End}, nil } t := p.state.curr switch t.Kind { case TokenType: p.advance() start := p.state.prev.Start if err := p.expect(TokenName, "types must have a name"); err != nil { return nil, err } name := p.state.prev if err := p.expect(TokenAssign, "type aliases must be defined with an assign"); err != nil { return nil, err } sig, err := p.parseSignature() if err != nil { return nil, err } return &AliasNode{ name, sig, start, p.state.prev.End, }, nil case TokenIf: p.advance() cond, err := p.expression(false) if err != nil { return nil, err } do, err := p.expression(true) if err != nil { return nil, err } var otherwise Node if p.accept(TokenElse) { otherwise, err = p.expression(p.state.curr.Kind != TokenIf) if err != nil { return nil, err } } return &ConditionalNode{ cond, do, otherwise, t.Start, t.End, }, nil case TokenReturn: p.advance() start := p.state.prev.Start v, err := p.expression(false) if err != nil { return nil, err } return &ReturnNode{ v, start, p.state.prev.End, }, nil case TokenWhile: p.advance() start := p.state.prev.Start cond, err := p.expression(false) if err != nil { return nil, err } logic, err := p.expression(true) if err != nil { return nil, err } return &LoopNode{ cond, logic, start, p.state.prev.End, }, nil case TokenFor: p.advance() start := p.state.prev.Start counter, err := p.expression(false) if err != nil { return nil, err } if err := p.expect(TokenIn, "for-loops must be for each item in an iterator"); err != nil { return nil, err } iterator, err := p.expression(false) if err != nil { return nil, err } logic, err := p.expression(true) if err != nil { return nil, err } return &ForNode{ counter, iterator, logic, start, p.state.prev.End, }, nil case TokenInclude: p.advance() start := p.state.prev.Start if err := p.expect(TokenString, "import requires a path/name to include"); err != nil { return nil, err } return &IncludeNode{ &StringNode{ p.state.prev.Lexeme[1 : len(p.state.prev.Lexeme)-1], p.state.prev.Lexeme, p.state.prev.Start, p.state.prev.End, }, start, p.state.prev.End, }, nil default: s, err := p.binary() if err != nil { return nil, err } if p.accept(TokenDeclare) || p.accept(TokenAssign) { isDeclaration := p.state.prev.Kind == TokenDeclare // possibly assign tuples; not implemented yet v, err := p.expression(false) if err != nil { return nil, err } start, _ := s.Bounds() _, end := v.Bounds() return &AssignNode{ s, v, isDeclaration, start, end, }, nil } return s, nil } } func isBinaryOperator(tokenType TokenKind) bool { switch tokenType { case TokenPlus, TokenMinus, TokenStar, TokenSlash, TokenPercent, TokenPipe, TokenDoubleAmpersand, TokenDoublePipe, TokenEquals, TokenBangEquals, TokenLessThan, TokenLessThanOrEqual, TokenGreaterThan, TokenGreaterThanOrEqual: return true default: return false } } func binaryPrecedence(op TokenKind) int { switch op { case TokenDoubleAmpersand, TokenDoublePipe: return 1 case TokenEquals, TokenBangEquals, TokenLessThan, TokenGreaterThan, TokenLessThanOrEqual, TokenGreaterThanOrEqual: return 2 case TokenPercent: return 3 case TokenPlus, TokenMinus, TokenPipe: return 5 case TokenStar, TokenSlash: return 10 default: panic("unimplemented") } } func tokenToBinaryOperation(tokenType TokenKind) BinaryOperation { switch tokenType { case TokenPlus: return BinaryAddition case TokenMinus: return BinarySubtraction case TokenStar: return BinaryMultiplication case TokenSlash: return BinaryDivision case TokenPercent: return BinaryModulo case TokenPipe: panic("unimplemented bitwise ops") case TokenDoubleAmpersand: return BinaryBooleanAnd case TokenDoublePipe: return BinaryBooleanOr case TokenEquals: return BinaryEquality case TokenBangEquals: return BinaryInequality case TokenLessThan: return BinaryLess case TokenLessThanOrEqual: return BinaryLessEqual case TokenGreaterThan: return BinaryGreater case TokenGreaterThanOrEqual: return BinaryGreaterEqual default: panic("unimplemented") } } func (p *Parser) binary() (Node, error) { t, err := p.chain() if err != nil { return nil, err } ops := NewStack[*Token](128) values := NewStack[Node](256) values.pushItem(t) reduce := func() { r := values.Pop() l := values.Pop() opToken := ops.Pop() op := tokenToBinaryOperation(opToken.Kind) start, _ := l.Bounds() _, end := r.Bounds() values.Push(&BinaryNode{ op, l, r, opToken, start, end, }) } for isBinaryOperator(p.state.curr.Kind) { for ops.Current > 0 && binaryPrecedence(p.state.curr.Kind) <= binaryPrecedence(ops.Peek().Kind) { reduce() } ops.Push(p.state.curr) p.advance() v, err := p.chain() if err != nil { return nil, err } values.Push(v) } for ops.Current > 0 { reduce() } return values.Pop(), nil } func (p *Parser) chain() (Node, error) { f, err := p.factor() if err != nil { return nil, err } for { if p.accept(TokenDot) { if err = p.expect(TokenName, "can only access properties by name"); err != nil { return nil, err } name := p.state.prev f = &AccessNode{ f, p.state.prev, name.Start, name.End, } if p.state.curr.Kind == TokenOpenParenthesis { args, err := p.parseArgs() if err != nil { return nil, err } f = &InvokeNode{ f, args, name.Start, p.state.prev.End, } } } else if p.state.curr.Kind == TokenOpenParenthesis { start := p.state.curr.Start args, err := p.parseArgs() if err != nil { return nil, err } f = &InvokeNode{ f, args, start, p.state.prev.End, } } else if p.accept(TokenOpenBracket) { start := p.state.prev.Start index, err := p.expression(false) if err != nil { return nil, err } if err := p.expect(TokenCloseBracket, "opening bracket must be closed"); err != nil { return nil, err } f = &IndexNode{ f, index, start, p.state.prev.End, } } else { break } } return f, nil } func (p *Parser) factor() (Node, error) { switch (*p.state.curr).Kind { case TokenString: p.advance() return &StringNode{ (*p.state.prev).Lexeme[1 : len((*p.state.prev).Lexeme)-1], (*p.state.prev).Lexeme, p.state.prev.Start, p.state.prev.End, }, nil case TokenInteger: p.advance() num, success := new(big.Int).SetString(p.state.prev.Lexeme, 10) if !success { return nil, p.error(fmt.Sprintf("cannot parse integer base 10: %s", p.state.prev.Lexeme), p.state.prev) } return &IntegerNode{ num, p.state.prev.Start, p.state.prev.End, }, nil case TokenFloat: p.advance() num, err := strconv.ParseFloat((*p.state.prev).Lexeme, FloatSize) if err != nil { return nil, p.error(fmt.Sprintf("Error parsing number: %v", err), p.state.prev) } return &FloatNode{ num, p.state.prev.Start, p.state.prev.End, }, nil case TokenHexadecimal: p.advance() start := (*p.state.prev).Start num, ok := new(big.Int).SetString(p.state.prev.Lexeme[2:], 16) if !ok { return nil, p.error(fmt.Sprintf("cannot parse hexadecimal: %v", p.state.prev.Lexeme), p.state.prev) } return &IntegerNode{ num, start, p.state.prev.End, }, nil case TokenTrue: p.advance() return &BooleanNode{ true, p.state.prev.Start, p.state.prev.End, }, nil case TokenFalse: p.advance() return &BooleanNode{ false, p.state.prev.Start, p.state.prev.End, }, nil case TokenNil: p.advance() return &NilNode{}, nil case TokenOpenBracket: p.advance() start := p.state.prev.Start // TODO: find better solution; current one is messy // Maybe perform better analysis to determine the kind of the list... if p.accept(TokenCloseBracket) { s, err := p.parseSignature() if err != nil { return nil, err } return &ListNode{ []Node{}, s, start, p.state.prev.End, }, nil } oldIgnoreNewline := p.state.ignoreNewLine p.state.ignoreNewLine = true var values []Node for !p.accept(TokenCloseBracket) { if len(values) > 0 { if err := p.expect(TokenComma, "list values must be separated by a comma"); err != nil { return nil, err } } value, err := p.expression(false) if err != nil { return nil, err } values = append(values, value) } p.state.ignoreNewLine = oldIgnoreNewline return &ListNode{ values, nil, start, p.state.prev.End, }, nil // unary minus case TokenMinus: p.advance() op := p.state.prev f, err := p.factor() if err != nil { return nil, err } return &UnaryNode{ UnaryNegate, f, op, op.Start, p.state.prev.End, }, nil case TokenBang: p.advance() op := p.state.prev v, err := p.factor() if err != nil { return nil, err } return &UnaryNode{ UnaryNot, v, op, op.Start, p.state.prev.End, }, nil case TokenName: p.advance() name := (*p.state.prev).Lexeme start := p.state.prev.Start nameEnd := p.state.prev.End if p.state.curr.Kind == TokenOpenParenthesis { args, err := p.parseArgs() if err != nil { return nil, err } return &InvokeNode{ &ReferenceNode{ name, start, nameEnd, }, args, start, p.state.prev.End, }, nil } return &ReferenceNode{ name, start, nameEnd, }, nil case TokenFunc: p.advance() start := p.state.prev.Start var name *Token if p.accept(TokenName) { // can be unnamed, but accept name if it is named name = p.state.prev } params, err := p.parseParams() if err != nil { return nil, err } var yield TypeSignature if p.accept(TokenArrow) { yield, err = p.parseSignature() if err != nil { return nil, err } } logic, err := p.expression(true) if err != nil { return nil, err } names := "*" if name != nil { names = name.Lexeme } fn := &FunctionNode{ names, params, yield, logic, start, p.state.prev.End, } if name != nil { return &AssignNode{ &ReferenceNode{name.Lexeme, name.Start, name.End}, fn, true, start, p.state.prev.End, }, nil } return fn, nil case TokenOpenParenthesis: p.advance() start := p.state.prev.Start oldCare := p.state.ignoreNewLine p.state.ignoreNewLine = true p.skipNewLines() // we're inside an object key := p.state.curr if p.acceptSeq(TokenName, TokenColon) { entries := map[string]Node{} for len(entries) == 0 || !p.accept(TokenCloseParenthesis) { if len(entries) != 0 { p.skipNewLines() key = p.state.curr if !p.acceptSeq(TokenName, TokenColon) { return nil, p.error("expected a record name", key) } } name := key.Lexeme if _, ok := entries[name]; ok { return nil, p.error("duplicate key; already defined.", key) } if p.accept(TokenComma) || p.accept(TokenCloseParenthesis) { entries[name] = &ReferenceNode{ name, key.Start, key.End, } if p.state.prev.Kind == TokenCloseParenthesis { break } continue } else { x, err := p.expression(false) if err != nil { return nil, err } entries[name] = x } if !p.accept(TokenComma) { if err := p.expect(TokenCloseParenthesis, "record must be closed"); err != nil { return nil, err } break } } p.state.ignoreNewLine = oldCare return &RecordNode{ entries, start, p.state.prev.End, }, nil } v, err := p.expression(false) if err != nil { return nil, err } if !p.accept(TokenComma) { if err := p.expect(TokenCloseParenthesis, "parenthesis must be closed"); err != nil { return nil, err } p.state.ignoreNewLine = oldCare return v, nil } items := []Node{v} for !p.accept(TokenCloseParenthesis) { i, err := p.expression(false) if err != nil { return nil, err } items = append(items, i) if !p.accept(TokenComma) { if err := p.expect(TokenCloseParenthesis, "tuples must be closed"); err != nil { return nil, err } break } } p.state.ignoreNewLine = oldCare return &TupleNode{ items, start, p.state.prev.End, }, nil case TokenBreakpoint: p.advance() return &BreakpointNode{ p.state.prev.Start, p.state.prev.End, }, nil case TokenOpenBrace: return p.expression(true) default: return nil, p.error(fmt.Sprintf("invalid factor %s", p.state.curr), p.state.curr) } } func (p *Parser) parseArgs() ([]Node, error) { args := make([]Node, 0) if err := p.expect(TokenOpenParenthesis, "arguments must be contained in parenthesis"); err != nil { return nil, err } if !p.accept(TokenCloseParenthesis) { c, err := p.expression(false) if err != nil { return nil, err } args = append(args, c) for !p.accept(TokenCloseParenthesis) { if err := p.expect(TokenComma, "arguments must be separated by comma"); err != nil { return nil, err } c, err = p.expression(false) if err != nil { return nil, err } args = append(args, c) } } return args, nil } // parseParams parse parameters and parentheses func (p *Parser) parseParams() ([]FunctionParameter, error) { if err := p.expect(TokenOpenParenthesis, "parameters must be in parentheses"); err != nil { return nil, err } params := make([]FunctionParameter, 0) if p.accept(TokenName) { name := (*p.state.prev).Lexeme if err := p.expect(TokenColon, "parameters must have a type separated by a colon"); err != nil { return nil, err } t, err := p.parseSignature() if err != nil { return nil, err } params = append(params, FunctionParameter{ name, t, }) for !p.accept(TokenCloseParenthesis) { if err := p.expect(TokenComma, "parameters must be separated by comma"); err != nil { return nil, err } if err := p.expect(TokenName, "parameters must have a name (cannot have trailing comma)"); err != nil { return nil, err } name = (*p.state.prev).Lexeme if err := p.expect(TokenColon, "parameters must have a type separated by a colon"); err != nil { return nil, err } t, err := p.parseSignature() if err != nil { return nil, err } params = append(params, FunctionParameter{ name, t, }) } } else { if err := p.expect(TokenCloseParenthesis, "must close parameter list"); err != nil { return nil, err } } return params, nil } func (p *Parser) parseSignature() (TypeSignature, error) { var s TypeSignature if p.accept(TokenOpenParenthesis) { oldCare := p.state.ignoreNewLine p.state.ignoreNewLine = true p.skipNewLines() // we're inside an object name := p.state.curr if p.acceptSeq(TokenName, TokenColon) { entries := map[string]TypeSignature{} for len(entries) == 0 || !p.accept(TokenCloseParenthesis) { if len(entries) != 0 { p.skipNewLines() name = p.state.curr if !p.acceptSeq(TokenName, TokenColon) { return nil, p.error("expected record member", p.state.curr) } } sig, err := p.parseSignature() if err != nil { return nil, err } entries[name.Lexeme] = sig if !p.accept(TokenComma) { if err := p.expect(TokenCloseParenthesis, "record must be closed"); err != nil { return nil, err } break } } return &RecordSignature{ entries, }, nil } v, err := p.parseSignature() if err != nil { return nil, err } if !p.accept(TokenComma) { if err := p.expect(TokenCloseParenthesis, "parenthesis must be closed"); err != nil { return nil, err } s = v } else { items := []TypeSignature{v} for !p.accept(TokenCloseParenthesis) { i, err := p.parseSignature() if err != nil { return nil, err } items = append(items, i) if !p.accept(TokenComma) { if err := p.expect(TokenCloseParenthesis, "tuples must be closed"); err != nil { return nil, err } break } } s = &TupleSignature{ items, } } p.state.ignoreNewLine = oldCare } else if p.accept(TokenFunc) { if err := p.expect(TokenOpenParenthesis, "func signature must have parentheses for parameters"); err != nil { return nil, err } var in []TypeSignature for !p.accept(TokenCloseParenthesis) { if len(in) > 0 { if err := p.expect(TokenComma, "parameter types must be separated by a comma"); err != nil { return nil, err } } sig, err := p.parseSignature() if err != nil { return nil, err } in = append(in, sig) } var out TypeSignature var err error if p.accept(TokenArrow) { out, err = p.parseSignature() if err != nil { return nil, err } } else { out = &NilSignature{} } s = &FunctionSignature{ in, out, } } else if p.accept(TokenOpenBracket) { inner, err := p.parseSignature() if err != nil { return nil, err } if err := p.expect(TokenCloseBracket, "list type must be enclosed in brackets"); err != nil { return nil, err } return &ListSignature{inner}, nil } else { if err := p.expect(TokenName, "type must be a name"); err != nil { return nil, err } name := (*p.state.prev).Lexeme switch name { case "str": s = &StringSignature{} case "int": s = &IntegerSignature{} case "float": s = &FloatSignature{} case "bool": s = &BooleanSignature{} case "any": s = &AnySignature{} default: s = &NamedSignature{ name, } } } if p.accept(TokenPipe) { other, err := p.parseSignature() if err != nil { return nil, err } return &CompositeSignature{ s, other, }, nil } return s, nil } func (p *Parser) skipNewLines() { for p.accept(TokenNewLine) { } }