we in era3 boys; overhauled expression system, and variables are now in maps

core/parser.go

@@ -62,7 +62,7 @@ func (p ParsingError) Format() string {
 		b.WriteRune(' ')
 	}
 
-	for i := 0; i < int(p.Causer.Length); i++ {
+	for i := 0; i < len(p.Causer.Lexeme); i++ {
 		b.WriteRune('^')
 	}
 	b.WriteRune('\n')
@@ -76,12 +76,13 @@ func (p ParsingError) Format() string {
 }
 
 type Parser struct {
-	source string
-	trace  []string
-	tokens []Token
-	prev   *Token
-	curr   *Token
-	pos    Pos
+	source        string
+	trace         []string
+	tokens        []Token
+	prev          *Token
+	curr          *Token
+	pos           Pos
+	ignoreNewLine bool
 }
 
 func NewParser(source string, trace []string, tokens []Token) *Parser {
@@ -143,12 +144,19 @@ func (p *Parser) Parse(path string) (*Program, error) {
 			imports = append(imports, Import{
 				p.prev.Lexeme[1 : len(p.prev.Lexeme)-1],
 				start,
-				p.prev.Start + p.prev.Length,
+				p.prev.End,
 			})
 			continue
 		}
 
-		b, err := p.block(true)
+		for p.accept(TokenNewLine) {
+		}
+
+		if p.curr.Type == TokenEOF {
+			break
+		}
+
+		b, err := p.expression(false)
 
 		if err != nil {
 			return nil, err
@@ -164,7 +172,7 @@ func (p *Parser) Parse(path string) (*Program, error) {
 		&BlockNode{
 			statements,
 			0,
-			p.curr.Start + p.curr.Length,
+			p.curr.End,
 		},
 		path,
 	}, nil
@@ -176,6 +184,12 @@ func (p *Parser) accept(tokenType TokenType) bool {
 		return false
 	}
 
+	if p.ignoreNewLine && tokenType != TokenNewLine {
+		for p.curr.Type == TokenNewLine {
+			p.advance()
+		}
+	}
+
 	if (*p.curr).Type == tokenType {
 		p.advance()
 		return true
@@ -220,6 +234,370 @@ func (p *Parser) error(error string, causer *Token) error {
 	}
 }
 
+func (p *Parser) expression(mustBeBlock bool) (Node, error) {
+	if mustBeBlock || p.accept(TokenOpenBrace) {
+		if mustBeBlock {
+			if err := p.expect(TokenOpenBrace, "expected block"); err != nil {
+				return nil, err
+			}
+		}
+
+		oldIgnoreNewline := p.ignoreNewLine
+		p.ignoreNewLine = false
+
+		start := p.prev.Start
+
+		var statements []Node
+		for !p.accept(TokenCloseBrace) {
+			if p.accept(TokenNewLine) {
+				continue
+			}
+
+			s, err := p.expression(false)
+			if err != nil {
+				return nil, err
+			}
+
+			statements = append(statements, s)
+
+			if !p.accept(TokenNewLine) {
+				if err := p.expect(TokenCloseBrace, "blocks must be closed"); err != nil {
+					return nil, err
+				}
+
+				break
+			}
+		}
+
+		p.ignoreNewLine = oldIgnoreNewline
+
+		return &BlockNode{statements, start, p.prev.End}, nil
+	}
+
+	t := p.curr
+	switch t.Type {
+	case TokenIf:
+		p.advance()
+
+		cond, err := p.expression(false)
+		if err != nil {
+			return nil, err
+		}
+
+		do, err := p.expression(true)
+		if err != nil {
+			return nil, err
+		}
+
+		var otherwise Node
+		if p.accept(TokenElse) {
+			otherwise, err = p.expression(p.curr.Type != TokenIf)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		return &ConditionalNode{
+			cond,
+			do,
+			otherwise,
+			t.Start,
+			t.End,
+		}, nil
+
+	case TokenFunc:
+		p.advance()
+		start := p.prev.Start
+
+		var name *Token
+		if p.accept(TokenName) { // can be unnamed, but accept name if it is named
+			name = p.prev
+		}
+
+		params, err := p.parseParams()
+		if err != nil {
+			return nil, err
+		}
+
+		var yield TypeSignature
+		if p.accept(TokenArrow) {
+			yield, err = p.parseSignature()
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		logic, err := p.expression(true)
+		if err != nil {
+			return nil, err
+		}
+
+		names := "*"
+		if name != nil {
+			names = name.Lexeme
+		}
+
+		fn := &FunctionNode{
+			names,
+			params,
+			yield,
+			logic,
+			start,
+			start + p.prev.End,
+		}
+
+		if name != nil {
+			return &AssignNode{
+				&ReferenceNode{name.Lexeme, name.Start, name.End},
+				fn,
+				true,
+				start,
+				start + p.prev.End,
+			}, nil
+		}
+
+		return fn, nil
+
+	case TokenReturn:
+		p.advance()
+		start := p.prev.Start
+
+		v, err := p.expression(false)
+		if err != nil {
+			return nil, err
+		}
+
+		return &ReturnNode{
+			v,
+			start,
+			p.prev.End,
+		}, nil
+
+	case TokenWhile:
+		p.advance()
+		start := p.prev.Start
+
+		cond, err := p.expression(false)
+		if err != nil {
+			return nil, err
+		}
+
+		logic, err := p.expression(true)
+		if err != nil {
+			return nil, err
+		}
+
+		return &LoopNode{
+			cond,
+			logic,
+			start,
+			p.prev.End,
+		}, nil
+
+	default:
+		s, err := p.binary()
+		if err != nil {
+			return nil, err
+		}
+
+		if p.accept(TokenDeclare) || p.accept(TokenAssign) {
+			isDeclaration := p.prev.Type == TokenDeclare
+
+			// possibly assign tuples; not implemented yet
+			v, err := p.expression(false)
+			if err != nil {
+				return nil, err
+			}
+
+			start, _ := s.Bounds()
+			_, end := v.Bounds()
+
+			return &AssignNode{
+				s,
+				v,
+				isDeclaration,
+				start,
+				end,
+			}, nil
+		}
+
+		return s, nil
+	}
+}
+
+func isBinaryOperator(tokenType TokenType) bool {
+	switch tokenType {
+	case TokenPlus, TokenMinus, TokenStar, TokenSlash, TokenPipe, TokenDoubleAmpersand, TokenDoublePipe, TokenEquals, TokenBangEquals, TokenLessThan, TokenLessThanOrEqual, TokenGreaterThan, TokenGreaterThanOrEqual:
+		return true
+	default:
+		return false
+	}
+}
+
+func binaryPrecedence(op TokenType) int {
+	switch op {
+	case TokenDoubleAmpersand, TokenDoublePipe:
+		return 1
+	case TokenEquals, TokenBangEquals, TokenLessThan, TokenGreaterThan, TokenLessThanOrEqual, TokenGreaterThanOrEqual:
+		return 2
+	case TokenPlus, TokenMinus, TokenPipe:
+		return 3
+	case TokenStar, TokenSlash:
+		return 5
+	default:
+		panic("unimplemented")
+	}
+}
+
+func tokenToBinaryOperation(tokenType TokenType) BinaryOperation {
+	switch tokenType {
+	case TokenPlus:
+		return BinaryAddition
+	case TokenMinus:
+		return BinarySubtraction
+	case TokenStar:
+		return BinaryMultiplication
+	case TokenSlash:
+		return BinaryDivision
+	case TokenPipe:
+		panic("unimplemented bitwise ops")
+	case TokenDoubleAmpersand:
+		return BinaryAnd
+	case TokenDoublePipe:
+		return BinaryOr
+
+	case TokenEquals:
+		return BinaryEquality
+	case TokenBangEquals:
+		return BinaryInequality
+	case TokenLessThan:
+		return BinaryLess
+	case TokenLessThanOrEqual:
+		return BinaryLessEqual
+	case TokenGreaterThan:
+		return BinaryGreater
+	case TokenGreaterThanOrEqual:
+		return BinaryGreaterEqual
+
+	default:
+		panic("unimplemented")
+	}
+}
+
+func (p *Parser) binary() (Node, error) {
+	t, err := p.chain()
+	if err != nil {
+		return nil, err
+	}
+
+	ops := NewStack[*Token](128)
+	values := NewStack[Node](256)
+	values.pushItem(t)
+
+	for isBinaryOperator(p.curr.Type) {
+		for ops.Current > 0 && binaryPrecedence(p.curr.Type) <= binaryPrecedence(ops.Peek().Type) {
+			r := values.Pop()
+			l := values.Pop()
+			op := tokenToBinaryOperation(ops.Pop().Type)
+
+			start, _ := l.Bounds()
+			_, end := l.Bounds()
+
+			values.Push(&BinaryNode{
+				op,
+				l,
+				r,
+				start,
+				end,
+			})
+		}
+
+		ops.Push(p.curr)
+		p.advance()
+
+		v, err := p.chain()
+		if err != nil {
+			return nil, err
+		}
+
+		values.Push(v)
+	}
+
+	for ops.Current > 0 {
+		r := values.Pop()
+		l := values.Pop()
+		op := tokenToBinaryOperation(ops.Pop().Type)
+
+		start, _ := l.Bounds()
+		_, end := l.Bounds()
+
+		values.Push(&BinaryNode{
+			op,
+			l,
+			r,
+			start,
+			end,
+		})
+	}
+
+	return values.Pop(), nil
+}
+
+func (p *Parser) chain() (Node, error) {
+	f, err := p.factor()
+	if err != nil {
+		return nil, err
+	}
+
+	for {
+		if p.accept(TokenDot) {
+			if err = p.expect(TokenName, "can only access properties by name"); err != nil {
+				return nil, err
+			}
+			name := p.prev
+
+			f = &AccessNode{
+				f,
+				p.prev.Lexeme,
+				name.Start,
+				name.End,
+			}
+
+			if p.curr.Type == TokenOpenParenthesis {
+				args, err := p.parseArgs()
+				if err != nil {
+					return nil, err
+				}
+
+				f = &InvokeNode{
+					f,
+					args,
+					name.Start,
+					p.prev.End,
+				}
+			}
+		} else if p.curr.Type == TokenOpenParenthesis {
+			start := p.curr.Start
+			args, err := p.parseArgs()
+			if err != nil {
+				return nil, err
+			}
+
+			f = &InvokeNode{
+				f,
+				args,
+
+				start,
+				p.prev.End,
+			}
+		} else {
+			break
+		}
+	}
+
+	return f, nil
+}
+
 func (p *Parser) factor() (Node, error) {
 	switch (*p.curr).Type {
 	case TokenString:
@@ -228,7 +606,7 @@ func (p *Parser) factor() (Node, error) {
 			(*p.prev).Lexeme[1 : len((*p.prev).Lexeme)-1],
 			(*p.prev).Lexeme,
 			p.prev.Start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenInteger:
@@ -242,7 +620,7 @@ func (p *Parser) factor() (Node, error) {
 		return &IntegerNode{
 			num,
 			p.prev.Start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenFloat:
@@ -256,7 +634,7 @@ func (p *Parser) factor() (Node, error) {
 		return &FloatNode{
 			num,
 			p.prev.Start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenHexadecimal:
@@ -270,7 +648,7 @@ func (p *Parser) factor() (Node, error) {
 		return &IntegerNode{
 			num,
 			start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenTrue:
@@ -278,14 +656,14 @@ func (p *Parser) factor() (Node, error) {
 		return &BooleanNode{
 			true,
 			p.prev.Start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 	case TokenFalse:
 		p.advance()
 		return &BooleanNode{
 			false,
 			p.prev.Start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenNil:
@@ -308,10 +686,13 @@ func (p *Parser) factor() (Node, error) {
 				[]Node{},
 				s,
 				start,
-				p.prev.Start + p.prev.Length,
+				p.prev.End,
 			}, nil
 		}
 
+		oldIgnoreNewline := p.ignoreNewLine
+		p.ignoreNewLine = true
+
 		var values []Node
 		for !p.accept(TokenCloseBracket) {
 			if len(values) > 0 {
@@ -328,11 +709,13 @@ func (p *Parser) factor() (Node, error) {
 			values = append(values, value)
 		}
 
+		p.ignoreNewLine = oldIgnoreNewline
+
 		return &ListNode{
 			values,
 			nil,
 			start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	// unary minus
@@ -348,7 +731,7 @@ func (p *Parser) factor() (Node, error) {
 			UnaryNegate,
 			f,
 			first.Start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenBang:
@@ -364,14 +747,14 @@ func (p *Parser) factor() (Node, error) {
 			UnaryNot,
 			v,
 			start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenName:
 		p.advance()
 		name := (*p.prev).Lexeme
 		start := p.prev.Start
-		nameEnd := start + p.prev.Length
+		nameEnd := p.prev.End
 
 		if p.curr.Type == TokenOpenParenthesis {
 			args, err := p.parseArgs()
@@ -379,16 +762,15 @@ func (p *Parser) factor() (Node, error) {
 				return nil, err
 			}
 
-			return &CallNode{
+			return &InvokeNode{
 				&ReferenceNode{
 					name,
 					start,
 					nameEnd,
 				},
 				args,
-				true,
 				start,
-				p.prev.Start + p.prev.Length,
+				p.prev.End,
 			}, nil
 		}
 
@@ -426,7 +808,7 @@ func (p *Parser) factor() (Node, error) {
 			sig,
 			b,
 			start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenOpenParenthesis:
@@ -441,8 +823,16 @@ func (p *Parser) factor() (Node, error) {
 
 		return v, nil
 
+	case TokenBreakpoint:
+		p.advance()
+
+		return &BreakpointNode{
+			p.prev.Start,
+			p.prev.End,
+		}, nil
+
 	default:
-		return nil, p.error("invalid factor", p.curr)
+		return nil, p.error(fmt.Sprintf("invalid factor %s", p.curr), p.curr)
 	}
 }
 
@@ -465,7 +855,7 @@ func (p *Parser) prop() (Node, error) {
 			v,
 			property,
 			start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}
 
 		// if called, also add
@@ -475,12 +865,11 @@ func (p *Parser) prop() (Node, error) {
 				return nil, err
 			}
 
-			v = &CallNode{
+			v = &InvokeNode{
 				v,
 				args,
-				true,
 				start,
-				p.prev.Start + p.prev.Length,
+				p.prev.End,
 			}
 		}
 	}
@@ -512,7 +901,7 @@ func (p *Parser) product() (Node, error) {
 			left,
 			f,
 			start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}
 	}
 
@@ -544,7 +933,7 @@ func (p *Parser) term() (Node, error) {
 			left,
 			pr,
 			start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}
 	}
 
@@ -591,7 +980,7 @@ func (p *Parser) comparison() (Node, error) {
 		left,
 		t,
 		start,
-		p.prev.Start + p.prev.Length,
+		p.prev.End,
 	}, nil
 }
 
@@ -625,7 +1014,7 @@ func (p *Parser) condition() (Node, error) {
 		left,
 		c,
 		start,
-		p.prev.Start + p.prev.Length,
+		p.prev.End,
 	}, nil
 }
 
@@ -664,19 +1053,18 @@ func (p *Parser) statement() (Node, error) {
 			then,
 			otherwise,
 			start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenName:
 		p.advance()
-		start := p.prev.Start
-		name := (*p.prev).Lexeme
+		name := p.prev
 
 		if (*p.curr).Type == TokenDot {
 			var v Node = &ReferenceNode{
-				name,
-				start,
-				p.prev.Start + p.prev.Length,
+				name.Lexeme,
+				name.Start,
+				name.End,
 			}
 
 			// parse chains of prop-getting ( "".split().join().length.round() )
@@ -689,8 +1077,8 @@ func (p *Parser) statement() (Node, error) {
 				v = &AccessNode{
 					v,
 					property,
-					start,
-					p.prev.Start + p.prev.Length,
+					name.Start,
+					p.prev.End,
 				}
 
 				// if called, also add
@@ -700,12 +1088,11 @@ func (p *Parser) statement() (Node, error) {
 						return nil, err
 					}
 
-					v = &CallNode{
+					v = &InvokeNode{
 						v,
 						args,
-						(*p.curr).Type == TokenDot, // if the chain is continued, keep the value.
-						start,
-						p.prev.Start + p.prev.Length,
+						name.Start,
+						p.prev.End,
 					}
 				}
 			}
@@ -717,16 +1104,15 @@ func (p *Parser) statement() (Node, error) {
 				return nil, err
 			}
 
-			return &CallNode{
+			return &InvokeNode{
 				&ReferenceNode{
-					name,
-					start,
-					start + Pos(len(name)),
+					name.Lexeme,
+					name.Start,
+					name.End,
 				},
 				args,
-				false,
-				start,
-				p.prev.Start + p.prev.Length,
+				name.Start,
+				p.prev.End,
 			}, nil
 		} else if p.accept(TokenAssign) || p.accept(TokenDeclare) {
 			isDeclaration := p.prev.Type == TokenDeclare
@@ -735,12 +1121,16 @@ func (p *Parser) statement() (Node, error) {
 				return nil, err
 			}
 
-			return &AssignNode{
-				name,
+			return &AssignNode{ // THIS COULD BE MORE PERMISSIVE; its a new system
+				&ReferenceNode{
+					name.Lexeme,
+					name.Start,
+					name.End,
+				},
 				c,
 				isDeclaration,
-				start,
-				p.prev.Start + p.prev.Length,
+				name.Start,
+				p.prev.End,
 			}, nil
 		}
 
@@ -754,7 +1144,7 @@ func (p *Parser) statement() (Node, error) {
 		if err := p.expect(TokenName, "function must have a name"); err != nil {
 			return nil, err
 		}
-		name := p.prev.Lexeme
+		name := p.prev
 
 		params, err := p.parseParams()
 		if err != nil {
@@ -775,18 +1165,22 @@ func (p *Parser) statement() (Node, error) {
 		}
 
 		return &AssignNode{
-			name,
+			&ReferenceNode{
+				name.Lexeme,
+				name.Start,
+				name.End,
+			},
 			&FunctionNode{
-				name,
+				name.Lexeme,
 				params,
 				yield,
 				b,
 				funcStart,
-				p.prev.Start + p.prev.Length,
+				p.prev.End,
 			},
 			true,
 			funcStart,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenWhile:
@@ -807,7 +1201,7 @@ func (p *Parser) statement() (Node, error) {
 			c,
 			b,
 			start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenReturn:
@@ -822,7 +1216,7 @@ func (p *Parser) statement() (Node, error) {
 		return &ReturnNode{
 			c,
 			start,
-			p.prev.Start + p.prev.Length,
+			p.prev.End,
 		}, nil
 
 	case TokenBreakpoint:
@@ -872,7 +1266,7 @@ func (p *Parser) block(canBeStatement bool) (Node, error) {
 	return &BlockNode{
 		statements,
 		start,
-		p.prev.Start + p.prev.Length,
+		p.prev.End,
 	}, nil
 }
 
@@ -884,7 +1278,7 @@ func (p *Parser) parseArgs() ([]Node, error) {
 	}
 
 	if !p.accept(TokenCloseParenthesis) {
-		c, err := p.condition()
+		c, err := p.expression(false)
 		if err != nil {
 			return nil, err
 		}
@@ -893,7 +1287,7 @@ func (p *Parser) parseArgs() ([]Node, error) {
 			if err := p.expect(TokenComma, "arguments must be separated by comma"); err != nil {
 				return nil, err
 			}
-			c, err = p.condition()
+			c, err = p.expression(false)
 			if err != nil {
 				return nil, err
 			}
@@ -1002,6 +1396,16 @@ func (p *Parser) parseSignature() (TypeSignature, error) {
 			in,
 			out,
 		}
+	} else if p.accept(TokenOpenBracket) {
+		inner, err := p.parseSignature()
+		if err != nil {
+			return nil, err
+		}
+		if err := p.expect(TokenCloseBracket, "list type must be enclosed in brackets"); err != nil {
+			return nil, err
+		}
+
+		return &ListSignature{inner}, nil
 	} else {
 		if err := p.expect(TokenName, "type must be a name"); err != nil {
 			return nil, err