we in era3 boys; overhauled expression system, and variables are now in maps

This commit is contained in:
Neemek 2026-07-09 23:10:49 +02:00
parent bf29e6c3dd
commit 43e450c207
Signed by: neemek
GPG key ID: 84FFE4D7D40AB25E
14 changed files with 945 additions and 452 deletions

View file

@ -8,107 +8,66 @@ import (
type AllTestCase struct {
src string
expectedStack []Value
expectedScope []map[string]Value
}
func GetAllTestCases() map[string]AllTestCase {
return map[string]AllTestCase{
"constant_number": {
"a := 1",
[]Value{
&VariableValue{
"a",
&IntegerValue{new(big.Int).SetInt64(1)},
0,
},
[]Value{&IntegerValue{new(big.Int).SetInt64(1)}},
[]map[string]Value{
{"a": &IntegerValue{new(big.Int).SetInt64(1)}},
},
},
"func": {
"fn sum(a: int, b: int) -> int {\n\treturn a + b\n}\n_ = sum(1, 2)",
[]Value{
&VariableValue{
"sum",
&FunctionValue{
Name: "sum",
Params: []FunctionParameter{
{
"a",
&IntegerSignature{},
},
{
"b",
&IntegerSignature{},
},
},
Chunk: &Chunk{
Bytecode: []Bytecode{
InstructionDescend,
InstructionGetLocal, 0,
InstructionGetLocal, 1,
InstructionAddInt,
InstructionReturn,
InstructionAscend,
},
Constants: []Value{&StringValue{"a"}, &StringValue{"b"}},
},
},
0,
},
},
"fn sum(a: int, b: int) -> int {\n\treturn a + b\n}\nres := sum(1, 2)",
[]Value{&IntegerValue{new(big.Int).SetInt64(3)}},
[]map[string]Value{},
},
"list": {
"a := [1.0, 2.0]",
[]Value{
&VariableValue{
"a",
&ListValue{
[]Value{
&FloatValue{1},
&FloatValue{2},
},
"a := [1.0, 2.0]\n{}",
[]Value{&NilValue{}},
[]map[string]Value{
{"a": &ListValue{
[]Value{
&FloatValue{1},
&FloatValue{2},
},
0,
},
}},
},
},
"constant_list_concat": {
"a := [1, 2] + [3]",
[]Value{
&VariableValue{
"a",
&ListValue{
[]Value{
&IntegerValue{big.NewInt(1)},
&IntegerValue{big.NewInt(2)},
&IntegerValue{big.NewInt(3)},
},
"a := [1, 2] + [3]\n{}",
[]Value{&NilValue{}},
[]map[string]Value{
{"a": &ListValue{
[]Value{
&IntegerValue{big.NewInt(1)},
&IntegerValue{big.NewInt(2)},
&IntegerValue{big.NewInt(3)},
},
0,
},
}},
},
},
"list_concat": {
"a := [1.0, 2.0]\nb := a + [3.0]",
[]Value{
&VariableValue{
"a",
&ListValue{
"a := [1.0, 2.0]\nb := a + [3.0]\nnil",
[]Value{&NilValue{}},
[]map[string]Value{
{
"a": &ListValue{
[]Value{
&FloatValue{1},
&FloatValue{2},
},
},
0,
},
&VariableValue{
"b",
&ListValue{
"b": &ListValue{
[]Value{
&FloatValue{1},
&FloatValue{2},
&FloatValue{3},
},
},
0,
},
},
},
@ -160,7 +119,13 @@ func TestAll(t *testing.T) {
}
t.Log("Comparing stacks")
CompareStacks(t, tc.expectedStack, vm.stack)
// expected scope == nil => we don't care
if tc.expectedScope != nil {
CompareScope(t, tc.expectedScope, vm.scope)
}
})
}
}

View file

@ -18,6 +18,9 @@ type Compiler struct {
source []rune
Warnings []CompilerError
// optimize Whether to attempt some optimization of the emitted bytecode
optimize bool
stack *Stack[LocalVariable]
}
@ -129,6 +132,7 @@ func NewCompiler(source []rune) *Compiler {
nil,
source,
[]CompilerError{},
false,
NewStack[LocalVariable](256),
}
@ -169,10 +173,14 @@ func (c *Compiler) Compile(p *Program) error {
}
}
for _, s := range p.Block.statements {
for i, s := range p.Block.statements {
if err := c.compile(s); err != nil {
return err
}
if i != len(p.Block.statements)-1 {
c.add(InstructionPop)
}
}
c.fileStack.Pop()
@ -205,7 +213,7 @@ func (c *Compiler) compile(tree Node) error {
if len(l.items) == 0 {
c.add(InstructionNewList)
} else if c.isTreeConstant(l) {
} else if c.optimize && c.isTreeConstant(l) {
v, err := c.compute(l)
if err != nil {
panic(err) // this shouldn't happen
@ -234,7 +242,7 @@ func (c *Compiler) compile(tree Node) error {
}
case UnaryNodeType:
if c.isTreeConstant(tree.(*UnaryNode).value) {
if c.optimize && c.isTreeConstant(tree.(*UnaryNode).value) {
v, err := c.compute(tree)
if err != nil {
return err
@ -277,12 +285,21 @@ func (c *Compiler) compile(tree Node) error {
c.add(InstructionNil)
case BlockNodeType:
if len(tree.(*BlockNode).statements) == 0 {
c.add(InstructionNil)
return nil
}
c.addDescend()
for _, n := range tree.(*BlockNode).statements {
for i, n := range tree.(*BlockNode).statements {
err := c.compile(n)
if err != nil {
return err
}
if i != len(tree.(*BlockNode).statements)-1 {
c.add(InstructionPop)
}
}
c.addAscend()
@ -298,7 +315,7 @@ func (c *Compiler) compile(tree Node) error {
return c.error(fmt.Sprintf("conditional requires boolean; cannot use non-boolean type %s", sig), n.condition)
}
if c.isTreeConstant(n.condition) {
if c.optimize && c.isTreeConstant(n.condition) {
v, err := c.compute(n.condition)
if err != nil {
return err
@ -336,13 +353,10 @@ func (c *Compiler) compile(tree Node) error {
}
// we store the position of the jump over the else code here
var jumpOverElse Pos
if n.otherwise != nil {
// this would jump over the else/otherwise block in the code
c.add(InstructionJump)
jumpOverElse = c.ip
c.advance(2)
}
// this would jump over the else/otherwise block in the code
c.add(InstructionJump)
jumpOverElse := c.ip
c.advance(2)
// put the u16 of where to jump if the condition was false
c.putU16(jumpByPos, uint16(c.ip-jumpByPos-2))
@ -352,9 +366,12 @@ func (c *Compiler) compile(tree Node) error {
if err != nil {
return err
}
c.putU16(jumpOverElse, uint16(c.ip-jumpOverElse-2))
} else {
c.add(InstructionNil)
}
c.putU16(jumpOverElse, uint16(c.ip-jumpOverElse-2))
case LoopNodeType:
n := tree.(*LoopNode)
@ -368,7 +385,7 @@ func (c *Compiler) compile(tree Node) error {
}
alwaysLoop := false
if c.isTreeConstant(n.condition) {
if c.optimize && c.isTreeConstant(n.condition) {
v, err := c.compute(n.condition)
if err != nil {
return err
@ -383,6 +400,8 @@ func (c *Compiler) compile(tree Node) error {
}
}
c.add(InstructionNil)
conditionPos := c.ip
jumpValuePos := Pos(0)
if !alwaysLoop {
@ -396,6 +415,8 @@ func (c *Compiler) compile(tree Node) error {
c.advance(2)
}
c.add(InstructionPop)
err = c.compile(n.do)
if err != nil {
return err
@ -412,26 +433,47 @@ func (c *Compiler) compile(tree Node) error {
case AssignNodeType:
n := tree.(*AssignNode)
if n.name == "_" {
// allow non-ish statements
err := c.compile(n.value)
if err != nil {
return err
}
c.add(InstructionPop)
} else {
if n.declare && c.isVarDeclaredHere(n.name) {
return c.error(fmt.Sprintf("%s is already declared in this scope", n.name), n)
switch n.dest.Type() {
case ReferenceNodeType:
d := n.dest.(*ReferenceNode)
if d.name == "_" {
return c.compile(n.value)
}
err := c.addSetVar(n.name, n.value, n.declare)
if err != nil {
if n.declare && c.isVarDeclaredHere(d.name) {
return c.error(fmt.Sprintf("%s is already declared in this scope", d.name), n)
}
if err := c.addSetVar(d.name, n.value, n.declare); err != nil {
return err
}
default:
return c.error(fmt.Sprintf("cannot assign to %s", n.dest.Type()), n.dest)
}
case CallNodeType:
n := tree.(*CallNode)
/*
if n.name == "_" {
// allow non-ish statements
err := c.compile(n.value)
if err != nil {
return err
}
c.add(InstructionPop)
} else {
if n.declare && c.isVarDeclaredHere(n.name) {
return c.error(fmt.Sprintf("%s is already declared in this scope", n.name), n)
}
err := c.addSetVar(n.name, n.value, n.declare)
if err != nil {
return err
}
}
*/
case InvokeNodeType:
n := tree.(*InvokeNode)
s, err := c.deduceSignature(n.source)
if err != nil {
@ -443,10 +485,6 @@ func (c *Compiler) compile(tree Node) error {
return c.error(fmt.Sprintf("cannot call non-function value of type %s", s), n)
}
if !n.keep && f.Out.Type() != TypeNil {
c.warn(fmt.Sprintf("shouldn't void result of function call (output is non-nil %s)", f.Out), n)
}
if len(n.args) != len(f.In) {
return c.error(fmt.Sprintf("wrong argument count: function of signature %s got %d, requires %d", f, len(n.args), len(f.In)), n)
}
@ -478,7 +516,7 @@ func (c *Compiler) compile(tree Node) error {
return c.error(fmt.Sprintf("argument #%d does not have expected type signature: got %s, requires %s", i, sig, f.In[i]), arg)
}
if c.isTreeConstant(arg) {
if c.optimize && c.isTreeConstant(arg) {
v, err := c.compute(arg)
if err != nil {
return err
@ -501,10 +539,6 @@ func (c *Compiler) compile(tree Node) error {
c.add(InstructionCall)
if !n.keep {
c.add(InstructionPop)
}
case FunctionNodeType:
n := tree.(*FunctionNode)
@ -588,7 +622,7 @@ func (c *Compiler) compile(tree Node) error {
}
func (c *Compiler) compileBinary(binary *BinaryNode) error {
if c.isTreeConstant(binary) {
if c.optimize && c.isTreeConstant(binary) {
v, err := c.compute(binary)
if err != nil {
return err
@ -825,8 +859,8 @@ func (c *Compiler) deduceSignature(tree Node) (TypeSignature, error) {
return nil, c.error(fmt.Sprintf("cannot access property from value of type %s", sig), n)
}
case CallNodeType:
n := tree.(*CallNode)
case InvokeNodeType:
n := tree.(*InvokeNode)
sig, err := c.deduceSignature(n.source)
if err != nil {
return nil, err
@ -989,29 +1023,37 @@ func (c *Compiler) affirmReturnSignature(tree Node, sig TypeSignature) error {
case AssignNodeType:
n := tree.(*AssignNode)
if !n.declare {
prev, err := c.getVarSignature(n.name, n)
if err != nil {
return err
switch n.dest.Type() {
case ReferenceNodeType:
name := n.dest.(*ReferenceNode).name
if !n.declare {
prev, err := c.getVarSignature(name, n)
if err != nil {
return err
}
sig, err := c.deduceSignature(n.value)
if err != nil {
return err
}
if !sig.Matches(prev) {
return c.error(fmt.Sprintf("cannot assign value of type %s to variable %s of type %s", sig, name, prev), n.value)
}
return nil
}
sig, err := c.deduceSignature(n.value)
if err != nil {
return err
}
if !sig.Matches(prev) {
return c.error(fmt.Sprintf("cannot assign value of type %s to variable %s of type %s", sig, n.name, prev), n.value)
}
return nil
c.registerVar(name, sig)
default:
return c.error("can neither assign nor declare to", n.dest)
}
sig, err := c.deduceSignature(n.value)
if err != nil {
return err
}
c.registerVar(n.name, sig)
default:
}
@ -1114,13 +1156,13 @@ func (c *Compiler) isTreeConstant(tree Node) bool {
return c.isTreeConstant(tree.(*UnaryNode).value)
case BinaryNodeType:
return c.isTreeConstant(tree.(*BinaryNode).Left) && c.isTreeConstant(tree.(*BinaryNode).Right)
case CallNodeType:
for _, arg := range tree.(*CallNode).args {
case InvokeNodeType:
for _, arg := range tree.(*InvokeNode).args {
if !c.isTreeConstant(arg) {
return false
}
}
return c.isTreeConstant(tree.(*CallNode).source)
return c.isTreeConstant(tree.(*InvokeNode).source)
case BlockNodeType, ConditionalNodeType, LoopNodeType, AssignNodeType, FunctionNodeType,
ReturnNodeType, AccessNodeType, BreakpointNodeType, ReferenceNodeType:
return false
@ -1202,7 +1244,7 @@ func (c *Compiler) compute(tree Node) (Value, error) {
return nil, c.error(fmt.Sprintf("unimplemented unary %s", v.Type()), n)
case *CallNode:
case *InvokeNode:
source, err := c.compute(n.source)
if err != nil {
return nil, err

View file

@ -30,6 +30,7 @@ func BenchmarkNewCompiler(b *testing.B) {
type CompileTestData struct {
program *Program
expectedStack []Value
expectedScope []map[string]Value
}
func GetCompileTestData() map[string]CompileTestData {
@ -40,7 +41,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&StringNode{
"Hello world!",
"\"Hello world!\"",
@ -54,21 +55,21 @@ func GetCompileTestData() map[string]CompileTestData {
},
"",
},
[]Value{
&VariableValue{
"a",
&StringValue{"Hello world!"},
0,
[]Value{&StringValue{"Hello world!"}},
[]map[string]Value{
{
"a": &StringValue{"Hello world!"},
},
},
},
/* these tests are so fucking unmaintainable
"conditional_false": {
&Program{
[]Import{},
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FloatNode{
0,
0, 0,
@ -84,7 +85,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FloatNode{
1,
0, 0,
@ -117,7 +118,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FloatNode{
0,
0, 0,
@ -133,7 +134,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FloatNode{
1,
0, 0,
@ -166,7 +167,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FloatNode{
0,
0, 0,
@ -182,7 +183,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FloatNode{
1,
0, 0,
@ -196,7 +197,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FloatNode{
2,
0, 0,
@ -228,7 +229,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FloatNode{
0,
0, 0,
@ -244,7 +245,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FloatNode{
1,
0, 0,
@ -258,7 +259,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FloatNode{
2,
0, 0,
@ -290,7 +291,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&BinaryNode{
BinaryAddition,
&FloatNode{
@ -325,7 +326,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"sum",
&ReferenceNode{name: "sum"},
&FunctionNode{
"sum",
[]FunctionParameter{
@ -411,7 +412,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FunctionNode{
"a",
[]FunctionParameter{},
@ -419,7 +420,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
"b",
&ReferenceNode{"b", 0, 0},
&FloatNode{
1,
0, 0,
@ -448,7 +449,6 @@ func GetCompileTestData() map[string]CompileTestData {
0, 0,
},
[]Node{},
false,
0, 0,
},
},
@ -488,7 +488,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
statements: []Node{
&AssignNode{
name: "a",
dest: &ReferenceNode{"a", 0, 0},
value: &ListNode{
items: []Node{
&FloatNode{value: 1},
@ -498,7 +498,7 @@ func GetCompileTestData() map[string]CompileTestData {
declare: true,
},
&AssignNode{
name: "b",
dest: &ReferenceNode{"b", 0, 0},
value: &ListNode{
items: []Node{
&StringNode{value: "Hello"},
@ -540,7 +540,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
name: "a",
dest: &ReferenceNode{"a", 0, 0},
value: &UnaryNode{
UnaryNegate,
&FloatNode{
@ -570,7 +570,7 @@ func GetCompileTestData() map[string]CompileTestData {
&BlockNode{
[]Node{
&AssignNode{
name: "a",
dest: &ReferenceNode{"a", 0, 0},
value: &UnaryNode{
UnaryNot,
&BooleanNode{
@ -593,6 +593,7 @@ func GetCompileTestData() map[string]CompileTestData {
},
},
},
*/
}
}
@ -642,6 +643,7 @@ func TestCompile(t *testing.T) {
t.Log("Executed bytecode")
CompareStacks(t, testCase.expectedStack, vm.stack)
CompareScope(t, testCase.expectedScope, vm.scope)
})
}
}
@ -691,7 +693,7 @@ func TestCompiler_CleanStack(t *testing.T) {
}
// make sure stack has only assigned values
for i := 0; i < int(vm.stack.Current); i++ {
for i := 1; i < int(vm.stack.Current); i++ {
v := vm.stack.items[i]
if v == nil || v.Type() != VariableValueType {

View file

@ -9,13 +9,13 @@ import (
type Token struct {
Type TokenType
Start Pos
Length Pos
End Pos
Line Pos
Lexeme string
}
func (t Token) String() string {
return fmt.Sprintf("token %s, '%s' %d -> %d, line %d", t.Type.String(), t.Lexeme, t.Start, t.Length, t.Line)
return fmt.Sprintf("token %s, '%s' %d -> %d, line %d", t.Type.String(), t.Lexeme, t.Start, t.End, t.Line)
}
type TokenType uint64
@ -71,6 +71,7 @@ const (
TokenPipe
TokenDoublePipe
TokenNewLine
TokenBreakpoint
TokenEOF
TokenError
@ -168,6 +169,8 @@ func (t TokenType) String() string {
return "hexadecimal"
case TokenArrow:
return "arrow"
case TokenNewLine:
return "newline"
}
panic("UNDEFINED TOKENTYPE STRING CONVERSION")
@ -212,6 +215,8 @@ func (l *Lexer) NextToken() (Token, error) {
l.advance()
switch c {
case '\n':
return l.makeToken(TokenNewLine), nil
case '+':
return l.makeToken(TokenPlus), nil
case '-':
@ -395,11 +400,11 @@ func (l *Lexer) NextToken() (Token, error) {
}
}
func NewToken(t TokenType, start Pos, length Pos, line Pos, lexeme string) Token {
func NewToken(t TokenType, start Pos, end Pos, line Pos, lexeme string) Token {
return Token{
Type: t,
Start: start,
Length: length,
End: end,
Line: line,
Lexeme: lexeme,
}
@ -421,7 +426,7 @@ func (l *Lexer) Tokenize() ([]Token, error) {
}
func (l *Lexer) makeToken(t TokenType) Token {
return NewToken(t, l.start, l.current-l.start, l.line, string(l.src[l.start:l.current]))
return NewToken(t, l.start, l.current, l.line, string(l.src[l.start:l.current]))
}
func (l *Lexer) peek() rune {
@ -470,7 +475,7 @@ func (l *Lexer) isAtEnd() bool {
}
func (l *Lexer) skipWhitespace() {
for !l.isAtEnd() && unicode.IsSpace(l.peek()) {
for !l.isAtEnd() && unicode.IsSpace(l.peek()) && l.peek() != '\n' {
l.advance()
}
}

View file

@ -37,17 +37,17 @@ func GetLexerTestData() map[string]LexerTestData {
"if_statement(10)": {
"if a >= 200 {\n write(\"Hello world!\")\n}",
[]TokenType{
TokenIf, TokenName, TokenGreaterThanOrEqual, TokenInteger, TokenOpenBrace,
TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenCloseBrace,
TokenEOF,
TokenIf, TokenName, TokenGreaterThanOrEqual, TokenInteger, TokenOpenBrace, TokenNewLine,
TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenNewLine,
TokenCloseBrace, TokenEOF,
},
},
"if_else_statement(20)": {
"if 23 * 2/3 > 32 {\n write(\"It is larger!\")\n} else {\n write(\"It is lower!\")\n}",
[]TokenType{
TokenIf, TokenInteger, TokenStar, TokenInteger, TokenSlash, TokenInteger, TokenGreaterThan, TokenInteger, TokenOpenBrace,
TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenCloseBrace,
TokenElse, TokenOpenBrace, TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenCloseBrace,
TokenIf, TokenInteger, TokenStar, TokenInteger, TokenSlash, TokenInteger, TokenGreaterThan, TokenInteger, TokenOpenBrace, TokenNewLine,
TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenNewLine, TokenCloseBrace,
TokenElse, TokenOpenBrace, TokenNewLine, TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenNewLine, TokenCloseBrace,
TokenEOF,
},
},
@ -77,7 +77,7 @@ func GetLexerTestData() map[string]LexerTestData {
},
"space_before_string": {
"\n \"\"",
[]TokenType{TokenString, TokenEOF},
[]TokenType{TokenNewLine, TokenString, TokenEOF},
},
"write_call": {
"write(\"Hello world\")",
@ -93,9 +93,9 @@ func GetLexerTestData() map[string]LexerTestData {
"3assignments_1condition": {
"a = 8 * 32\nb = a > 256\nc = a <= 256\n!b == c",
[]TokenType{
TokenName, TokenAssign, TokenInteger, TokenStar, TokenInteger,
TokenName, TokenAssign, TokenName, TokenGreaterThan, TokenInteger,
TokenName, TokenAssign, TokenName, TokenLessThanOrEqual, TokenInteger,
TokenName, TokenAssign, TokenInteger, TokenStar, TokenInteger, TokenNewLine,
TokenName, TokenAssign, TokenName, TokenGreaterThan, TokenInteger, TokenNewLine,
TokenName, TokenAssign, TokenName, TokenLessThanOrEqual, TokenInteger, TokenNewLine,
TokenBang, TokenName, TokenEquals, TokenName, TokenEOF,
},
},
@ -103,14 +103,14 @@ func GetLexerTestData() map[string]LexerTestData {
"fn sum(a, b) {\n return a + b\n}",
[]TokenType{
TokenFunc, TokenName, TokenOpenParenthesis, TokenName, TokenComma, TokenName, TokenCloseParenthesis,
TokenOpenBrace, TokenReturn, TokenName, TokenPlus, TokenName, TokenCloseBrace,
TokenOpenBrace, TokenNewLine, TokenReturn, TokenName, TokenPlus, TokenName, TokenNewLine, TokenCloseBrace,
},
},
"while_loop": {
"while a < 5 {\n a = a + 1\n}",
[]TokenType{
TokenWhile, TokenName, TokenLessThan, TokenInteger, TokenOpenBrace,
TokenName, TokenAssign, TokenName, TokenPlus, TokenInteger, TokenCloseBrace,
TokenWhile, TokenName, TokenLessThan, TokenInteger, TokenOpenBrace, TokenNewLine,
TokenName, TokenAssign, TokenName, TokenPlus, TokenInteger, TokenNewLine, TokenCloseBrace, TokenEOF,
},
},
"lambda": {
@ -119,7 +119,7 @@ func GetLexerTestData() map[string]LexerTestData {
"}",
[]TokenType{
TokenName, TokenDeclare, TokenFunc, TokenOpenParenthesis, TokenName, TokenComma, TokenName, TokenCloseParenthesis,
TokenOpenBrace, TokenReturn, TokenName, TokenPlus, TokenName, TokenCloseBrace,
TokenOpenBrace, TokenNewLine, TokenReturn, TokenName, TokenPlus, TokenName, TokenNewLine, TokenCloseBrace,
},
},
"list": {

View file

@ -28,12 +28,14 @@ const (
BooleanNodeType
NilNodeType
ListNodeType
TupleNodeType
BinaryNodeType
UnaryNodeType
BlockNodeType
ConditionalNodeType
LoopNodeType
AssignNodeType
InvokeNodeType
CallNodeType
FunctionNodeType
ReturnNodeType
@ -65,7 +67,7 @@ func (n NodeType) String() string {
return "Loop"
case AssignNodeType:
return "Assign"
case CallNodeType:
case InvokeNodeType:
return "Call"
case FunctionNodeType:
return "Function"
@ -73,12 +75,16 @@ func (n NodeType) String() string {
return "Return"
case ListNodeType:
return "List"
case TupleNodeType:
return "Tuple"
case AccessNodeType:
return "Access"
case BreakpointNodeType:
return "Breakpoint"
case UnaryNodeType:
return "Unary"
case CallNodeType:
return "Call"
}
return "Invalid Node Type"
}
@ -192,6 +198,37 @@ func (n ListNode) Bounds() (Pos, Pos) {
return n.start, n.end
}
type TupleNode struct {
items []Node
start Pos
end Pos
}
func (n TupleNode) Type() NodeType {
return TupleNodeType
}
func (n TupleNode) String() string {
sb := strings.Builder{}
sb.WriteString("(")
for i, item := range n.items {
if i > 0 {
sb.WriteString(", ")
}
sb.WriteString(item.String())
}
sb.WriteString(")")
return sb.String()
}
func (n TupleNode) Bounds() (Pos, Pos) {
return n.start, n.end
}
type AccessNode struct {
source Node
property string
@ -281,14 +318,14 @@ func (n BinaryOperation) Symbol() string {
return "<"
case BinaryGreater:
return ">"
case BinaryAnd:
return "&&"
case BinaryOr:
return "||"
case BinaryLessEqual:
return "<="
case BinaryGreaterEqual:
return ">="
case BinaryAnd:
return "&&"
case BinaryOr:
return "||"
}
panic("unsupported binary operation to symbol conversion for " + n.String())
@ -479,7 +516,7 @@ func (n LoopNode) Bounds() (Pos, Pos) {
// AssignNode assignment
type AssignNode struct {
name string
dest Node
value Node
declare bool
@ -492,18 +529,39 @@ func (n AssignNode) Type() NodeType {
}
func (n AssignNode) String() string {
return fmt.Sprintf("set %s to %s", n.name, n.value)
return fmt.Sprintf("set %s to %s", n.dest, n.value)
}
func (n AssignNode) Bounds() (Pos, Pos) {
return n.start, n.end
}
// CallNode function call
type CallNode struct {
// InvokeNode function call
type InvokeNode struct {
source Node
args []Node
keep bool
start Pos
end Pos
}
func (n InvokeNode) Type() NodeType {
return InvokeNodeType
}
func (n InvokeNode) String() string {
return fmt.Sprintf("invoke %s with args (%s)", n.source.String(), n.args)
}
func (n InvokeNode) Bounds() (Pos, Pos) {
return n.start, n.end
}
// CallNode call a function of a value
type CallNode struct {
source Node
name Token
args []Node
start Pos
end Pos
@ -514,7 +572,7 @@ func (n CallNode) Type() NodeType {
}
func (n CallNode) String() string {
return fmt.Sprintf("call %s with args (%s)", n.source.String(), n.args)
return fmt.Sprintf("call %s on %s with args (%s)", n.name, n.source.String(), n.args)
}
func (n CallNode) Bounds() (Pos, Pos) {

View file

@ -62,7 +62,7 @@ func (p ParsingError) Format() string {
b.WriteRune(' ')
}
for i := 0; i < int(p.Causer.Length); i++ {
for i := 0; i < len(p.Causer.Lexeme); i++ {
b.WriteRune('^')
}
b.WriteRune('\n')
@ -76,12 +76,13 @@ func (p ParsingError) Format() string {
}
type Parser struct {
source string
trace []string
tokens []Token
prev *Token
curr *Token
pos Pos
source string
trace []string
tokens []Token
prev *Token
curr *Token
pos Pos
ignoreNewLine bool
}
func NewParser(source string, trace []string, tokens []Token) *Parser {
@ -143,12 +144,19 @@ func (p *Parser) Parse(path string) (*Program, error) {
imports = append(imports, Import{
p.prev.Lexeme[1 : len(p.prev.Lexeme)-1],
start,
p.prev.Start + p.prev.Length,
p.prev.End,
})
continue
}
b, err := p.block(true)
for p.accept(TokenNewLine) {
}
if p.curr.Type == TokenEOF {
break
}
b, err := p.expression(false)
if err != nil {
return nil, err
@ -164,7 +172,7 @@ func (p *Parser) Parse(path string) (*Program, error) {
&BlockNode{
statements,
0,
p.curr.Start + p.curr.Length,
p.curr.End,
},
path,
}, nil
@ -176,6 +184,12 @@ func (p *Parser) accept(tokenType TokenType) bool {
return false
}
if p.ignoreNewLine && tokenType != TokenNewLine {
for p.curr.Type == TokenNewLine {
p.advance()
}
}
if (*p.curr).Type == tokenType {
p.advance()
return true
@ -220,6 +234,370 @@ func (p *Parser) error(error string, causer *Token) error {
}
}
func (p *Parser) expression(mustBeBlock bool) (Node, error) {
if mustBeBlock || p.accept(TokenOpenBrace) {
if mustBeBlock {
if err := p.expect(TokenOpenBrace, "expected block"); err != nil {
return nil, err
}
}
oldIgnoreNewline := p.ignoreNewLine
p.ignoreNewLine = false
start := p.prev.Start
var statements []Node
for !p.accept(TokenCloseBrace) {
if p.accept(TokenNewLine) {
continue
}
s, err := p.expression(false)
if err != nil {
return nil, err
}
statements = append(statements, s)
if !p.accept(TokenNewLine) {
if err := p.expect(TokenCloseBrace, "blocks must be closed"); err != nil {
return nil, err
}
break
}
}
p.ignoreNewLine = oldIgnoreNewline
return &BlockNode{statements, start, p.prev.End}, nil
}
t := p.curr
switch t.Type {
case TokenIf:
p.advance()
cond, err := p.expression(false)
if err != nil {
return nil, err
}
do, err := p.expression(true)
if err != nil {
return nil, err
}
var otherwise Node
if p.accept(TokenElse) {
otherwise, err = p.expression(p.curr.Type != TokenIf)
if err != nil {
return nil, err
}
}
return &ConditionalNode{
cond,
do,
otherwise,
t.Start,
t.End,
}, nil
case TokenFunc:
p.advance()
start := p.prev.Start
var name *Token
if p.accept(TokenName) { // can be unnamed, but accept name if it is named
name = p.prev
}
params, err := p.parseParams()
if err != nil {
return nil, err
}
var yield TypeSignature
if p.accept(TokenArrow) {
yield, err = p.parseSignature()
if err != nil {
return nil, err
}
}
logic, err := p.expression(true)
if err != nil {
return nil, err
}
names := "*"
if name != nil {
names = name.Lexeme
}
fn := &FunctionNode{
names,
params,
yield,
logic,
start,
start + p.prev.End,
}
if name != nil {
return &AssignNode{
&ReferenceNode{name.Lexeme, name.Start, name.End},
fn,
true,
start,
start + p.prev.End,
}, nil
}
return fn, nil
case TokenReturn:
p.advance()
start := p.prev.Start
v, err := p.expression(false)
if err != nil {
return nil, err
}
return &ReturnNode{
v,
start,
p.prev.End,
}, nil
case TokenWhile:
p.advance()
start := p.prev.Start
cond, err := p.expression(false)
if err != nil {
return nil, err
}
logic, err := p.expression(true)
if err != nil {
return nil, err
}
return &LoopNode{
cond,
logic,
start,
p.prev.End,
}, nil
default:
s, err := p.binary()
if err != nil {
return nil, err
}
if p.accept(TokenDeclare) || p.accept(TokenAssign) {
isDeclaration := p.prev.Type == TokenDeclare
// possibly assign tuples; not implemented yet
v, err := p.expression(false)
if err != nil {
return nil, err
}
start, _ := s.Bounds()
_, end := v.Bounds()
return &AssignNode{
s,
v,
isDeclaration,
start,
end,
}, nil
}
return s, nil
}
}
func isBinaryOperator(tokenType TokenType) bool {
switch tokenType {
case TokenPlus, TokenMinus, TokenStar, TokenSlash, TokenPipe, TokenDoubleAmpersand, TokenDoublePipe, TokenEquals, TokenBangEquals, TokenLessThan, TokenLessThanOrEqual, TokenGreaterThan, TokenGreaterThanOrEqual:
return true
default:
return false
}
}
func binaryPrecedence(op TokenType) int {
switch op {
case TokenDoubleAmpersand, TokenDoublePipe:
return 1
case TokenEquals, TokenBangEquals, TokenLessThan, TokenGreaterThan, TokenLessThanOrEqual, TokenGreaterThanOrEqual:
return 2
case TokenPlus, TokenMinus, TokenPipe:
return 3
case TokenStar, TokenSlash:
return 5
default:
panic("unimplemented")
}
}
func tokenToBinaryOperation(tokenType TokenType) BinaryOperation {
switch tokenType {
case TokenPlus:
return BinaryAddition
case TokenMinus:
return BinarySubtraction
case TokenStar:
return BinaryMultiplication
case TokenSlash:
return BinaryDivision
case TokenPipe:
panic("unimplemented bitwise ops")
case TokenDoubleAmpersand:
return BinaryAnd
case TokenDoublePipe:
return BinaryOr
case TokenEquals:
return BinaryEquality
case TokenBangEquals:
return BinaryInequality
case TokenLessThan:
return BinaryLess
case TokenLessThanOrEqual:
return BinaryLessEqual
case TokenGreaterThan:
return BinaryGreater
case TokenGreaterThanOrEqual:
return BinaryGreaterEqual
default:
panic("unimplemented")
}
}
func (p *Parser) binary() (Node, error) {
t, err := p.chain()
if err != nil {
return nil, err
}
ops := NewStack[*Token](128)
values := NewStack[Node](256)
values.pushItem(t)
for isBinaryOperator(p.curr.Type) {
for ops.Current > 0 && binaryPrecedence(p.curr.Type) <= binaryPrecedence(ops.Peek().Type) {
r := values.Pop()
l := values.Pop()
op := tokenToBinaryOperation(ops.Pop().Type)
start, _ := l.Bounds()
_, end := l.Bounds()
values.Push(&BinaryNode{
op,
l,
r,
start,
end,
})
}
ops.Push(p.curr)
p.advance()
v, err := p.chain()
if err != nil {
return nil, err
}
values.Push(v)
}
for ops.Current > 0 {
r := values.Pop()
l := values.Pop()
op := tokenToBinaryOperation(ops.Pop().Type)
start, _ := l.Bounds()
_, end := l.Bounds()
values.Push(&BinaryNode{
op,
l,
r,
start,
end,
})
}
return values.Pop(), nil
}
func (p *Parser) chain() (Node, error) {
f, err := p.factor()
if err != nil {
return nil, err
}
for {
if p.accept(TokenDot) {
if err = p.expect(TokenName, "can only access properties by name"); err != nil {
return nil, err
}
name := p.prev
f = &AccessNode{
f,
p.prev.Lexeme,
name.Start,
name.End,
}
if p.curr.Type == TokenOpenParenthesis {
args, err := p.parseArgs()
if err != nil {
return nil, err
}
f = &InvokeNode{
f,
args,
name.Start,
p.prev.End,
}
}
} else if p.curr.Type == TokenOpenParenthesis {
start := p.curr.Start
args, err := p.parseArgs()
if err != nil {
return nil, err
}
f = &InvokeNode{
f,
args,
start,
p.prev.End,
}
} else {
break
}
}
return f, nil
}
func (p *Parser) factor() (Node, error) {
switch (*p.curr).Type {
case TokenString:
@ -228,7 +606,7 @@ func (p *Parser) factor() (Node, error) {
(*p.prev).Lexeme[1 : len((*p.prev).Lexeme)-1],
(*p.prev).Lexeme,
p.prev.Start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenInteger:
@ -242,7 +620,7 @@ func (p *Parser) factor() (Node, error) {
return &IntegerNode{
num,
p.prev.Start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenFloat:
@ -256,7 +634,7 @@ func (p *Parser) factor() (Node, error) {
return &FloatNode{
num,
p.prev.Start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenHexadecimal:
@ -270,7 +648,7 @@ func (p *Parser) factor() (Node, error) {
return &IntegerNode{
num,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenTrue:
@ -278,14 +656,14 @@ func (p *Parser) factor() (Node, error) {
return &BooleanNode{
true,
p.prev.Start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenFalse:
p.advance()
return &BooleanNode{
false,
p.prev.Start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenNil:
@ -308,10 +686,13 @@ func (p *Parser) factor() (Node, error) {
[]Node{},
s,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
}
oldIgnoreNewline := p.ignoreNewLine
p.ignoreNewLine = true
var values []Node
for !p.accept(TokenCloseBracket) {
if len(values) > 0 {
@ -328,11 +709,13 @@ func (p *Parser) factor() (Node, error) {
values = append(values, value)
}
p.ignoreNewLine = oldIgnoreNewline
return &ListNode{
values,
nil,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
// unary minus
@ -348,7 +731,7 @@ func (p *Parser) factor() (Node, error) {
UnaryNegate,
f,
first.Start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenBang:
@ -364,14 +747,14 @@ func (p *Parser) factor() (Node, error) {
UnaryNot,
v,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenName:
p.advance()
name := (*p.prev).Lexeme
start := p.prev.Start
nameEnd := start + p.prev.Length
nameEnd := p.prev.End
if p.curr.Type == TokenOpenParenthesis {
args, err := p.parseArgs()
@ -379,16 +762,15 @@ func (p *Parser) factor() (Node, error) {
return nil, err
}
return &CallNode{
return &InvokeNode{
&ReferenceNode{
name,
start,
nameEnd,
},
args,
true,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
}
@ -426,7 +808,7 @@ func (p *Parser) factor() (Node, error) {
sig,
b,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenOpenParenthesis:
@ -441,8 +823,16 @@ func (p *Parser) factor() (Node, error) {
return v, nil
case TokenBreakpoint:
p.advance()
return &BreakpointNode{
p.prev.Start,
p.prev.End,
}, nil
default:
return nil, p.error("invalid factor", p.curr)
return nil, p.error(fmt.Sprintf("invalid factor %s", p.curr), p.curr)
}
}
@ -465,7 +855,7 @@ func (p *Parser) prop() (Node, error) {
v,
property,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}
// if called, also add
@ -475,12 +865,11 @@ func (p *Parser) prop() (Node, error) {
return nil, err
}
v = &CallNode{
v = &InvokeNode{
v,
args,
true,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}
}
}
@ -512,7 +901,7 @@ func (p *Parser) product() (Node, error) {
left,
f,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}
}
@ -544,7 +933,7 @@ func (p *Parser) term() (Node, error) {
left,
pr,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}
}
@ -591,7 +980,7 @@ func (p *Parser) comparison() (Node, error) {
left,
t,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
}
@ -625,7 +1014,7 @@ func (p *Parser) condition() (Node, error) {
left,
c,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
}
@ -664,19 +1053,18 @@ func (p *Parser) statement() (Node, error) {
then,
otherwise,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenName:
p.advance()
start := p.prev.Start
name := (*p.prev).Lexeme
name := p.prev
if (*p.curr).Type == TokenDot {
var v Node = &ReferenceNode{
name,
start,
p.prev.Start + p.prev.Length,
name.Lexeme,
name.Start,
name.End,
}
// parse chains of prop-getting ( "".split().join().length.round() )
@ -689,8 +1077,8 @@ func (p *Parser) statement() (Node, error) {
v = &AccessNode{
v,
property,
start,
p.prev.Start + p.prev.Length,
name.Start,
p.prev.End,
}
// if called, also add
@ -700,12 +1088,11 @@ func (p *Parser) statement() (Node, error) {
return nil, err
}
v = &CallNode{
v = &InvokeNode{
v,
args,
(*p.curr).Type == TokenDot, // if the chain is continued, keep the value.
start,
p.prev.Start + p.prev.Length,
name.Start,
p.prev.End,
}
}
}
@ -717,16 +1104,15 @@ func (p *Parser) statement() (Node, error) {
return nil, err
}
return &CallNode{
return &InvokeNode{
&ReferenceNode{
name,
start,
start + Pos(len(name)),
name.Lexeme,
name.Start,
name.End,
},
args,
false,
start,
p.prev.Start + p.prev.Length,
name.Start,
p.prev.End,
}, nil
} else if p.accept(TokenAssign) || p.accept(TokenDeclare) {
isDeclaration := p.prev.Type == TokenDeclare
@ -735,12 +1121,16 @@ func (p *Parser) statement() (Node, error) {
return nil, err
}
return &AssignNode{
name,
return &AssignNode{ // THIS COULD BE MORE PERMISSIVE; its a new system
&ReferenceNode{
name.Lexeme,
name.Start,
name.End,
},
c,
isDeclaration,
start,
p.prev.Start + p.prev.Length,
name.Start,
p.prev.End,
}, nil
}
@ -754,7 +1144,7 @@ func (p *Parser) statement() (Node, error) {
if err := p.expect(TokenName, "function must have a name"); err != nil {
return nil, err
}
name := p.prev.Lexeme
name := p.prev
params, err := p.parseParams()
if err != nil {
@ -775,18 +1165,22 @@ func (p *Parser) statement() (Node, error) {
}
return &AssignNode{
name,
&ReferenceNode{
name.Lexeme,
name.Start,
name.End,
},
&FunctionNode{
name,
name.Lexeme,
params,
yield,
b,
funcStart,
p.prev.Start + p.prev.Length,
p.prev.End,
},
true,
funcStart,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenWhile:
@ -807,7 +1201,7 @@ func (p *Parser) statement() (Node, error) {
c,
b,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenReturn:
@ -822,7 +1216,7 @@ func (p *Parser) statement() (Node, error) {
return &ReturnNode{
c,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
case TokenBreakpoint:
@ -872,7 +1266,7 @@ func (p *Parser) block(canBeStatement bool) (Node, error) {
return &BlockNode{
statements,
start,
p.prev.Start + p.prev.Length,
p.prev.End,
}, nil
}
@ -884,7 +1278,7 @@ func (p *Parser) parseArgs() ([]Node, error) {
}
if !p.accept(TokenCloseParenthesis) {
c, err := p.condition()
c, err := p.expression(false)
if err != nil {
return nil, err
}
@ -893,7 +1287,7 @@ func (p *Parser) parseArgs() ([]Node, error) {
if err := p.expect(TokenComma, "arguments must be separated by comma"); err != nil {
return nil, err
}
c, err = p.condition()
c, err = p.expression(false)
if err != nil {
return nil, err
}
@ -1002,6 +1396,16 @@ func (p *Parser) parseSignature() (TypeSignature, error) {
in,
out,
}
} else if p.accept(TokenOpenBracket) {
inner, err := p.parseSignature()
if err != nil {
return nil, err
}
if err := p.expect(TokenCloseBracket, "list type must be enclosed in brackets"); err != nil {
return nil, err
}
return &ListSignature{inner}, nil
} else {
if err := p.expect(TokenName, "type must be a name"); err != nil {
return nil, err

View file

@ -63,7 +63,7 @@ func GetTokenTestData() map[string]TokenTestData {
&BlockNode{
[]Node{
&AssignNode{
"_",
&ReferenceNode{"_", 0, 0},
&BinaryNode{
BinaryAddition,
&FloatNode{
@ -93,7 +93,7 @@ func GetTokenTestData() map[string]TokenTestData {
&BlockNode{
[]Node{
&AssignNode{
"hello",
&ReferenceNode{"hello", 0, 0},
&StringNode{
"Hello world!",
"\"Hello world!\"",
@ -118,7 +118,7 @@ func GetTokenTestData() map[string]TokenTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&BinaryNode{
BinaryAddition,
&FloatNode{
@ -173,7 +173,7 @@ func GetTokenTestData() map[string]TokenTestData {
&BlockNode{
[]Node{
&AssignNode{
"_",
&ReferenceNode{"_", 0, 0},
&BinaryNode{
BinarySubtraction,
&BinaryNode{
@ -253,7 +253,7 @@ func GetTokenTestData() map[string]TokenTestData {
&BlockNode{
[]Node{
&AssignNode{
"_",
&ReferenceNode{"_", 0, 0},
&BinaryNode{
BinaryEquality,
&FloatNode{
@ -304,7 +304,7 @@ func GetTokenTestData() map[string]TokenTestData {
do: &BlockNode{
[]Node{
&AssignNode{
"b",
&ReferenceNode{"b", 0, 0},
&FloatNode{
1,
0, 0,
@ -357,7 +357,7 @@ func GetTokenTestData() map[string]TokenTestData {
do: &BlockNode{
[]Node{
&AssignNode{
"b",
&ReferenceNode{"b", 0, 0},
&FloatNode{
1,
0, 0,
@ -371,7 +371,7 @@ func GetTokenTestData() map[string]TokenTestData {
otherwise: &BlockNode{
[]Node{
&AssignNode{
"b",
&ReferenceNode{"b", 0, 0},
&FloatNode{
0,
0, 0,
@ -432,7 +432,7 @@ func GetTokenTestData() map[string]TokenTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FunctionNode{
"*",
[]FunctionParameter{
@ -503,7 +503,7 @@ func GetTokenTestData() map[string]TokenTestData {
&BlockNode{
[]Node{
&AssignNode{
"a",
&ReferenceNode{"a", 0, 0},
&FunctionNode{
"a",
[]FunctionParameter{
@ -559,7 +559,7 @@ func GetTokenTestData() map[string]TokenTestData {
&BlockNode{
[]Node{
&AssignNode{
"p",
&ReferenceNode{"p", 0, 0},
&AccessNode{
&ReferenceNode{
"a",
@ -607,7 +607,7 @@ func GetTokenTestData() map[string]TokenTestData {
&BlockNode{
[]Node{
&AssignNode{
"data",
&ReferenceNode{"data", 0, 0},
&ListNode{
[]Node{
&ReferenceNode{
@ -749,11 +749,8 @@ func NodeEquality(t *testing.T, n1 Node, n2 Node) {
NodeEquality(t, n1.(*LoopNode).do, n2.(*LoopNode).do)
case AssignNodeType:
if n1.(*AssignNode).name != n2.(*AssignNode).name {
t.Errorf("Assigned value name is not the same (%s and %s)", n1.(*AssignNode).name, n2.(*AssignNode).name)
} else {
t.Logf("Assigned value name matches (%s)", n1.(*AssignNode).name)
}
t.Logf("Checking if value destination matches")
NodeEquality(t, n1.(*AssignNode).dest, n2.(*AssignNode).dest)
if n1.(*AssignNode).declare != n2.(*AssignNode).declare {
t.Errorf("Not same type of assigning (1: %v; 2: %v)", n1.(*AssignNode).declare, n2.(*AssignNode).declare)
@ -762,22 +759,18 @@ func NodeEquality(t *testing.T, n1 Node, n2 Node) {
t.Logf("Checking equality of assignment values")
NodeEquality(t, n1.(*AssignNode).value, n2.(*AssignNode).value)
case CallNodeType:
n := n1.(*CallNode)
m := n2.(*CallNode)
case InvokeNodeType:
n := n1.(*InvokeNode)
m := n2.(*InvokeNode)
NodeEquality(t, n.source, m.source)
if n.keep == m.keep {
t.Logf("Call node keep modifier doesn't match (%v and %v)", n.keep, m.keep)
}
if len(n.args) != len(m.args) {
t.Fatalf("Call node arguments count does not match (%d and %d)", len(n.args), m.args)
}
for i, arg := range m.args {
NodeEquality(t, n1.(*CallNode).args[i], arg)
NodeEquality(t, n1.(*InvokeNode).args[i], arg)
}
case FunctionNodeType:
@ -958,7 +951,7 @@ func TestParser_Parse(t *testing.T) {
tree, err := p.Parse("")
if err != nil {
t.Fatalf("Unexpected error(s): %s", err.(ParsingError).Description)
t.Fatalf("Unexpected error(s): %s", err.(ParsingError).Format())
}
t.Logf("Checking parsed tree")

View file

@ -5,6 +5,30 @@ import (
"testing"
)
func CompareScope(t *testing.T, expectedScope []map[string]Value, actualScope *Scope) {
s := actualScope
for i := len(expectedScope) - 1; i >= 0; i-- {
if s == nil {
t.Fatal("scope cut unexpecetantly short")
}
for name, value := range expectedScope[i] {
v, ok := s.current[name]
if !ok {
t.Errorf("variable %s not found in correct scope", name)
continue
}
if !v.Equals(value) {
t.Errorf("variable %s has value %s but expected %s", name, v.String(), value.String())
} else {
t.Logf("variable %s has expected value %s", name, v.String())
}
}
}
}
func CompareStacks[T Value](t *testing.T, expected []T, actual *Stack[T]) {
if actual.Current != Pos(len(expected)) {
t.Errorf("Unexpected stack size. Expected %d, got %d", len(expected), actual.Current)

View file

@ -610,7 +610,7 @@ func (v *FunctionValue) Type() ValueType {
}
func (v *FunctionValue) String() string {
return fmt.Sprintf("<function name=%s>", v.Name)
return fmt.Sprintf("<function name=%s block=%p>", v.Name, v.Chunk)
}
func (v *FunctionValue) DebugString() string {
@ -619,7 +619,6 @@ func (v *FunctionValue) DebugString() string {
func (v *FunctionValue) Equals(other Value) bool {
return other.Type() == FunctionValueType &&
v.Name == other.(*FunctionValue).Name &&
v.Chunk == other.(*FunctionValue).Chunk
}
@ -675,43 +674,3 @@ func (v *BuiltinFunctionValue) Clone() Value {
v.Constant,
}
}
// VariableValue a value wrapper for variables kept on the stack
type VariableValue struct {
name string
value Value
scope Pos
}
func (v *VariableValue) Type() ValueType {
return VariableValueType
}
func (v *VariableValue) String() string {
return fmt.Sprintf("<variable name=%s value=%s scope=%d>", v.name, v.value, v.scope)
// variables should not be accessed on the stack; normal values should be pushed and popped predictably
//panic("tried getting string value of a unreachable value")
}
func (v *VariableValue) DebugString() string {
return v.String()
}
func (v *VariableValue) Equals(other Value) bool {
return other.Type() == VariableValueType &&
v.name == other.(*VariableValue).name &&
v.value.Equals(other.(*VariableValue).value)
}
func (v *VariableValue) Get(_ string) (Value, error) {
return nil, errors.New("variables have no properties")
}
func (v *VariableValue) Clone() Value {
return &VariableValue{
v.name,
v.value.Clone(),
v.scope,
}
}

View file

@ -70,20 +70,6 @@ func CompareValues(t *testing.T, got Value, want Value) {
t.Errorf("builtin function parameter count mismatch: got %v, want %v", n, m)
}
case VariableValueType:
n := got.(*VariableValue)
m := want.(*VariableValue)
if n.name != m.name {
t.Errorf("variable name mismatch: got %v, want %v", n.name, m.name)
}
if n.scope != m.scope {
t.Errorf("variable scope mismatch: got %v, want %v", n.scope, m.scope)
}
CompareValues(t, n.value, m.value)
case ListValueType:
n := got.(*ListValue)
m := want.(*ListValue)

View file

@ -104,6 +104,8 @@ const (
// InstructionSwap swap the two top items on the stack (1, 2 -> 2, 1)
InstructionSwap
// InstructionDuplicate push a copy of the item on top of the stack (1 -> 1, 1)
InstructionDuplicate
// InstructionAnd pop two booleans and push true if both are true
InstructionAnd
@ -234,6 +236,8 @@ func (b Bytecode) String() string {
return "ACCESS_PROPERTY"
case InstructionConcatLists:
return "CONCAT_LISTS"
case InstructionDuplicate:
return "DUPLICATE"
}
return "UNDEFINED"
}
@ -267,6 +271,30 @@ func (c Chunk) String() string {
return b.String()
}
func (c *Chunk) Equals(other *Chunk) bool {
if len(c.Bytecode) != len(other.Bytecode) {
return false
}
for i, bc := range c.Bytecode {
if other.Bytecode[i] != bc {
return false
}
}
if len(c.Constants) != len(other.Constants) {
return false
}
for i := 0; i < len(c.Constants); i++ {
if other.Constants[i] != c.Constants[i] {
return false
}
}
return true
}
func NewChunk(bytecode []Bytecode, constants []Value) *Chunk {
return &Chunk{bytecode, constants}
}
@ -327,23 +355,26 @@ type VM struct {
chunk *Chunk
// instruction pointer
ip Pos
scope Pos
ip Pos
// global variable storage
globals map[string]Value
variableEnd Pos
globals map[string]Value
// local variable storage
scope *Scope
stack *Stack[Value]
call *Stack[Call]
}
type Scope struct {
current map[string]Value
parent *Scope
}
type Call struct {
chunk *Chunk
ip Pos
stackEnd Pos
variableEnd Pos
scope Pos
chunk *Chunk
ip Pos
scope *Scope
}
var DefaultGlobals = map[string]Value{
@ -355,7 +386,7 @@ var DefaultGlobals = map[string]Value{
},
func(_ *VM, this Value, v []Value) (Value, error) {
println(v[0].String())
return nil, nil
return &NilValue{}, nil
},
nil,
false,
@ -368,7 +399,7 @@ var DefaultGlobals = map[string]Value{
},
func(_ *VM, this Value, v []Value) (Value, error) {
print(v[0].String())
return nil, nil
return &NilValue{}, nil
},
nil,
false,
@ -630,6 +661,9 @@ func NewVM(chunk *Chunk, stackSize Pos, callstackSize Pos) *VM {
call: NewStack[Call](callstackSize),
globals: DefaultGlobals,
scope: &Scope{
current: map[string]Value{},
},
}
return vm
@ -646,24 +680,20 @@ func (vm *VM) Next() bool {
case InstructionReturn:
if vm.call.Current == 0 {
return false
} else {
v := vm.stack.Pop()
c := vm.call.Pop()
// reset stack current and variable end and scope
vm.variableEnd = c.variableEnd
vm.stack.Current = c.stackEnd
vm.scope = c.scope
// reset to calling position
vm.ip = c.ip
vm.chunk = c.chunk
vm.purgeVars()
vm.stack.Push(v)
}
v := vm.stack.Pop()
c := vm.call.Pop()
// reset stack current and variable end and scope
vm.scope = c.scope
// reset to calling position
vm.ip = c.ip
vm.chunk = c.chunk
vm.stack.Push(v)
case InstructionPop:
vm.stack.Pop()
@ -805,28 +835,21 @@ func (vm *VM) Next() bool {
switch f := v.(type) {
case *FunctionValue:
vm.call.Push(Call{
chunk: vm.chunk,
ip: vm.ip,
stackEnd: vm.stack.Current - Pos(len(f.Params)),
variableEnd: vm.variableEnd,
scope: vm.scope,
chunk: vm.chunk,
ip: vm.ip,
scope: vm.scope,
})
vm.descend()
for i := len(f.Params) - 1; i >= 0; i-- {
p := vm.stack.Current - Pos(len(f.Params)) + Pos(i)
vm.stack.items[p] = &VariableValue{
f.Params[i].Name,
vm.stack.items[p],
vm.scope,
}
vm.addVar(f.Params[i].Name, vm.stack.Pop())
}
if f.Parent != nil {
vm.addVar("this", f.Parent)
}
vm.variableEnd = vm.stack.Current
vm.chunk = f.Chunk
vm.ip = 0
case *BuiltinFunctionValue:
@ -868,24 +891,18 @@ func (vm *VM) Next() bool {
return false
}
vm.stack.Push(v.value)
vm.stack.Push(v)
case InstructionSetLocal:
value := vm.stack.Pop().(Value)
value := vm.stack.Peek().Clone()
name := vm.GetConstant(vm.NextByte()).(*StringValue).Text
v := vm.getVar(name)
if v == nil {
vm.error(fmt.Sprintf("cannot set local: undefined variable %s", name))
}
v.value = value.Clone()
vm.setVar(name, value)
case InstructionDeclareLocal:
vm.addVar(
vm.GetConstant(vm.NextByte()).(*StringValue).Text,
vm.stack.Pop().Clone(),
vm.stack.Peek().Clone(),
)
case InstructionGetGlobal:
@ -954,6 +971,9 @@ func (vm *VM) Next() bool {
vm.stack.Push(r, l)
case InstructionDuplicate:
vm.stack.Push(vm.stack.Peek().Clone())
case InstructionAccessProperty:
source := vm.stack.Pop()
property := vm.ReadConstant()
@ -973,6 +993,7 @@ func (vm *VM) Next() bool {
vm.stack.Push(member)
case InstructionBreakpoint:
vm.stack.Push(&NilValue{})
default:
panic("invalid byte code")
@ -985,11 +1006,9 @@ func (vm *VM) Call(v Value, args []Value) (Value, error) {
switch f := v.(type) {
case *FunctionValue:
vm.call.Push(Call{
chunk: vm.chunk,
ip: vm.ip,
stackEnd: vm.stack.Current,
variableEnd: vm.variableEnd,
scope: vm.scope,
chunk: vm.chunk,
ip: vm.ip,
scope: vm.scope,
})
for i := 0; i < len(f.Params); i++ {
@ -1000,8 +1019,6 @@ func (vm *VM) Call(v Value, args []Value) (Value, error) {
vm.addVar("this", f.Parent)
}
vm.variableEnd = vm.stack.Current
vm.chunk = f.Chunk
vm.ip = 0
@ -1028,6 +1045,14 @@ func (vm *VM) TryNextByte() (Bytecode, error) {
return 0, errors.New("there are no more instructions")
}
for int(vm.ip) >= len(vm.chunk.Bytecode) && vm.call.Current > 0 {
c := vm.call.Pop()
vm.ip = c.ip
vm.chunk = c.chunk
vm.scope = c.scope
}
v := vm.chunk.Bytecode[vm.ip]
vm.ip++
@ -1045,53 +1070,55 @@ func (vm *VM) NextByte() Bytecode {
}
func (vm *VM) ascend() {
vm.scope--
if vm.scope < 0 {
if vm.scope.parent == nil {
panic("invalid scope")
}
vm.purgeVars()
}
// purgeVars remove all variables not within scope
func (vm *VM) purgeVars() {
for ; vm.variableEnd > 0 && vm.stack.items[vm.variableEnd-1].(*VariableValue).scope > vm.scope; vm.variableEnd-- {
vm.stack.Pop()
}
vm.scope = vm.scope.parent
}
func (vm *VM) descend() {
vm.scope++
old := vm.scope
vm.scope = &Scope{
map[string]Value{},
old,
}
}
func (vm *VM) addVar(name string, value Value) {
vm.variableEnd++
vm.stack.Push(&VariableValue{
name,
value,
vm.scope,
})
vm.scope.current[name] = value
}
func (vm *VM) getVar(name string) *VariableValue {
for i := vm.variableEnd - 1; i >= 0; i-- {
v, ok := vm.stack.items[i].(*VariableValue)
func (vm *VM) getVar(name string) Value {
s := vm.scope
if !ok {
continue
}
if v.name == name {
for s != nil {
if v, ok := s.current[name]; ok {
return v
}
s = s.parent
}
return nil
}
func (vm *VM) setVar(name string, v Value) {
s := vm.scope
for s != nil {
if _, ok := s.current[name]; ok {
s.current[name] = v
break
}
s = s.parent
}
}
func (vm *VM) HasNext() bool {
return vm.ip < Pos(len(vm.chunk.Bytecode))
return vm.ip < Pos(len(vm.chunk.Bytecode)) || vm.call.Current > 0
}
func (vm *VM) GetConstant(id Bytecode) Value {

View file

@ -95,10 +95,12 @@ func BenchmarkNewVM(b *testing.B) {
func GetExecutionTestData() map[string]struct {
chunk *Chunk
resultingStack []Value
resultingScope []map[string]Value
} {
return map[string]struct {
chunk *Chunk
resultingStack []Value
resultingScope []map[string]Value
}{
"two_plus_one": {
NewChunk([]Bytecode{
@ -112,6 +114,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&FloatValue{3},
},
[]map[string]Value{},
},
"push_constant": {
NewChunk(
@ -125,6 +128,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&FloatValue{1},
},
[]map[string]Value{},
},
"push_true": {
NewChunk(
@ -136,6 +140,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&BoolValue{true},
},
[]map[string]Value{},
},
"push_false": {
NewChunk(
@ -147,6 +152,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&BoolValue{false},
},
[]map[string]Value{},
},
"push_nil": {
NewChunk(
@ -158,6 +164,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&NilValue{},
},
[]map[string]Value{},
},
"empty": {
NewChunk(
@ -165,6 +172,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{},
),
[]Value{},
[]map[string]Value{},
},
// (2 + 1) * 5 / (6 - 2)
"full_arithmetic": {
@ -187,6 +195,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&FloatValue{3.75},
},
[]map[string]Value{},
},
"equality_true": {
NewChunk(
@ -202,6 +211,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&BoolValue{true},
},
[]map[string]Value{},
},
"equality_false": {
NewChunk(
@ -217,6 +227,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&BoolValue{false},
},
[]map[string]Value{},
},
"inequality_false": {
NewChunk(
@ -232,6 +243,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&BoolValue{false},
},
[]map[string]Value{},
},
"inequality_true": {
NewChunk(
@ -247,6 +259,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&BoolValue{true},
},
[]map[string]Value{},
},
"not_true": {
NewChunk(
@ -259,6 +272,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&BoolValue{false},
},
[]map[string]Value{},
},
"not_false": {
NewChunk(
@ -271,6 +285,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&BoolValue{true},
},
[]map[string]Value{},
},
"jump": {
NewChunk(
@ -286,6 +301,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&FloatValue{1},
},
[]map[string]Value{},
},
"jump_false/false": {
NewChunk(
@ -302,6 +318,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&FloatValue{1},
},
[]map[string]Value{},
},
"jump_false/true": {
NewChunk(
@ -318,6 +335,7 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&FloatValue{0}, &FloatValue{1},
},
[]map[string]Value{},
},
"declare_local": {
NewChunk(
@ -329,11 +347,10 @@ func GetExecutionTestData() map[string]struct {
&FloatValue{0}, &StringValue{"a"},
},
),
[]Value{
&VariableValue{
"a",
&FloatValue{0},
0,
[]Value{&FloatValue{0}},
[]map[string]Value{
{
"a": &FloatValue{0},
},
},
},
@ -342,18 +359,19 @@ func GetExecutionTestData() map[string]struct {
[]Bytecode{
InstructionConstant, 0,
InstructionDeclareLocal, 1,
InstructionPop,
InstructionConstant, 2,
InstructionSetLocal, 1, // reassign
InstructionPop,
},
[]Value{
&FloatValue{0}, &StringValue{"a"}, &FloatValue{1},
},
),
[]Value{
&VariableValue{
"a",
&FloatValue{1},
0,
[]Value{},
[]map[string]Value{
{
"a": &FloatValue{1},
},
},
},
@ -362,29 +380,30 @@ func GetExecutionTestData() map[string]struct {
[]Bytecode{
InstructionConstant, 0,
InstructionDeclareLocal, 1,
InstructionGetLocal, 1, // reassign
},
[]Value{
&FloatValue{0}, &StringValue{"a"},
},
),
[]Value{
&VariableValue{
"a",
&FloatValue{0},
0,
},
&FloatValue{0},
},
[]map[string]Value{
{
"a": &FloatValue{0},
},
},
},
"get_reassigned_local": {
NewChunk(
[]Bytecode{
InstructionConstant, 0,
InstructionDeclareLocal, 1,
InstructionPop,
InstructionGetLocal, 1,
InstructionConstant, 2,
InstructionSetLocal, 1, // reassign
InstructionPop,
InstructionGetLocal, 1,
},
[]Value{
@ -392,26 +411,29 @@ func GetExecutionTestData() map[string]struct {
},
),
[]Value{
&VariableValue{
"a",
&FloatValue{1},
0,
},
&FloatValue{0},
&FloatValue{1},
},
[]map[string]Value{
{
"a": &FloatValue{1},
},
},
},
"variable_scope": {
NewChunk(
[]Bytecode{
InstructionConstant, 0,
InstructionDeclareLocal, 1,
InstructionPop,
InstructionDescend,
InstructionConstant, 2,
InstructionDeclareLocal, 3,
InstructionPop,
InstructionDescend,
InstructionConstant, 4,
InstructionDeclareLocal, 5,
InstructionPop,
InstructionAscend,
InstructionAscend,
},
@ -421,11 +443,10 @@ func GetExecutionTestData() map[string]struct {
&FloatValue{2}, &StringValue{"c"},
},
),
[]Value{
&VariableValue{
"a",
&FloatValue{0},
0,
[]Value{},
[]map[string]Value{
{
"a": &FloatValue{0},
},
},
},
@ -469,12 +490,14 @@ func GetExecutionTestData() map[string]struct {
[]Value{
&FloatValue{3},
},
[]map[string]Value{},
},
"function_calling_function": {
NewChunk(
[]Bytecode{
InstructionConstant, 3,
InstructionDeclareLocal, 4,
InstructionPop,
InstructionConstant, 0,
InstructionConstant, 1,
InstructionConstant, 2,
@ -533,9 +556,11 @@ func GetExecutionTestData() map[string]struct {
},
),
[]Value{
&VariableValue{
"square",
&FunctionValue{
&FloatValue{5},
},
[]map[string]Value{
{
"square": &FunctionValue{
Name: "square",
Params: []FunctionParameter{
{
@ -555,9 +580,7 @@ func GetExecutionTestData() map[string]struct {
},
),
},
0,
},
&FloatValue{5},
},
},
"list_concat": {
@ -590,6 +613,7 @@ func GetExecutionTestData() map[string]struct {
},
},
},
[]map[string]Value{},
},
}
}

View file

@ -1,6 +1,10 @@
fn double(a: int) -> int {
return 2*a
2*a
}
a := 1
a = 2
println(double(2) == 4)