package core import ( "testing" ) type LexerTestData struct { source string expectedTokens []TokenKind } func GetLexerTestData() map[string]LexerTestData { return map[string]LexerTestData{ "hello_world_string(1)": { "\"Hello world\"", []TokenKind{TokenString, TokenEOF}, }, "empty_string(1)": { "\"\"", []TokenKind{TokenString, TokenEOF}, }, "simple number(1)": { "1024", []TokenKind{TokenInteger, TokenEOF}, }, "simple_arithmetics(7)": { "1 + 23 / 4 * 3", []TokenKind{ TokenInteger, TokenPlus, TokenInteger, TokenSlash, TokenInteger, TokenStar, TokenInteger, TokenEOF, }, }, "condition(3)": { "a <= 200", []TokenKind{TokenName, TokenLessThanOrEqual, TokenInteger, TokenEOF}, }, "if_statement(10)": { "if a >= 200 {\n write(\"Hello world!\")\n}", []TokenKind{ TokenIf, TokenName, TokenGreaterThanOrEqual, TokenInteger, TokenOpenBrace, TokenNewLine, TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenNewLine, TokenCloseBrace, TokenEOF, }, }, "if_else_statement(20)": { "if 23 * 2/3 > 32 {\n write(\"It is larger!\")\n} else {\n write(\"It is lower!\")\n}", []TokenKind{ TokenIf, TokenInteger, TokenStar, TokenInteger, TokenSlash, TokenInteger, TokenGreaterThan, TokenInteger, TokenOpenBrace, TokenNewLine, TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenNewLine, TokenCloseBrace, TokenElse, TokenOpenBrace, TokenNewLine, TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenNewLine, TokenCloseBrace, TokenEOF, }, }, "empty_string": { "", []TokenKind{TokenEOF}, }, "full_arithmetic_equality": { "a + 2 == 10 * 2 / 3", []TokenKind{ TokenName, TokenPlus, TokenInteger, TokenEquals, TokenInteger, TokenStar, TokenInteger, TokenSlash, TokenInteger, TokenEOF, }, }, "name": { "print", []TokenKind{TokenName, TokenEOF}, }, "bunch_of_parentheses": { "(((())))", []TokenKind{ TokenOpenParenthesis, TokenOpenParenthesis, TokenOpenParenthesis, TokenOpenParenthesis, TokenCloseParenthesis, TokenCloseParenthesis, TokenCloseParenthesis, TokenCloseParenthesis, TokenEOF, }, }, "space_before_string": { "\n \"\"", []TokenKind{TokenNewLine, TokenString, TokenEOF}, }, "write_call": { "write(\"Hello world\")", []TokenKind{TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenEOF}, }, "complex_comparison": { "!(h__elo123 >= 1)", []TokenKind{ TokenBang, TokenOpenParenthesis, TokenName, TokenGreaterThanOrEqual, TokenInteger, TokenCloseParenthesis, TokenEOF, }, }, "3assignments_1condition": { "a = 8 * 32\nb = a > 256\nc = a <= 256\n!b == c", []TokenKind{ TokenName, TokenAssign, TokenInteger, TokenStar, TokenInteger, TokenNewLine, TokenName, TokenAssign, TokenName, TokenGreaterThan, TokenInteger, TokenNewLine, TokenName, TokenAssign, TokenName, TokenLessThanOrEqual, TokenInteger, TokenNewLine, TokenBang, TokenName, TokenEquals, TokenName, TokenEOF, }, }, "function": { "fn sum(a, b) {\n return a + b\n}", []TokenKind{ TokenFunc, TokenName, TokenOpenParenthesis, TokenName, TokenComma, TokenName, TokenCloseParenthesis, TokenOpenBrace, TokenNewLine, TokenReturn, TokenName, TokenPlus, TokenName, TokenNewLine, TokenCloseBrace, }, }, "while_loop": { "while a < 5 {\n a = a + 1\n}", []TokenKind{ TokenWhile, TokenName, TokenLessThan, TokenInteger, TokenOpenBrace, TokenNewLine, TokenName, TokenAssign, TokenName, TokenPlus, TokenInteger, TokenNewLine, TokenCloseBrace, TokenEOF, }, }, "lambda": { "sum := fn(a, b) {\n" + " return a + b\n" + "}", []TokenKind{ TokenName, TokenDeclare, TokenFunc, TokenOpenParenthesis, TokenName, TokenComma, TokenName, TokenCloseParenthesis, TokenOpenBrace, TokenNewLine, TokenReturn, TokenName, TokenPlus, TokenName, TokenNewLine, TokenCloseBrace, }, }, "list": { "data := [3, 1, 4, 1]", []TokenKind{ TokenName, TokenDeclare, TokenOpenBracket, TokenInteger, TokenComma, TokenInteger, TokenComma, TokenInteger, TokenComma, TokenInteger, TokenCloseBracket, }, }, } } func TestLexer_NextToken(t *testing.T) { data := GetLexerTestData() for name, tc := range data { t.Run(name, func(t *testing.T) { lex := NewLexer(tc.source) t.Logf("Testing source: '%s'", tc.source) for _, expectedType := range tc.expectedTokens { tok, err := lex.NextToken() if err != nil { t.Errorf("Unexpected error while parsing token '%s'. Error: %s", expectedType, err) continue } if tok.Type != expectedType { t.Errorf("Expected token type '%s' but got '%s'", expectedType, tok.Type) } else { t.Logf("Got expected token type '%s'", expectedType) } } }) } } func TestNewLexer(t *testing.T) { lex := NewLexer("example source") if lex == nil { t.Fatalf("Lexer was not initialized correctly.") } if lex.start != 0 { t.Errorf("Lexer start position was not initialized correctly.") } if lex.current != 0 { t.Errorf("Lexer current position was not initialized correctly.") } if string(lex.src) != "example source" { t.Errorf("Lexer lexer was not initialized correctly.") } t.Log("Successfully initialized lexer") } // lexer NextToken provides an error when it comes across an invalid token func TestLexer_NextTokenErrors(t *testing.T) { invalidCodes := []string{ // Invalid tokens "^", "@", "$&", "ยจ", // Non-ending string (in same line) "\"", "Hini minit \"mini moe", "\"this is some test\ncontent\"", "\n\"Hello world", } for _, code := range invalidCodes { lex := NewLexer(code) tok, err := lex.NextToken() for err == nil && tok.Type != TokenEOF { tok, err = lex.NextToken() } if err == nil { t.Errorf("Expected error for invalid code '%s'", code) } else { t.Logf("Got an expected error (%s) for invalid code '%s'", err.Error(), code) } } } func BenchmarkNewLexer(b *testing.B) { for i := 0; i < b.N; i++ { _ = NewLexer("example source") } } func BenchmarkLexer_NextToken(b *testing.B) { data := GetLexerTestData() for name, tc := range data { b.Run(name, func(b *testing.B) { for i := 0; i < b.N; i++ { lex := NewLexer(tc.source) tok, err := lex.NextToken() for err == nil && tok.Type != TokenEOF { tok, err = lex.NextToken() } } }) } }