anglais/core/lexer_test.go

package core

import (
	"testing"
)

type LexerTestData struct {
	source         string
	expectedTokens []TokenKind
}

func GetLexerTestData() map[string]LexerTestData {
	return map[string]LexerTestData{
		"hello_world_string(1)": {
			"\"Hello world\"",
			[]TokenKind{TokenString, TokenEOF},
		},
		"empty_string(1)": {
			"\"\"",
			[]TokenKind{TokenString, TokenEOF},
		},
		"simple number(1)": {
			"1024",
			[]TokenKind{TokenInteger, TokenEOF},
		},
		"simple_arithmetics(7)": {
			"1 + 23 / 4 * 3",
			[]TokenKind{
				TokenInteger, TokenPlus, TokenInteger, TokenSlash,
				TokenInteger, TokenStar, TokenInteger, TokenEOF,
			},
		},
		"condition(3)": {
			"a <= 200",
			[]TokenKind{TokenName, TokenLessThanOrEqual, TokenInteger, TokenEOF},
		},
		"if_statement(10)": {
			"if a >= 200 {\n    write(\"Hello world!\")\n}",
			[]TokenKind{
				TokenIf, TokenName, TokenGreaterThanOrEqual, TokenInteger, TokenOpenBrace, TokenNewLine,
				TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenNewLine,
				TokenCloseBrace, TokenEOF,
			},
		},
		"if_else_statement(20)": {
			"if 23 * 2/3 > 32 {\n    write(\"It is larger!\")\n} else {\n    write(\"It is lower!\")\n}",
			[]TokenKind{
				TokenIf, TokenInteger, TokenStar, TokenInteger, TokenSlash, TokenInteger, TokenGreaterThan, TokenInteger, TokenOpenBrace, TokenNewLine,
				TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenNewLine, TokenCloseBrace,
				TokenElse, TokenOpenBrace, TokenNewLine, TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenNewLine, TokenCloseBrace,
				TokenEOF,
			},
		},
		"empty_string": {
			"",
			[]TokenKind{TokenEOF},
		},
		"full_arithmetic_equality": {
			"a + 2 == 10 * 2 / 3",
			[]TokenKind{
				TokenName, TokenPlus, TokenInteger, TokenEquals,
				TokenInteger, TokenStar, TokenInteger, TokenSlash, TokenInteger,
				TokenEOF,
			},
		},
		"name": {
			"print",
			[]TokenKind{TokenName, TokenEOF},
		},
		"bunch_of_parentheses": {
			"(((())))",
			[]TokenKind{
				TokenOpenParenthesis, TokenOpenParenthesis, TokenOpenParenthesis, TokenOpenParenthesis,
				TokenCloseParenthesis, TokenCloseParenthesis, TokenCloseParenthesis, TokenCloseParenthesis,
				TokenEOF,
			},
		},
		"space_before_string": {
			"\n   	\"\"",
			[]TokenKind{TokenNewLine, TokenString, TokenEOF},
		},
		"write_call": {
			"write(\"Hello world\")",
			[]TokenKind{TokenName, TokenOpenParenthesis, TokenString, TokenCloseParenthesis, TokenEOF},
		},
		"complex_comparison": {
			"!(h__elo123  >= 1)",
			[]TokenKind{
				TokenBang, TokenOpenParenthesis, TokenName, TokenGreaterThanOrEqual, TokenInteger, TokenCloseParenthesis,
				TokenEOF,
			},
		},
		"3assignments_1condition": {
			"a = 8 * 32\nb = a > 256\nc = a <= 256\n!b == c",
			[]TokenKind{
				TokenName, TokenAssign, TokenInteger, TokenStar, TokenInteger, TokenNewLine,
				TokenName, TokenAssign, TokenName, TokenGreaterThan, TokenInteger, TokenNewLine,
				TokenName, TokenAssign, TokenName, TokenLessThanOrEqual, TokenInteger, TokenNewLine,
				TokenBang, TokenName, TokenEquals, TokenName, TokenEOF,
			},
		},
		"function": {
			"fn sum(a, b) {\n    return a + b\n}",
			[]TokenKind{
				TokenFunc, TokenName, TokenOpenParenthesis, TokenName, TokenComma, TokenName, TokenCloseParenthesis,
				TokenOpenBrace, TokenNewLine, TokenReturn, TokenName, TokenPlus, TokenName, TokenNewLine, TokenCloseBrace,
			},
		},
		"while_loop": {
			"while a < 5 {\n    a = a + 1\n}",
			[]TokenKind{
				TokenWhile, TokenName, TokenLessThan, TokenInteger, TokenOpenBrace, TokenNewLine,
				TokenName, TokenAssign, TokenName, TokenPlus, TokenInteger, TokenNewLine, TokenCloseBrace, TokenEOF,
			},
		},
		"lambda": {
			"sum := fn(a, b) {\n" +
				"    return a + b\n" +
				"}",
			[]TokenKind{
				TokenName, TokenDeclare, TokenFunc, TokenOpenParenthesis, TokenName, TokenComma, TokenName, TokenCloseParenthesis,
				TokenOpenBrace, TokenNewLine, TokenReturn, TokenName, TokenPlus, TokenName, TokenNewLine, TokenCloseBrace,
			},
		},
		"list": {
			"data := [3, 1, 4, 1]",
			[]TokenKind{
				TokenName, TokenDeclare, TokenOpenBracket, TokenInteger, TokenComma, TokenInteger, TokenComma, TokenInteger, TokenComma, TokenInteger, TokenCloseBracket,
			},
		},
	}
}

func TestLexer_NextToken(t *testing.T) {
	data := GetLexerTestData()

	for name, tc := range data {
		t.Run(name, func(t *testing.T) {
			lex := NewLexer(tc.source)

			t.Logf("Testing source: '%s'", tc.source)

			for _, expectedType := range tc.expectedTokens {
				tok, err := lex.NextToken()

				if err != nil {
					t.Errorf("Unexpected error while parsing token '%s'. Error: %s", expectedType, err)
					continue
				}

				if tok.Kind != expectedType {
					t.Errorf("Expected token type '%s' but got '%s'", expectedType, tok.Kind)
				} else {
					t.Logf("Got expected token type '%s'", expectedType)
				}
			}
		})
	}
}

func TestNewLexer(t *testing.T) {
	lex := NewLexer("example source")

	if lex == nil {
		t.Fatalf("Lexer was not initialized correctly.")
	}

	if lex.start != 0 {
		t.Errorf("Lexer start position was not initialized correctly.")
	}

	if lex.current != 0 {
		t.Errorf("Lexer current position was not initialized correctly.")
	}

	if string(lex.src) != "example source" {
		t.Errorf("Lexer lexer was not initialized correctly.")
	}

	t.Log("Successfully initialized lexer")
}

// lexer NextToken provides an error when it comes across an invalid token
func TestLexer_NextTokenErrors(t *testing.T) {
	invalidCodes := []string{
		// Invalid tokens
		"^", "@", "$&", "¨",
		// Non-ending string (in same line)
		"\"", "Hini minit \"mini moe", "\"this is some test\ncontent\"", "\n\"Hello world",
	}

	for _, code := range invalidCodes {
		lex := NewLexer(code)
		tok, err := lex.NextToken()

		for err == nil && tok.Kind != TokenEOF {
			tok, err = lex.NextToken()
		}

		if err == nil {
			t.Errorf("Expected error for invalid code '%s'", code)
		} else {
			t.Logf("Got an expected error (%s) for invalid code '%s'", err.Error(), code)
		}
	}
}

func BenchmarkNewLexer(b *testing.B) {
	for i := 0; i < b.N; i++ {
		_ = NewLexer("example source")
	}
}

func BenchmarkLexer_NextToken(b *testing.B) {
	data := GetLexerTestData()

	for name, tc := range data {
		b.Run(name, func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				lex := NewLexer(tc.source)
				tok, err := lex.NextToken()

				for err == nil && tok.Kind != TokenEOF {
					tok, err = lex.NextToken()
				}
			}
		})
	}

}