anglais/core/lexer.go

package core

import (
	"errors"
	"fmt"
	"unicode"
)

type Token struct {
	Type   TokenKind
	Start  Pos
	End    Pos
	Line   Pos
	Lexeme string
}

func (t Token) Bounds() (Pos, Pos) {
	return t.Start, t.End
}

func (t Token) String() string {
	return fmt.Sprintf("token %s, '%s' %d -> %d, line %d", t.Type.String(), t.Lexeme, t.Start, t.End, t.Line)
}

type TokenKind uint64

const (
	TokenPlus TokenKind = iota
	TokenMinus
	TokenStar
	TokenSlash
	TokenPercent
	TokenBang
	TokenSemicolon

	TokenInteger
	TokenFloat
	TokenHexadecimal
	TokenString
	TokenName

	TokenOpenParenthesis
	TokenCloseParenthesis
	TokenOpenBracket
	TokenCloseBracket
	TokenOpenBrace
	TokenCloseBrace

	TokenTrue
	TokenFalse
	TokenNil

	TokenFunc
	TokenReturn
	TokenWhile
	TokenVar
	TokenIf
	TokenElse
	TokenInclude
	TokenType
	TokenFor
	TokenIn

	TokenComma
	TokenDot
	TokenColon
	TokenArrow

	TokenAssign
	TokenDeclare
	TokenBangEquals
	TokenEquals
	TokenGreaterThan
	TokenLessThan
	TokenGreaterThanOrEqual
	TokenLessThanOrEqual

	TokenDoubleAmpersand
	TokenPipe
	TokenDoublePipe

	TokenNewLine
	TokenBreakpoint
	TokenEOF
	TokenError
)

func (t TokenKind) String() string {
	switch t {
	case TokenPlus:
		return "plus"
	case TokenMinus:
		return "minus"
	case TokenStar:
		return "star"
	case TokenSlash:
		return "slash"
	case TokenBang:
		return "bang"
	case TokenFloat:
		return "float"
	case TokenInteger:
		return "integer"
	case TokenString:
		return "string"
	case TokenTrue:
		return "true"
	case TokenFalse:
		return "false"
	case TokenNil:
		return "nil"
	case TokenOpenParenthesis:
		return "open parenthesis"
	case TokenCloseParenthesis:
		return "close parenthesis"
	case TokenOpenBrace:
		return "open brace"
	case TokenCloseBrace:
		return "close brace"
	case TokenVar:
		return "var"
	case TokenIf:
		return "if"
	case TokenElse:
		return "else"
	case TokenAssign:
		return "equals"
	case TokenBangEquals:
		return "equals"
	case TokenEquals:
		return "double equals"
	case TokenGreaterThan:
		return "greater than"
	case TokenLessThan:
		return "less than"
	case TokenGreaterThanOrEqual:
		return "greater than or equal"
	case TokenLessThanOrEqual:
		return "less than or equal"
	case TokenName:
		return "name"
	case TokenEOF:
		return "EOF"
	case TokenError:
		return "error"
	case TokenSemicolon:
		return "semicolon"
	case TokenDeclare:
		return "declare"
	case TokenFunc:
		return "fn"
	case TokenReturn:
		return "return"
	case TokenWhile:
		return "while"
	case TokenComma:
		return "comma"
	case TokenDot:
		return "dot"
	case TokenBreakpoint:
		return "breakpoint"
	case TokenDoubleAmpersand:
		return "double ampersand"
	case TokenDoublePipe:
		return "double pipe"
	case TokenOpenBracket:
		return "open bracket"
	case TokenCloseBracket:
		return "close bracket"
	case TokenInclude:
		return "include"
	case TokenColon:
		return "colon"
	case TokenPipe:
		return "pipe"
	case TokenHexadecimal:
		return "hexadecimal"
	case TokenArrow:
		return "arrow"
	case TokenNewLine:
		return "newline"
	case TokenType:
		return "type"
	case TokenFor:
		return "for"
	case TokenIn:
		return "in"
	}

	panic("UNDEFINED TOKENTYPE STRING CONVERSION")
}

var Keywords = map[string]TokenKind{
	"true":       TokenTrue,
	"false":      TokenFalse,
	"nil":        TokenNil,
	"if":         TokenIf,
	"else":       TokenElse,
	"include":    TokenInclude,
	"var":        TokenVar,
	"fn":         TokenFunc,
	"return":     TokenReturn,
	"while":      TokenWhile,
	"breakpoint": TokenBreakpoint,
	"type":       TokenType,
	"for":        TokenFor,
	"in":         TokenIn,
}

type Lexer struct {
	src     []rune
	start   Pos
	current Pos
	line    Pos
}

func NewLexer(src string) *Lexer {
	return &Lexer{
		src:     []rune(src),
		start:   0,
		current: 0,
		line:    0,
	}
}

func (l *Lexer) NextToken() (Token, error) {
	l.skipWhitespace()

	// if at end of source
	if l.isAtEnd() {
		return l.makeToken(TokenEOF), nil
	}

	// skip comments
	if l.match('#') {
		for !l.match('\n') {
			l.advance()
		}

		return l.NextToken()
	}

	l.start = l.current

	var c = l.src[l.current]
	l.advance()

	switch c {
	case '\n':
		return l.makeToken(TokenNewLine), nil
	case '+':
		return l.makeToken(TokenPlus), nil
	case '-':
		if l.accept('>') {
			return l.makeToken(TokenArrow), nil
		}
		return l.makeToken(TokenMinus), nil
	case '*':
		return l.makeToken(TokenStar), nil
	case '/':
		if l.accept('*') {
			for !l.isAtEnd() {
				if l.accept('*') && l.accept('/') {
					break
				}
				l.advance()
			}
			return l.NextToken()
		}

		return l.makeToken(TokenSlash), nil
	case '%':
		return l.makeToken(TokenPercent), nil
	case '(':
		return l.makeToken(TokenOpenParenthesis), nil
	case ')':
		return l.makeToken(TokenCloseParenthesis), nil
	case '[':
		return l.makeToken(TokenOpenBracket), nil
	case ']':
		return l.makeToken(TokenCloseBracket), nil
	case '{':
		return l.makeToken(TokenOpenBrace), nil
	case '}':
		return l.makeToken(TokenCloseBrace), nil
	case ';':
		return l.makeToken(TokenSemicolon), nil
	case ',':
		return l.makeToken(TokenComma), nil
	case '.':
		return l.makeToken(TokenDot), nil
	case ':':
		if l.accept('=') {
			return l.makeToken(TokenDeclare), nil
		}

		return l.makeToken(TokenColon), nil
	case '!':
		if l.accept('=') {
			return l.makeToken(TokenBangEquals), nil
		}

		return l.makeToken(TokenBang), nil
	case '=':
		if l.accept('=') {
			return l.makeToken(TokenEquals), nil
		}

		return l.makeToken(TokenAssign), nil
	case '>':
		if l.accept('=') {
			return l.makeToken(TokenGreaterThanOrEqual), nil
		}

		return l.makeToken(TokenGreaterThan), nil
	case '<':
		if l.accept('=') {
			return l.makeToken(TokenLessThanOrEqual), nil
		}

		return l.makeToken(TokenLessThan), nil

	case '&':
		if l.accept('&') {
			return l.makeToken(TokenDoubleAmpersand), nil
		}

		return l.makeToken(TokenError), errors.New("malformed token (got '&', expected '&' to follow)")

	case '|':
		if l.accept('|') {
			return l.makeToken(TokenDoublePipe), nil
		}

		return l.makeToken(TokenPipe), nil

	case '"':
		// include ending quote
		for !l.accept('"') {
			if l.match('\n') {
				return l.makeToken(TokenError), errors.New("string did not end in current line")
			}

			if l.isAtEnd() {
				return l.makeToken(TokenError), errors.New("string did not before end of source")
			}

			l.advance()
		}

		return l.makeToken(TokenString), nil

	case '\'':
		// include ending quote
		for !l.accept('\'') {
			if l.match('\n') {
				return l.makeToken(TokenError), errors.New("string did not end in current line")
			}

			if l.isAtEnd() {
				return l.makeToken(TokenError), errors.New("string did not before end of source")
			}

			l.advance()
		}

		return l.makeToken(TokenString), nil

	default:
		if l.isAlpha(c) {
			// assemble variable
			for l.isAlphaNumeric(l.peek()) {
				l.advance()
			}

			lexeme := string(l.src[l.start:l.current])
			if k, ok := Keywords[lexeme]; ok {
				return l.makeToken(k), nil
			}

			return l.makeToken(TokenName), nil
		} else if c == '0' && l.peek() != '.' {
			if l.peek() == 'x' {
				l.advance()

				// hex
				for unicode.In(l.peek(), unicode.Hex_Digit) {
					l.advance()
				}

				return l.makeToken(TokenHexadecimal), nil
			}

			return l.makeToken(TokenInteger), nil
		} else if unicode.IsDigit(c) {
			for unicode.IsDigit(l.peek()) {
				l.advance()
			}

			// if the number has a float-part
			if l.accept('.') {
				for unicode.IsDigit(l.peek()) {
					l.advance()
				}
				return l.makeToken(TokenFloat), nil
			}

			return l.makeToken(TokenInteger), nil

		}

		return l.makeToken(TokenError), errors.New(fmt.Sprintf("invalid token %c", c))
	}
}

func NewToken(t TokenKind, start Pos, end Pos, line Pos, lexeme string) Token {
	return Token{
		Type:   t,
		Start:  start,
		End:    end,
		Line:   line,
		Lexeme: lexeme,
	}
}

func (l *Lexer) Tokenize() ([]Token, error) {
	tokens := make([]Token, 0)

	tok, err := l.NextToken()
	for ; err == nil; tok, err = l.NextToken() {
		tokens = append(tokens, tok)

		if tok.Type == TokenEOF {
			break
		}
	}

	return tokens, err
}

func (l *Lexer) makeToken(t TokenKind) Token {
	return NewToken(t, l.start, l.current, l.line, string(l.src[l.start:l.current]))
}

func (l *Lexer) peek() rune {
	if l.isAtEnd() {
		return 0
	}

	return l.src[l.current]
}

func (l *Lexer) match(c rune) bool {
	return l.peek() == c
}

func (l *Lexer) accept(c rune) bool {
	if l.match(c) {
		l.advance()
		return true
	}

	return false
}

func (l *Lexer) isAlpha(c rune) bool {
	return unicode.IsLetter(c) || c == '_'
}

func (l *Lexer) isAlphaNumeric(c rune) bool {
	return l.isAlpha(c) || unicode.IsDigit(c)
}

func (l *Lexer) advance() {
	if l.isAtEnd() {
		return
	}

	if l.src[l.current] == '\n' {
		l.line++
	}

	l.current++
}

func (l *Lexer) isAtEnd() bool {
	return l.current >= Pos(len(l.src))
}

func (l *Lexer) skipWhitespace() {
	for !l.isAtEnd() && unicode.IsSpace(l.peek()) && l.peek() != '\n' {
		l.advance()
	}
}