package core import ( "errors" "fmt" "unicode" ) type Token struct { Kind TokenKind Start Pos End Pos Line Pos Lexeme string } func (t Token) Bounds() (Pos, Pos) { return t.Start, t.End } func (t Token) String() string { return fmt.Sprintf("token %s, '%s' %d -> %d, line %d", t.Kind.String(), t.Lexeme, t.Start, t.End, t.Line) } type TokenKind uint64 const ( TokenPlus TokenKind = iota TokenMinus TokenStar TokenSlash TokenPercent TokenBang TokenSemicolon TokenInteger TokenFloat TokenHexadecimal TokenString TokenName TokenOpenParenthesis TokenCloseParenthesis TokenOpenBracket TokenCloseBracket TokenOpenBrace TokenCloseBrace TokenTrue TokenFalse TokenNil TokenFunc TokenReturn TokenWhile TokenVar TokenIf TokenElse TokenInclude TokenType TokenFor TokenIn TokenComma TokenDot TokenColon TokenArrow TokenAssign TokenDeclare TokenBangEquals TokenEquals TokenGreaterThan TokenLessThan TokenGreaterThanOrEqual TokenLessThanOrEqual TokenDoubleAmpersand TokenPipe TokenDoublePipe TokenNewLine TokenBreakpoint TokenEOF TokenError ) func (t TokenKind) String() string { switch t { case TokenPlus: return "plus" case TokenMinus: return "minus" case TokenStar: return "star" case TokenSlash: return "slash" case TokenBang: return "bang" case TokenFloat: return "float" case TokenInteger: return "integer" case TokenString: return "string" case TokenTrue: return "true" case TokenFalse: return "false" case TokenNil: return "nil" case TokenOpenParenthesis: return "open parenthesis" case TokenCloseParenthesis: return "close parenthesis" case TokenOpenBrace: return "open brace" case TokenCloseBrace: return "close brace" case TokenVar: return "var" case TokenIf: return "if" case TokenElse: return "else" case TokenAssign: return "equals" case TokenBangEquals: return "equals" case TokenEquals: return "double equals" case TokenGreaterThan: return "greater than" case TokenLessThan: return "less than" case TokenGreaterThanOrEqual: return "greater than or equal" case TokenLessThanOrEqual: return "less than or equal" case TokenName: return "name" case TokenEOF: return "EOF" case TokenError: return "error" case TokenSemicolon: return "semicolon" case TokenDeclare: return "declare" case TokenFunc: return "fn" case TokenReturn: return "return" case TokenWhile: return "while" case TokenComma: return "comma" case TokenDot: return "dot" case TokenBreakpoint: return "breakpoint" case TokenDoubleAmpersand: return "double ampersand" case TokenDoublePipe: return "double pipe" case TokenOpenBracket: return "open bracket" case TokenCloseBracket: return "close bracket" case TokenInclude: return "include" case TokenColon: return "colon" case TokenPipe: return "pipe" case TokenHexadecimal: return "hexadecimal" case TokenArrow: return "arrow" case TokenNewLine: return "newline" case TokenType: return "type" case TokenFor: return "for" case TokenIn: return "in" case TokenPercent: return "percent" } panic("UNDEFINED TOKENTYPE STRING CONVERSION") } var Keywords = map[string]TokenKind{ "true": TokenTrue, "false": TokenFalse, "nil": TokenNil, "if": TokenIf, "else": TokenElse, "include": TokenInclude, "var": TokenVar, "fn": TokenFunc, "return": TokenReturn, "while": TokenWhile, "breakpoint": TokenBreakpoint, "type": TokenType, "for": TokenFor, "in": TokenIn, } type Lexer struct { src []rune start Pos current Pos line Pos } func NewLexer(src string) *Lexer { return &Lexer{ src: []rune(src), start: 0, current: 0, line: 0, } } func (l *Lexer) NextToken() (Token, error) { l.skipWhitespace() // if at end of source if l.isAtEnd() { return l.makeToken(TokenEOF), nil } // skip comments if l.match('#') { for !l.match('\n') { l.advance() } return l.NextToken() } l.start = l.current var c = l.src[l.current] l.advance() switch c { case '\n': return l.makeToken(TokenNewLine), nil case '+': return l.makeToken(TokenPlus), nil case '-': if l.accept('>') { return l.makeToken(TokenArrow), nil } return l.makeToken(TokenMinus), nil case '*': return l.makeToken(TokenStar), nil case '/': if l.accept('*') { for !l.isAtEnd() { if l.accept('*') && l.accept('/') { break } l.advance() } return l.NextToken() } return l.makeToken(TokenSlash), nil case '%': return l.makeToken(TokenPercent), nil case '(': return l.makeToken(TokenOpenParenthesis), nil case ')': return l.makeToken(TokenCloseParenthesis), nil case '[': return l.makeToken(TokenOpenBracket), nil case ']': return l.makeToken(TokenCloseBracket), nil case '{': return l.makeToken(TokenOpenBrace), nil case '}': return l.makeToken(TokenCloseBrace), nil case ';': return l.makeToken(TokenSemicolon), nil case ',': return l.makeToken(TokenComma), nil case '.': return l.makeToken(TokenDot), nil case ':': if l.accept('=') { return l.makeToken(TokenDeclare), nil } return l.makeToken(TokenColon), nil case '!': if l.accept('=') { return l.makeToken(TokenBangEquals), nil } return l.makeToken(TokenBang), nil case '=': if l.accept('=') { return l.makeToken(TokenEquals), nil } return l.makeToken(TokenAssign), nil case '>': if l.accept('=') { return l.makeToken(TokenGreaterThanOrEqual), nil } return l.makeToken(TokenGreaterThan), nil case '<': if l.accept('=') { return l.makeToken(TokenLessThanOrEqual), nil } return l.makeToken(TokenLessThan), nil case '&': if l.accept('&') { return l.makeToken(TokenDoubleAmpersand), nil } return l.makeToken(TokenError), errors.New("malformed token (got '&', expected '&' to follow)") case '|': if l.accept('|') { return l.makeToken(TokenDoublePipe), nil } return l.makeToken(TokenPipe), nil case '"': // include ending quote for !l.accept('"') { if l.match('\n') { return l.makeToken(TokenError), errors.New("string did not end in current line") } if l.isAtEnd() { return l.makeToken(TokenError), errors.New("string did not before end of source") } l.advance() } return l.makeToken(TokenString), nil case '\'': // include ending quote for !l.accept('\'') { if l.match('\n') { return l.makeToken(TokenError), errors.New("string did not end in current line") } if l.isAtEnd() { return l.makeToken(TokenError), errors.New("string did not before end of source") } l.advance() } return l.makeToken(TokenString), nil default: if l.isAlpha(c) { // assemble variable for l.isAlphaNumeric(l.peek()) { l.advance() } lexeme := string(l.src[l.start:l.current]) if k, ok := Keywords[lexeme]; ok { return l.makeToken(k), nil } return l.makeToken(TokenName), nil } else if c == '0' && l.peek() != '.' { if l.peek() == 'x' { l.advance() // hex for unicode.In(l.peek(), unicode.Hex_Digit) { l.advance() } return l.makeToken(TokenHexadecimal), nil } return l.makeToken(TokenInteger), nil } else if unicode.IsDigit(c) { for unicode.IsDigit(l.peek()) { l.advance() } // if the number has a float-part if l.accept('.') { for unicode.IsDigit(l.peek()) { l.advance() } return l.makeToken(TokenFloat), nil } return l.makeToken(TokenInteger), nil } return l.makeToken(TokenError), errors.New(fmt.Sprintf("invalid token %c", c)) } } func NewToken(t TokenKind, start Pos, end Pos, line Pos, lexeme string) Token { return Token{ Kind: t, Start: start, End: end, Line: line, Lexeme: lexeme, } } func (l *Lexer) Tokenize() ([]Token, error) { tokens := make([]Token, 0) tok, err := l.NextToken() for ; err == nil; tok, err = l.NextToken() { tokens = append(tokens, tok) if tok.Kind == TokenEOF { break } } return tokens, err } func (l *Lexer) makeToken(t TokenKind) Token { return NewToken(t, l.start, l.current, l.line, string(l.src[l.start:l.current])) } func (l *Lexer) peek() rune { if l.isAtEnd() { return 0 } return l.src[l.current] } func (l *Lexer) match(c rune) bool { return l.peek() == c } func (l *Lexer) accept(c rune) bool { if l.match(c) { l.advance() return true } return false } func (l *Lexer) isAlpha(c rune) bool { return unicode.IsLetter(c) || c == '_' } func (l *Lexer) isAlphaNumeric(c rune) bool { return l.isAlpha(c) || unicode.IsDigit(c) } func (l *Lexer) advance() { if l.isAtEnd() { return } if l.src[l.current] == '\n' { l.line++ } l.current++ } func (l *Lexer) isAtEnd() bool { return l.current >= Pos(len(l.src)) } func (l *Lexer) skipWhitespace() { for !l.isAtEnd() && unicode.IsSpace(l.peek()) && l.peek() != '\n' { l.advance() } }