neemek/anglais
1
1
Fork 0
Code Issues Pull requests Releases 1 Wiki Activity Actions

Create tokens from runes to allow unicode characters (e.g. greek, chinese)

Browse source
This commit is contained in:
Neemek 2024-12-28 17:04:30 +01:00
parent e1f438edbe
commit 74bf43d387
Signed by: neemek
GPG key ID: 28360A8951CD0E9B
2 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

14
core/lexer.go
View file

@ -150,7 +150,7 @@ func (t TokenType) String() string {
}
type Lexer struct {
src string
src []rune
start Pos
current Pos
line Pos
@ -158,7 +158,7 @@ type Lexer struct {
func NewLexer(src string) *Lexer {
return &Lexer{
src: src,
src: []rune(src),
start: 0,
current: 0,
line: 0,
@ -288,7 +288,7 @@ func (l *Lexer) NextToken() (Token, error) {
l.advance()
}
switch l.src[l.start:l.current] {
switch string(l.src[l.start:l.current]) {
case "true":
return l.makeToken(TokenTrue), nil
case "false":
@ -357,7 +357,7 @@ func (l *Lexer) Tokenize() ([]Token, error) {
}
func (l *Lexer) makeToken(t TokenType) Token {
return NewToken(t, l.start, l.current-l.start, l.line, l.src[l.start:l.current])
return NewToken(t, l.start, l.current-l.start, l.line, string(l.src[l.start:l.current]))
}
func (l *Lexer) peek() rune {
@ -365,7 +365,7 @@ func (l *Lexer) peek() rune {
return 0
}
return []rune(l.src)[l.current]
return l.src[l.current]
}
func (l *Lexer) match(c rune) bool {
@ -390,7 +390,7 @@ func (l *Lexer) advance() {
return
}
if []rune(l.src)[l.current] == '\n' {
if l.src[l.current] == '\n' {
l.line++
}
@ -398,7 +398,7 @@ func (l *Lexer) advance() {
}
func (l *Lexer) isAtEnd() bool {
return l.current >= Pos(len([]rune(l.src)))
return l.current >= Pos(len(l.src))
}
func (l *Lexer) skipWhitespace() {

2
core/lexer_test.go
View file

@ -167,7 +167,7 @@ func TestNewLexer(t *testing.T) {
t.Errorf("Lexer current position was not initialized correctly.")
}
if lex.src != "example source" {
if string(lex.src) != "example source" {
t.Errorf("Lexer lexer was not initialized correctly.")
}