neemek/anglais
1
1
Fork 0
Code Issues Pull requests Releases 1 Wiki Activity Actions

Create tokens from runes to allow unicode characters (e.g. greek, chinese)

Browse source
This commit is contained in:
Neemek 2024-12-28 17:04:30 +01:00
parent e1f438edbe
commit 74bf43d387
Signed by: neemek
GPG key ID: 28360A8951CD0E9B
2 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

14
core/lexer.go
View file

@ -150,7 +150,7 @@ func (t TokenType) String() string {
} }
type Lexer struct { type Lexer struct {
src string src []rune
start Pos start Pos
current Pos current Pos
line Pos line Pos
@ -158,7 +158,7 @@ type Lexer struct {
func NewLexer(src string) *Lexer { func NewLexer(src string) *Lexer {
return &Lexer{ return &Lexer{
src: src, src: []rune(src),
start: 0, start: 0,
current: 0, current: 0,
line: 0, line: 0,
@ -288,7 +288,7 @@ func (l *Lexer) NextToken() (Token, error) {
l.advance() l.advance()
} }
switch l.src[l.start:l.current] { switch string(l.src[l.start:l.current]) {
case "true": case "true":
return l.makeToken(TokenTrue), nil return l.makeToken(TokenTrue), nil
case "false": case "false":
@ -357,7 +357,7 @@ func (l *Lexer) Tokenize() ([]Token, error) {
} }
func (l *Lexer) makeToken(t TokenType) Token { func (l *Lexer) makeToken(t TokenType) Token {
return NewToken(t, l.start, l.current-l.start, l.line, l.src[l.start:l.current]) return NewToken(t, l.start, l.current-l.start, l.line, string(l.src[l.start:l.current]))
} }
func (l *Lexer) peek() rune { func (l *Lexer) peek() rune {
@ -365,7 +365,7 @@ func (l *Lexer) peek() rune {
return 0 return 0
} }
return []rune(l.src)[l.current] return l.src[l.current]
} }
func (l *Lexer) match(c rune) bool { func (l *Lexer) match(c rune) bool {
@ -390,7 +390,7 @@ func (l *Lexer) advance() {
return return
} }
if []rune(l.src)[l.current] == '\n' { if l.src[l.current] == '\n' {
l.line++ l.line++
} }
@ -398,7 +398,7 @@ func (l *Lexer) advance() {
} }
func (l *Lexer) isAtEnd() bool { func (l *Lexer) isAtEnd() bool {
return l.current >= Pos(len([]rune(l.src))) return l.current >= Pos(len(l.src))
} }
func (l *Lexer) skipWhitespace() { func (l *Lexer) skipWhitespace() {

2
core/lexer_test.go
View file

@ -167,7 +167,7 @@ func TestNewLexer(t *testing.T) {
t.Errorf("Lexer current position was not initialized correctly.") t.Errorf("Lexer current position was not initialized correctly.")
} }
if lex.src != "example source" { if string(lex.src) != "example source" {
t.Errorf("Lexer lexer was not initialized correctly.") t.Errorf("Lexer lexer was not initialized correctly.")
} }