diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index d17e19932..c32658637 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -219,3 +219,17 @@ func (l *lexer) scanString(quote rune) (n int) { } return } + +func (l *lexer) scanRawString(quote rune) (n int) { + ch := l.next() // read character after back tick + for ch != quote { + if ch == eof { + l.error("literal not terminated") + return + } + ch = l.next() + n++ + } + l.emitValue(String, l.input[l.start+1:l.end-1]) + return +} diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go index d4bed139e..feecf045d 100644 --- a/parser/lexer/lexer_test.go +++ b/parser/lexer/lexer_test.go @@ -49,6 +49,19 @@ func TestLex(t *testing.T) { {Kind: EOF}, }, }, + { + "`backtick` `hello\u263Aworld` `hello\n\tworld` `hello\"world'` `\xC3\xBF\u263A\U000003A8` `❤️`", + []Token{ + {Kind: String, Value: `backtick`}, + {Kind: String, Value: `hello☺world`}, + {Kind: String, Value: `hello + world`}, + {Kind: String, Value: `hello"world'`}, + {Kind: String, Value: `ÿ☺Ψ`}, + {Kind: String, Value: "❤️"}, + {Kind: EOF}, + }, + }, { "a and orb().val #.", []Token{ diff --git a/parser/lexer/state.go b/parser/lexer/state.go index 5d82329eb..9999fd3c5 100644 --- a/parser/lexer/state.go +++ b/parser/lexer/state.go @@ -23,6 +23,8 @@ func root(l *lexer) stateFn { l.error("%v", err) } l.emitValue(String, str) + case r == '`': + l.scanRawString(r) case '0' <= r && r <= '9': l.backup() return number diff --git a/parser/parser_test.go b/parser/parser_test.go index 2af7635ed..fbeb24696 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -24,6 +24,11 @@ func TestParse(t *testing.T) { `"str"`, &StringNode{Value: "str"}, }, + { + "`hello\nworld`", + &StringNode{Value: `hello +world`}, + }, { "3", &IntegerNode{Value: 3},