From b0b3ca86ffbaa80f32bcd406e04b2c82dd918c50 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Thu, 6 Nov 2014 11:15:33 -0500 Subject: [PATCH 1/2] Fix parsing of raw strings containing unclosed parentheses and brackets Strings like r'[' and r'(' were causing the rest of the document to be parsed as if it were inside the regex. Now we terminate the regex at the closing quote in all cases. This is a backport of https://github.com/textmate/python.tmbundle/commit/2567262452b389f7042ef3026e0443f6336d1735 --- grammars/python.cson | 128 ++++++++++++----------------------- spec/python-spec.coffee | 144 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 86 deletions(-) diff --git a/grammars/python.cson b/grammars/python.cson index df8bf91..10c4a5e 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -994,36 +994,23 @@ ] } { - 'begin': '([uU]r)(")' - 'beginCaptures': + 'captures': '1': 'name': 'storage.type.string.python' '2': 'name': 'punctuation.definition.string.begin.python' - 'comment': 'double-quoted raw string' - 'end': '((?<=")(")|")|(\\n)' - 'endCaptures': - '1': - 'name': 'punctuation.definition.string.end.python' - '2': - 'name': 'meta.empty-string.double.python' '3': - 'name': 'invalid.illegal.unclosed-string.python' + 'patterns': [ + 'include': '#constant_placeholder' + 'include': '#escaped_unicode_char' + 'include': '#escaped_char' + 'include': '#regular_expressions' + ] + '4': + 'name': 'punctuation.definition.string.end.python' + 'comment': 'double-quoted raw string' + 'match': '([uU]r)(")((?:[^"\\\\]|\\\\.)*)(")' 'name': 'string.quoted.double.single-line.unicode-raw-regex.python' - 'patterns': [ - { - 'include': '#constant_placeholder' - } - { - 'include': '#escaped_unicode_char' - } - { - 'include': '#escaped_char' - } - { - 'include': '#regular_expressions' - } - ] } { 'begin': '([uU]R)(")' @@ -1055,33 +1042,22 @@ ] } { - 'begin': '(r)(")' - 'beginCaptures': + 'captures': '1': 'name': 'storage.type.string.python' '2': 'name': 'punctuation.definition.string.begin.python' - 'comment': 'double-quoted raw string' - 'end': '((?<=")(")|")|(\\n)' - 'endCaptures': - '1': - 'name': 'punctuation.definition.string.end.python' - '2': - 'name': 'meta.empty-string.double.python' '3': - 'name': 'invalid.illegal.unclosed-string.python' + 'patterns': [ + 'include': '#constant_placeholder' + 'include': '#escaped_char' + 'include': '#regular_expressions' + ] + '4': + 'name': 'punctuation.definition.string.end.python' + 'comment': 'double-quoted raw string' + 'match': '(r)(")((?:[^"\\\\]|\\\\.)*)(")' 'name': 'string.quoted.double.single-line.raw-regex.python' - 'patterns': [ - { - 'include': '#constant_placeholder' - } - { - 'include': '#escaped_char' - } - { - 'include': '#regular_expressions' - } - ] } { 'begin': '(R)(")' @@ -1386,34 +1362,23 @@ ] } { - 'begin': '([uU]r)(\')' - 'beginCaptures': + 'captures': '1': 'name': 'storage.type.string.python' '2': 'name': 'punctuation.definition.string.begin.python' - 'comment': 'single quoted raw string' - 'end': '(\')|(\\n)' - 'endCaptures': - '1': + '3': + 'patterns': [ + 'include': '#constant_placeholder' + 'include': '#escaped_unicode_char' + 'include': '#escaped_char' + 'include': '#regular_expressions' + ] + '4': 'name': 'punctuation.definition.string.end.python' - '2': - 'name': 'invalid.illegal.unclosed-string.python' + 'comment': 'single quoted raw string' + 'match': '([uU]r)(\')((?:[^\'\\\\]|\\\\.)*)(\')' 'name': 'string.quoted.single.single-line.unicode-raw-regex.python' - 'patterns': [ - { - 'include': '#constant_placeholder' - } - { - 'include': '#escaped_unicode_char' - } - { - 'include': '#escaped_char' - } - { - 'include': '#regular_expressions' - } - ] } { 'begin': '([uU]R)(\')' @@ -1443,31 +1408,22 @@ ] } { - 'begin': '(r)(\')' - 'beginCaptures': + 'captures': '1': 'name': 'storage.type.string.python' '2': 'name': 'punctuation.definition.string.begin.python' - 'comment': 'single quoted raw string' - 'end': '(\')|(\\n)' - 'endCaptures': - '1': + '3': + 'patterns': [ + 'include': '#constant_placeholder' + 'include': '#escaped_char' + 'include': '#regular_expressions' + ] + '4': 'name': 'punctuation.definition.string.end.python' - '2': - 'name': 'invalid.illegal.unclosed-string.python' + 'comment': 'single quoted raw string' + 'match': '(r)(\')((?:[^\'\\\\]|\\\\.)*)(\')' 'name': 'string.quoted.single.single-line.raw-regex.python' - 'patterns': [ - { - 'include': '#constant_placeholder' - } - { - 'include': '#escaped_char' - } - { - 'include': '#regular_expressions' - } - ] } { 'begin': '(R)(\')' diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index 18fc213..4467286 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -35,3 +35,147 @@ describe "Python grammar", -> expect(tokens[1][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.python', 'punctuation.definition.string.end.python'] expect(tokens[1][2]).not.toBeDefined() + + it "terminates a single-quoted raw string containing opening parenthesis at closing quote", -> + tokens = grammar.tokenizeLines("r'(' #foo") + + expect(tokens[0][0].value).toBe 'r' + expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'storage.type.string.python'] + expect(tokens[0][1].value).toBe "'" + expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.begin.python'] + expect(tokens[0][2].value).toBe '(' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[0][3].value).toBe "'" + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][4].value).toBe ' ' + expect(tokens[0][4].scopes).toEqual ['source.python'] + expect(tokens[0][5].value).toBe '#' + expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][6].value).toBe 'foo' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + + it "terminates a single-quoted raw string containing opening bracket at closing quote", -> + tokens = grammar.tokenizeLines("r'[' #foo") + + expect(tokens[0][0].value).toBe 'r' + expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'storage.type.string.python'] + expect(tokens[0][1].value).toBe "'" + expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.begin.python'] + expect(tokens[0][2].value).toBe '[' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] + expect(tokens[0][3].value).toBe "'" + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][4].value).toBe ' ' + expect(tokens[0][4].scopes).toEqual ['source.python'] + expect(tokens[0][5].value).toBe '#' + expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][6].value).toBe 'foo' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + + it "terminates a double-quoted raw string containing opening parenthesis at closing quote", -> + tokens = grammar.tokenizeLines('r"(" #foo') + + expect(tokens[0][0].value).toBe 'r' + expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'storage.type.string.python'] + expect(tokens[0][1].value).toBe '"' + expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.begin.python'] + expect(tokens[0][2].value).toBe '(' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[0][3].value).toBe '"' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][4].value).toBe ' ' + expect(tokens[0][4].scopes).toEqual ['source.python'] + expect(tokens[0][5].value).toBe '#' + expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][6].value).toBe 'foo' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + + it "terminates a double-quoted raw string containing opening bracket at closing quote", -> + tokens = grammar.tokenizeLines('r"[" #foo') + + expect(tokens[0][0].value).toBe 'r' + expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'storage.type.string.python'] + expect(tokens[0][1].value).toBe '"' + expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.begin.python'] + expect(tokens[0][2].value).toBe '[' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] + expect(tokens[0][3].value).toBe '"' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][4].value).toBe ' ' + expect(tokens[0][4].scopes).toEqual ['source.python'] + expect(tokens[0][5].value).toBe '#' + expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][6].value).toBe 'foo' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + + it "terminates a unicode single-quoted raw string containing opening parenthesis at closing quote", -> + tokens = grammar.tokenizeLines("ur'(' #foo") + + expect(tokens[0][0].value).toBe 'ur' + expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'storage.type.string.python'] + expect(tokens[0][1].value).toBe "'" + expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python'] + expect(tokens[0][2].value).toBe '(' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[0][3].value).toBe "'" + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][4].value).toBe ' ' + expect(tokens[0][4].scopes).toEqual ['source.python'] + expect(tokens[0][5].value).toBe '#' + expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][6].value).toBe 'foo' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + + it "terminates a unicode single-quoted raw string containing opening bracket at closing quote", -> + tokens = grammar.tokenizeLines("ur'[' #foo") + + expect(tokens[0][0].value).toBe 'ur' + expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'storage.type.string.python'] + expect(tokens[0][1].value).toBe "'" + expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python'] + expect(tokens[0][2].value).toBe '[' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] + expect(tokens[0][3].value).toBe "'" + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][4].value).toBe ' ' + expect(tokens[0][4].scopes).toEqual ['source.python'] + expect(tokens[0][5].value).toBe '#' + expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][6].value).toBe 'foo' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + + it "terminates a unicode double-quoted raw string containing opening parenthesis at closing quote", -> + tokens = grammar.tokenizeLines('ur"(" #foo') + + expect(tokens[0][0].value).toBe 'ur' + expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'storage.type.string.python'] + expect(tokens[0][1].value).toBe '"' + expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python'] + expect(tokens[0][2].value).toBe '(' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[0][3].value).toBe '"' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][4].value).toBe ' ' + expect(tokens[0][4].scopes).toEqual ['source.python'] + expect(tokens[0][5].value).toBe '#' + expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][6].value).toBe 'foo' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + + it "terminates a unicode double-quoted raw string containing opening bracket at closing quote", -> + tokens = grammar.tokenizeLines('ur"[" #foo') + + expect(tokens[0][0].value).toBe 'ur' + expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'storage.type.string.python'] + expect(tokens[0][1].value).toBe '"' + expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python'] + expect(tokens[0][2].value).toBe '[' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] + expect(tokens[0][3].value).toBe '"' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][4].value).toBe ' ' + expect(tokens[0][4].scopes).toEqual ['source.python'] + expect(tokens[0][5].value).toBe '#' + expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][6].value).toBe 'foo' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] From c2120ec667049b07940a667ecbc02be4785fb714 Mon Sep 17 00:00:00 2001 From: Adam Roben Date: Thu, 6 Nov 2014 12:40:32 -0500 Subject: [PATCH 2/2] Make sure things other than regexes get parsed inside raw strings My CSON formatting was bad, which caused only the regular expression grammar to get parsed inside of raw strings. --- grammars/python.cson | 28 +++--- spec/python-spec.coffee | 192 ++++++++++++++++++++++------------------ 2 files changed, 118 insertions(+), 102 deletions(-) diff --git a/grammars/python.cson b/grammars/python.cson index 10c4a5e..c0757fa 100644 --- a/grammars/python.cson +++ b/grammars/python.cson @@ -1001,10 +1001,10 @@ 'name': 'punctuation.definition.string.begin.python' '3': 'patterns': [ - 'include': '#constant_placeholder' - 'include': '#escaped_unicode_char' - 'include': '#escaped_char' - 'include': '#regular_expressions' + {'include': '#constant_placeholder'} + {'include': '#escaped_unicode_char'} + {'include': '#escaped_char'} + {'include': '#regular_expressions'} ] '4': 'name': 'punctuation.definition.string.end.python' @@ -1049,9 +1049,9 @@ 'name': 'punctuation.definition.string.begin.python' '3': 'patterns': [ - 'include': '#constant_placeholder' - 'include': '#escaped_char' - 'include': '#regular_expressions' + {'include': '#constant_placeholder'} + {'include': '#escaped_char'} + {'include': '#regular_expressions'} ] '4': 'name': 'punctuation.definition.string.end.python' @@ -1369,10 +1369,10 @@ 'name': 'punctuation.definition.string.begin.python' '3': 'patterns': [ - 'include': '#constant_placeholder' - 'include': '#escaped_unicode_char' - 'include': '#escaped_char' - 'include': '#regular_expressions' + {'include': '#constant_placeholder'} + {'include': '#escaped_unicode_char'} + {'include': '#escaped_char'} + {'include': '#regular_expressions'} ] '4': 'name': 'punctuation.definition.string.end.python' @@ -1415,9 +1415,9 @@ 'name': 'punctuation.definition.string.begin.python' '3': 'patterns': [ - 'include': '#constant_placeholder' - 'include': '#escaped_char' - 'include': '#regular_expressions' + {'include': '#constant_placeholder'} + {'include': '#escaped_char'} + {'include': '#regular_expressions'} ] '4': 'name': 'punctuation.definition.string.end.python' diff --git a/spec/python-spec.coffee b/spec/python-spec.coffee index 4467286..28f1b5d 100644 --- a/spec/python-spec.coffee +++ b/spec/python-spec.coffee @@ -37,145 +37,161 @@ describe "Python grammar", -> expect(tokens[1][2]).not.toBeDefined() it "terminates a single-quoted raw string containing opening parenthesis at closing quote", -> - tokens = grammar.tokenizeLines("r'(' #foo") + tokens = grammar.tokenizeLines("r'%d(' #foo") expect(tokens[0][0].value).toBe 'r' expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'storage.type.string.python'] expect(tokens[0][1].value).toBe "'" expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.begin.python'] - expect(tokens[0][2].value).toBe '(' - expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] - expect(tokens[0][3].value).toBe "'" - expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] - expect(tokens[0][4].value).toBe ' ' - expect(tokens[0][4].scopes).toEqual ['source.python'] - expect(tokens[0][5].value).toBe '#' - expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] - expect(tokens[0][6].value).toBe 'foo' - expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + expect(tokens[0][2].value).toBe '%d' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'constant.other.placeholder.python'] + expect(tokens[0][3].value).toBe '(' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[0][4].value).toBe "'" + expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][5].value).toBe ' ' + expect(tokens[0][5].scopes).toEqual ['source.python'] + expect(tokens[0][6].value).toBe '#' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][7].value).toBe 'foo' + expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] it "terminates a single-quoted raw string containing opening bracket at closing quote", -> - tokens = grammar.tokenizeLines("r'[' #foo") + tokens = grammar.tokenizeLines("r'%d[' #foo") expect(tokens[0][0].value).toBe 'r' expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'storage.type.string.python'] expect(tokens[0][1].value).toBe "'" expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.begin.python'] - expect(tokens[0][2].value).toBe '[' - expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] - expect(tokens[0][3].value).toBe "'" - expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] - expect(tokens[0][4].value).toBe ' ' - expect(tokens[0][4].scopes).toEqual ['source.python'] - expect(tokens[0][5].value).toBe '#' - expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] - expect(tokens[0][6].value).toBe 'foo' - expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + expect(tokens[0][2].value).toBe '%d' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'constant.other.placeholder.python'] + expect(tokens[0][3].value).toBe '[' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] + expect(tokens[0][4].value).toBe "'" + expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][5].value).toBe ' ' + expect(tokens[0][5].scopes).toEqual ['source.python'] + expect(tokens[0][6].value).toBe '#' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][7].value).toBe 'foo' + expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] it "terminates a double-quoted raw string containing opening parenthesis at closing quote", -> - tokens = grammar.tokenizeLines('r"(" #foo') + tokens = grammar.tokenizeLines('r"%d(" #foo') expect(tokens[0][0].value).toBe 'r' expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'storage.type.string.python'] expect(tokens[0][1].value).toBe '"' expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.begin.python'] - expect(tokens[0][2].value).toBe '(' - expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] - expect(tokens[0][3].value).toBe '"' - expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] - expect(tokens[0][4].value).toBe ' ' - expect(tokens[0][4].scopes).toEqual ['source.python'] - expect(tokens[0][5].value).toBe '#' - expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] - expect(tokens[0][6].value).toBe 'foo' - expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + expect(tokens[0][2].value).toBe '%d' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'constant.other.placeholder.python'] + expect(tokens[0][3].value).toBe '(' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[0][4].value).toBe '"' + expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][5].value).toBe ' ' + expect(tokens[0][5].scopes).toEqual ['source.python'] + expect(tokens[0][6].value).toBe '#' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][7].value).toBe 'foo' + expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] it "terminates a double-quoted raw string containing opening bracket at closing quote", -> - tokens = grammar.tokenizeLines('r"[" #foo') + tokens = grammar.tokenizeLines('r"%d[" #foo') expect(tokens[0][0].value).toBe 'r' expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'storage.type.string.python'] expect(tokens[0][1].value).toBe '"' expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.begin.python'] - expect(tokens[0][2].value).toBe '[' - expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] - expect(tokens[0][3].value).toBe '"' - expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] - expect(tokens[0][4].value).toBe ' ' - expect(tokens[0][4].scopes).toEqual ['source.python'] - expect(tokens[0][5].value).toBe '#' - expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] - expect(tokens[0][6].value).toBe 'foo' - expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + expect(tokens[0][2].value).toBe '%d' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'constant.other.placeholder.python'] + expect(tokens[0][3].value).toBe '[' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] + expect(tokens[0][4].value).toBe '"' + expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][5].value).toBe ' ' + expect(tokens[0][5].scopes).toEqual ['source.python'] + expect(tokens[0][6].value).toBe '#' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][7].value).toBe 'foo' + expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] it "terminates a unicode single-quoted raw string containing opening parenthesis at closing quote", -> - tokens = grammar.tokenizeLines("ur'(' #foo") + tokens = grammar.tokenizeLines("ur'%d(' #foo") expect(tokens[0][0].value).toBe 'ur' expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'storage.type.string.python'] expect(tokens[0][1].value).toBe "'" expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python'] - expect(tokens[0][2].value).toBe '(' - expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] - expect(tokens[0][3].value).toBe "'" - expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] - expect(tokens[0][4].value).toBe ' ' - expect(tokens[0][4].scopes).toEqual ['source.python'] - expect(tokens[0][5].value).toBe '#' - expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] - expect(tokens[0][6].value).toBe 'foo' - expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + expect(tokens[0][2].value).toBe '%d' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'constant.other.placeholder.python'] + expect(tokens[0][3].value).toBe '(' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[0][4].value).toBe "'" + expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][5].value).toBe ' ' + expect(tokens[0][5].scopes).toEqual ['source.python'] + expect(tokens[0][6].value).toBe '#' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][7].value).toBe 'foo' + expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] it "terminates a unicode single-quoted raw string containing opening bracket at closing quote", -> - tokens = grammar.tokenizeLines("ur'[' #foo") + tokens = grammar.tokenizeLines("ur'%d[' #foo") expect(tokens[0][0].value).toBe 'ur' expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'storage.type.string.python'] expect(tokens[0][1].value).toBe "'" expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python'] - expect(tokens[0][2].value).toBe '[' - expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] - expect(tokens[0][3].value).toBe "'" - expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] - expect(tokens[0][4].value).toBe ' ' - expect(tokens[0][4].scopes).toEqual ['source.python'] - expect(tokens[0][5].value).toBe '#' - expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] - expect(tokens[0][6].value).toBe 'foo' - expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + expect(tokens[0][2].value).toBe '%d' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'constant.other.placeholder.python'] + expect(tokens[0][3].value).toBe '[' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] + expect(tokens[0][4].value).toBe "'" + expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][5].value).toBe ' ' + expect(tokens[0][5].scopes).toEqual ['source.python'] + expect(tokens[0][6].value).toBe '#' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][7].value).toBe 'foo' + expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] it "terminates a unicode double-quoted raw string containing opening parenthesis at closing quote", -> - tokens = grammar.tokenizeLines('ur"(" #foo') + tokens = grammar.tokenizeLines('ur"%d(" #foo') expect(tokens[0][0].value).toBe 'ur' expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'storage.type.string.python'] expect(tokens[0][1].value).toBe '"' expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python'] - expect(tokens[0][2].value).toBe '(' - expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] - expect(tokens[0][3].value).toBe '"' - expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] - expect(tokens[0][4].value).toBe ' ' - expect(tokens[0][4].scopes).toEqual ['source.python'] - expect(tokens[0][5].value).toBe '#' - expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] - expect(tokens[0][6].value).toBe 'foo' - expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + expect(tokens[0][2].value).toBe '%d' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'constant.other.placeholder.python'] + expect(tokens[0][3].value).toBe '(' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp'] + expect(tokens[0][4].value).toBe '"' + expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][5].value).toBe ' ' + expect(tokens[0][5].scopes).toEqual ['source.python'] + expect(tokens[0][6].value).toBe '#' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][7].value).toBe 'foo' + expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] it "terminates a unicode double-quoted raw string containing opening bracket at closing quote", -> - tokens = grammar.tokenizeLines('ur"[" #foo') + tokens = grammar.tokenizeLines('ur"%d[" #foo') expect(tokens[0][0].value).toBe 'ur' expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'storage.type.string.python'] expect(tokens[0][1].value).toBe '"' expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python'] - expect(tokens[0][2].value).toBe '[' - expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] - expect(tokens[0][3].value).toBe '"' - expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] - expect(tokens[0][4].value).toBe ' ' - expect(tokens[0][4].scopes).toEqual ['source.python'] - expect(tokens[0][5].value).toBe '#' - expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] - expect(tokens[0][6].value).toBe 'foo' - expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python'] + expect(tokens[0][2].value).toBe '%d' + expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'constant.other.placeholder.python'] + expect(tokens[0][3].value).toBe '[' + expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp'] + expect(tokens[0][4].value).toBe '"' + expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python'] + expect(tokens[0][5].value).toBe ' ' + expect(tokens[0][5].scopes).toEqual ['source.python'] + expect(tokens[0][6].value).toBe '#' + expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python'] + expect(tokens[0][7].value).toBe 'foo' + expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python']