Skip to content
This repository was archived by the owner on Dec 15, 2022. It is now read-only.

Fix parsing of raw strings containing unclosed parentheses and brackets #37

Merged
merged 2 commits into from
Nov 7, 2014
Merged

Fix parsing of raw strings containing unclosed parentheses and brackets #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b0b3ca8
Fix parsing of raw strings containing unclosed parentheses and brackets
aroben Nov 6, 2014
c2120ec
Make sure things other than regexes get parsed inside raw strings
aroben Nov 6, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 42 additions & 86 deletions grammars/python.cson
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -994,36 +994,23 @@
]
}
{
'begin': '([uU]r)(")'
'beginCaptures':
'captures':
'1':
'name': 'storage.type.string.python'
'2':
'name': 'punctuation.definition.string.begin.python'
'comment': 'double-quoted raw string'
'end': '((?<=")(")|")|(\\n)'
'endCaptures':
'1':
'name': 'punctuation.definition.string.end.python'
'2':
'name': 'meta.empty-string.double.python'
'3':
'name': 'invalid.illegal.unclosed-string.python'
'patterns': [
{'include': '#constant_placeholder'}
{'include': '#escaped_unicode_char'}
{'include': '#escaped_char'}
{'include': '#regular_expressions'}
]
'4':
'name': 'punctuation.definition.string.end.python'
'comment': 'double-quoted raw string'
'match': '([uU]r)(")((?:[^"\\\\]|\\\\.)*)(")'
'name': 'string.quoted.double.single-line.unicode-raw-regex.python'
'patterns': [
{
'include': '#constant_placeholder'
}
{
'include': '#escaped_unicode_char'
}
{
'include': '#escaped_char'
}
{
'include': '#regular_expressions'
}
]
}
{
'begin': '([uU]R)(")'
Expand Down Expand Up @@ -1055,33 +1042,22 @@
]
}
{
'begin': '(r)(")'
'beginCaptures':
'captures':
'1':
'name': 'storage.type.string.python'
'2':
'name': 'punctuation.definition.string.begin.python'
'comment': 'double-quoted raw string'
'end': '((?<=")(")|")|(\\n)'
'endCaptures':
'1':
'name': 'punctuation.definition.string.end.python'
'2':
'name': 'meta.empty-string.double.python'
'3':
'name': 'invalid.illegal.unclosed-string.python'
'patterns': [
{'include': '#constant_placeholder'}
{'include': '#escaped_char'}
{'include': '#regular_expressions'}
]
'4':
'name': 'punctuation.definition.string.end.python'
'comment': 'double-quoted raw string'
'match': '(r)(")((?:[^"\\\\]|\\\\.)*)(")'
'name': 'string.quoted.double.single-line.raw-regex.python'
'patterns': [
{
'include': '#constant_placeholder'
}
{
'include': '#escaped_char'
}
{
'include': '#regular_expressions'
}
]
}
{
'begin': '(R)(")'
Expand Down Expand Up @@ -1386,34 +1362,23 @@
]
}
{
'begin': '([uU]r)(\')'
'beginCaptures':
'captures':
'1':
'name': 'storage.type.string.python'
'2':
'name': 'punctuation.definition.string.begin.python'
'comment': 'single quoted raw string'
'end': '(\')|(\\n)'
'endCaptures':
'1':
'3':
'patterns': [
{'include': '#constant_placeholder'}
{'include': '#escaped_unicode_char'}
{'include': '#escaped_char'}
{'include': '#regular_expressions'}
]
'4':
'name': 'punctuation.definition.string.end.python'
'2':
'name': 'invalid.illegal.unclosed-string.python'
'comment': 'single quoted raw string'
'match': '([uU]r)(\')((?:[^\'\\\\]|\\\\.)*)(\')'
'name': 'string.quoted.single.single-line.unicode-raw-regex.python'
'patterns': [
{
'include': '#constant_placeholder'
}
{
'include': '#escaped_unicode_char'
}
{
'include': '#escaped_char'
}
{
'include': '#regular_expressions'
}
]
}
{
'begin': '([uU]R)(\')'
Expand Down Expand Up @@ -1443,31 +1408,22 @@
]
}
{
'begin': '(r)(\')'
'beginCaptures':
'captures':
'1':
'name': 'storage.type.string.python'
'2':
'name': 'punctuation.definition.string.begin.python'
'comment': 'single quoted raw string'
'end': '(\')|(\\n)'
'endCaptures':
'1':
'3':
'patterns': [
{'include': '#constant_placeholder'}
{'include': '#escaped_char'}
{'include': '#regular_expressions'}
]
'4':
'name': 'punctuation.definition.string.end.python'
'2':
'name': 'invalid.illegal.unclosed-string.python'
'comment': 'single quoted raw string'
'match': '(r)(\')((?:[^\'\\\\]|\\\\.)*)(\')'
'name': 'string.quoted.single.single-line.raw-regex.python'
'patterns': [
{
'include': '#constant_placeholder'
}
{
'include': '#escaped_char'
}
{
'include': '#regular_expressions'
}
]
}
{
'begin': '(R)(\')'
Expand Down
160 changes: 160 additions & 0 deletions spec/python-spec.coffee
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,163 @@ describe "Python grammar", ->
expect(tokens[1][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.python', 'punctuation.definition.string.end.python']

expect(tokens[1][2]).not.toBeDefined()

it "terminates a single-quoted raw string containing opening parenthesis at closing quote", ->
tokens = grammar.tokenizeLines("r'%d(' #foo")

expect(tokens[0][0].value).toBe 'r'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe "'"
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '%d'
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'constant.other.placeholder.python']
expect(tokens[0][3].value).toBe '('
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
expect(tokens[0][4].value).toBe "'"
expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][5].value).toBe ' '
expect(tokens[0][5].scopes).toEqual ['source.python']
expect(tokens[0][6].value).toBe '#'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][7].value).toBe 'foo'
expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a single-quoted raw string containing opening bracket at closing quote", ->
tokens = grammar.tokenizeLines("r'%d[' #foo")

expect(tokens[0][0].value).toBe 'r'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe "'"
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '%d'
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'constant.other.placeholder.python']
expect(tokens[0][3].value).toBe '['
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
expect(tokens[0][4].value).toBe "'"
expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][5].value).toBe ' '
expect(tokens[0][5].scopes).toEqual ['source.python']
expect(tokens[0][6].value).toBe '#'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][7].value).toBe 'foo'
expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a double-quoted raw string containing opening parenthesis at closing quote", ->
tokens = grammar.tokenizeLines('r"%d(" #foo')

expect(tokens[0][0].value).toBe 'r'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe '"'
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '%d'
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'constant.other.placeholder.python']
expect(tokens[0][3].value).toBe '('
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
expect(tokens[0][4].value).toBe '"'
expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][5].value).toBe ' '
expect(tokens[0][5].scopes).toEqual ['source.python']
expect(tokens[0][6].value).toBe '#'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][7].value).toBe 'foo'
expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a double-quoted raw string containing opening bracket at closing quote", ->
tokens = grammar.tokenizeLines('r"%d[" #foo')

expect(tokens[0][0].value).toBe 'r'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe '"'
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '%d'
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'constant.other.placeholder.python']
expect(tokens[0][3].value).toBe '['
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
expect(tokens[0][4].value).toBe '"'
expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][5].value).toBe ' '
expect(tokens[0][5].scopes).toEqual ['source.python']
expect(tokens[0][6].value).toBe '#'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][7].value).toBe 'foo'
expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a unicode single-quoted raw string containing opening parenthesis at closing quote", ->
tokens = grammar.tokenizeLines("ur'%d(' #foo")

expect(tokens[0][0].value).toBe 'ur'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe "'"
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '%d'
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'constant.other.placeholder.python']
expect(tokens[0][3].value).toBe '('
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
expect(tokens[0][4].value).toBe "'"
expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][5].value).toBe ' '
expect(tokens[0][5].scopes).toEqual ['source.python']
expect(tokens[0][6].value).toBe '#'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][7].value).toBe 'foo'
expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a unicode single-quoted raw string containing opening bracket at closing quote", ->
tokens = grammar.tokenizeLines("ur'%d[' #foo")

expect(tokens[0][0].value).toBe 'ur'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe "'"
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '%d'
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'constant.other.placeholder.python']
expect(tokens[0][3].value).toBe '['
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
expect(tokens[0][4].value).toBe "'"
expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][5].value).toBe ' '
expect(tokens[0][5].scopes).toEqual ['source.python']
expect(tokens[0][6].value).toBe '#'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][7].value).toBe 'foo'
expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a unicode double-quoted raw string containing opening parenthesis at closing quote", ->
tokens = grammar.tokenizeLines('ur"%d(" #foo')

expect(tokens[0][0].value).toBe 'ur'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe '"'
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '%d'
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'constant.other.placeholder.python']
expect(tokens[0][3].value).toBe '('
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
expect(tokens[0][4].value).toBe '"'
expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][5].value).toBe ' '
expect(tokens[0][5].scopes).toEqual ['source.python']
expect(tokens[0][6].value).toBe '#'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][7].value).toBe 'foo'
expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a unicode double-quoted raw string containing opening bracket at closing quote", ->
tokens = grammar.tokenizeLines('ur"%d[" #foo')

expect(tokens[0][0].value).toBe 'ur'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe '"'
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '%d'
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'constant.other.placeholder.python']
expect(tokens[0][3].value).toBe '['
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
expect(tokens[0][4].value).toBe '"'
expect(tokens[0][4].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][5].value).toBe ' '
expect(tokens[0][5].scopes).toEqual ['source.python']
expect(tokens[0][6].value).toBe '#'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][7].value).toBe 'foo'
expect(tokens[0][7].scopes).toEqual ['source.python', 'comment.line.number-sign.python']