Skip to content
This repository was archived by the owner on Dec 15, 2022. It is now read-only.

Fix parsing of raw strings containing unclosed parentheses and brackets #37

Merged
merged 2 commits into from
Nov 7, 2014
Merged

Fix parsing of raw strings containing unclosed parentheses and brackets #37

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b0b3ca8
Fix parsing of raw strings containing unclosed parentheses and brackets
aroben Nov 6, 2014
c2120ec
Make sure things other than regexes get parsed inside raw strings
aroben Nov 6, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 42 additions & 86 deletions grammars/python.cson
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -994,36 +994,23 @@
]
}
{
'begin': '([uU]r)(")'
'beginCaptures':
'captures':
'1':
'name': 'storage.type.string.python'
'2':
'name': 'punctuation.definition.string.begin.python'
'comment': 'double-quoted raw string'
'end': '((?<=")(")|")|(\\n)'
'endCaptures':
'1':
'name': 'punctuation.definition.string.end.python'
'2':
'name': 'meta.empty-string.double.python'
'3':
'name': 'invalid.illegal.unclosed-string.python'
'patterns': [
'include': '#constant_placeholder'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will combine to a single object with duplicate include keys when compiled to JavaSript:

'patterns': [
    {
      'include': '#constant_placeholder',
      'include': '#escaped_unicode_char',
      'include': '#escaped_char',
      'include': '#regular_expressions'
    }
  ]

You probably want it to be:

'patterns': [
  {'include': '#constant_placeholder'}
  {'include': '#escaped_unicode_char'}
  {'include': '#escaped_char'}
  {'include': '#regular_expressions'}
]

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right. I guess I should make the tests sensitive to that bug.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed and tested.

'include': '#escaped_unicode_char'
'include': '#escaped_char'
'include': '#regular_expressions'
]
'4':
'name': 'punctuation.definition.string.end.python'
'comment': 'double-quoted raw string'
'match': '([uU]r)(")((?:[^"\\\\]|\\\\.)*)(")'
'name': 'string.quoted.double.single-line.unicode-raw-regex.python'
'patterns': [
{
'include': '#constant_placeholder'
}
{
'include': '#escaped_unicode_char'
}
{
'include': '#escaped_char'
}
{
'include': '#regular_expressions'
}
]
}
{
'begin': '([uU]R)(")'
Expand Down Expand Up @@ -1055,33 +1042,22 @@
]
}
{
'begin': '(r)(")'
'beginCaptures':
'captures':
'1':
'name': 'storage.type.string.python'
'2':
'name': 'punctuation.definition.string.begin.python'
'comment': 'double-quoted raw string'
'end': '((?<=")(")|")|(\\n)'
'endCaptures':
'1':
'name': 'punctuation.definition.string.end.python'
'2':
'name': 'meta.empty-string.double.python'
'3':
'name': 'invalid.illegal.unclosed-string.python'
'patterns': [
'include': '#constant_placeholder'
'include': '#escaped_char'
'include': '#regular_expressions'
]
'4':
'name': 'punctuation.definition.string.end.python'
'comment': 'double-quoted raw string'
'match': '(r)(")((?:[^"\\\\]|\\\\.)*)(")'
'name': 'string.quoted.double.single-line.raw-regex.python'
'patterns': [
{
'include': '#constant_placeholder'
}
{
'include': '#escaped_char'
}
{
'include': '#regular_expressions'
}
]
}
{
'begin': '(R)(")'
Expand Down Expand Up @@ -1386,34 +1362,23 @@
]
}
{
'begin': '([uU]r)(\')'
'beginCaptures':
'captures':
'1':
'name': 'storage.type.string.python'
'2':
'name': 'punctuation.definition.string.begin.python'
'comment': 'single quoted raw string'
'end': '(\')|(\\n)'
'endCaptures':
'1':
'3':
'patterns': [
'include': '#constant_placeholder'
'include': '#escaped_unicode_char'
'include': '#escaped_char'
'include': '#regular_expressions'
]
'4':
'name': 'punctuation.definition.string.end.python'
'2':
'name': 'invalid.illegal.unclosed-string.python'
'comment': 'single quoted raw string'
'match': '([uU]r)(\')((?:[^\'\\\\]|\\\\.)*)(\')'
'name': 'string.quoted.single.single-line.unicode-raw-regex.python'
'patterns': [
{
'include': '#constant_placeholder'
}
{
'include': '#escaped_unicode_char'
}
{
'include': '#escaped_char'
}
{
'include': '#regular_expressions'
}
]
}
{
'begin': '([uU]R)(\')'
Expand Down Expand Up @@ -1443,31 +1408,22 @@
]
}
{
'begin': '(r)(\')'
'beginCaptures':
'captures':
'1':
'name': 'storage.type.string.python'
'2':
'name': 'punctuation.definition.string.begin.python'
'comment': 'single quoted raw string'
'end': '(\')|(\\n)'
'endCaptures':
'1':
'3':
'patterns': [
'include': '#constant_placeholder'
'include': '#escaped_char'
'include': '#regular_expressions'
]
'4':
'name': 'punctuation.definition.string.end.python'
'2':
'name': 'invalid.illegal.unclosed-string.python'
'comment': 'single quoted raw string'
'match': '(r)(\')((?:[^\'\\\\]|\\\\.)*)(\')'
'name': 'string.quoted.single.single-line.raw-regex.python'
'patterns': [
{
'include': '#constant_placeholder'
}
{
'include': '#escaped_char'
}
{
'include': '#regular_expressions'
}
]
}
{
'begin': '(R)(\')'
Expand Down
144 changes: 144 additions & 0 deletions spec/python-spec.coffee
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,147 @@ describe "Python grammar", ->
expect(tokens[1][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.python', 'punctuation.definition.string.end.python']

expect(tokens[1][2]).not.toBeDefined()

it "terminates a single-quoted raw string containing opening parenthesis at closing quote", ->
tokens = grammar.tokenizeLines("r'(' #foo")

expect(tokens[0][0].value).toBe 'r'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe "'"
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '('
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
expect(tokens[0][3].value).toBe "'"
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][4].value).toBe ' '
expect(tokens[0][4].scopes).toEqual ['source.python']
expect(tokens[0][5].value).toBe '#'
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][6].value).toBe 'foo'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a single-quoted raw string containing opening bracket at closing quote", ->
tokens = grammar.tokenizeLines("r'[' #foo")

expect(tokens[0][0].value).toBe 'r'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe "'"
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '['
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
expect(tokens[0][3].value).toBe "'"
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][4].value).toBe ' '
expect(tokens[0][4].scopes).toEqual ['source.python']
expect(tokens[0][5].value).toBe '#'
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][6].value).toBe 'foo'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a double-quoted raw string containing opening parenthesis at closing quote", ->
tokens = grammar.tokenizeLines('r"(" #foo')

expect(tokens[0][0].value).toBe 'r'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe '"'
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '('
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
expect(tokens[0][3].value).toBe '"'
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][4].value).toBe ' '
expect(tokens[0][4].scopes).toEqual ['source.python']
expect(tokens[0][5].value).toBe '#'
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][6].value).toBe 'foo'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a double-quoted raw string containing opening bracket at closing quote", ->
tokens = grammar.tokenizeLines('r"[" #foo')

expect(tokens[0][0].value).toBe 'r'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe '"'
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '['
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
expect(tokens[0][3].value).toBe '"'
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][4].value).toBe ' '
expect(tokens[0][4].scopes).toEqual ['source.python']
expect(tokens[0][5].value).toBe '#'
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][6].value).toBe 'foo'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a unicode single-quoted raw string containing opening parenthesis at closing quote", ->
tokens = grammar.tokenizeLines("ur'(' #foo")

expect(tokens[0][0].value).toBe 'ur'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe "'"
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '('
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
expect(tokens[0][3].value).toBe "'"
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][4].value).toBe ' '
expect(tokens[0][4].scopes).toEqual ['source.python']
expect(tokens[0][5].value).toBe '#'
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][6].value).toBe 'foo'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a unicode single-quoted raw string containing opening bracket at closing quote", ->
tokens = grammar.tokenizeLines("ur'[' #foo")

expect(tokens[0][0].value).toBe 'ur'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe "'"
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '['
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
expect(tokens[0][3].value).toBe "'"
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.single.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][4].value).toBe ' '
expect(tokens[0][4].scopes).toEqual ['source.python']
expect(tokens[0][5].value).toBe '#'
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][6].value).toBe 'foo'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a unicode double-quoted raw string containing opening parenthesis at closing quote", ->
tokens = grammar.tokenizeLines('ur"(" #foo')

expect(tokens[0][0].value).toBe 'ur'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe '"'
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '('
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'meta.group.regexp', 'punctuation.definition.group.regexp']
expect(tokens[0][3].value).toBe '"'
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][4].value).toBe ' '
expect(tokens[0][4].scopes).toEqual ['source.python']
expect(tokens[0][5].value).toBe '#'
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][6].value).toBe 'foo'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']

it "terminates a unicode double-quoted raw string containing opening bracket at closing quote", ->
tokens = grammar.tokenizeLines('ur"[" #foo')

expect(tokens[0][0].value).toBe 'ur'
expect(tokens[0][0].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'storage.type.string.python']
expect(tokens[0][1].value).toBe '"'
expect(tokens[0][1].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.begin.python']
expect(tokens[0][2].value).toBe '['
expect(tokens[0][2].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'constant.other.character-class.set.regexp', 'punctuation.definition.character-class.regexp']
expect(tokens[0][3].value).toBe '"'
expect(tokens[0][3].scopes).toEqual ['source.python', 'string.quoted.double.single-line.unicode-raw-regex.python', 'punctuation.definition.string.end.python']
expect(tokens[0][4].value).toBe ' '
expect(tokens[0][4].scopes).toEqual ['source.python']
expect(tokens[0][5].value).toBe '#'
expect(tokens[0][5].scopes).toEqual ['source.python', 'comment.line.number-sign.python', 'punctuation.definition.comment.python']
expect(tokens[0][6].value).toBe 'foo'
expect(tokens[0][6].scopes).toEqual ['source.python', 'comment.line.number-sign.python']