Skip to content

Commit e376a66

Browse files
mysticateamarijnh
mysticatea
authored and
marijnh
committed
add numeric separators
1 parent d20ade2 commit e376a66

File tree

6 files changed

+247
-13
lines changed

6 files changed

+247
-13
lines changed

‎acorn-loose/src/expression.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ lp.parseExprAtom = function() {
266266
node = this.startNode()
267267
node.value = this.tok.value
268268
node.raw = this.input.slice(this.tok.start, this.tok.end)
269-
if (this.tok.type === tt.num && node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1)
269+
if (this.tok.type === tt.num && node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1).replace(/_/g, "")
270270
this.next()
271271
return this.finishNode(node, "Literal")
272272

‎acorn/src/expression.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -517,7 +517,7 @@ pp.parseLiteral = function(value) {
517517
let node = this.startNode()
518518
node.value = value
519519
node.raw = this.input.slice(this.start, this.end)
520-
if (node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1)
520+
if (node.raw.charCodeAt(node.raw.length - 1) === 110) node.bigint = node.raw.slice(0, -1).replace(/_/g, "")
521521
this.next()
522522
return this.finishNode(node, "Literal")
523523
}

‎acorn/src/tokenize.js‎

Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -438,30 +438,67 @@ pp.readRegexp = function() {
438438
// were read, the integer value otherwise. When `len` is given, this
439439
// will return `null` unless the integer has exactly `len` digits.
440440

441-
pp.readInt = function(radix, len) {
442-
let start = this.pos, total = 0
443-
for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
441+
pp.readInt = function(radix, len, maybeLegacyOctalNumericLiteral) {
442+
// `len` is used for character escape sequences. In that case, disallow separators.
443+
const allowSeparators = this.options.ecmaVersion >= 12 && len === undefined
444+
445+
// `maybeLegacyOctalNumericLiteral` is true if it doesn't have prefix (0x,0o,0b)
446+
// and isn't fraction part nor exponent part. In that case, if the first digit
447+
// is zero then disallow separators.
448+
const isLegacyOctalNumericLiteral = maybeLegacyOctalNumericLiteral && this.input.charCodeAt(this.pos) === 48
449+
450+
let start = this.pos, total = 0, lastCode = 0
451+
for (let i = 0, e = len == null ? Infinity : len; i < e; ++i, ++this.pos) {
444452
let code = this.input.charCodeAt(this.pos), val
453+
454+
if (allowSeparators && code === 95) {
455+
if (isLegacyOctalNumericLiteral) this.raiseRecoverable(this.pos, "Numeric separator is not allowed in legacy octal numeric literals")
456+
if (lastCode === 95) this.raiseRecoverable(this.pos, "Numeric separator must be exactly one underscore")
457+
if (i === 0) this.raiseRecoverable(this.pos, "Numeric separator is not allowed at the first of digits")
458+
lastCode = code
459+
continue
460+
}
461+
445462
if (code >= 97) val = code - 97 + 10 // a
446463
else if (code >= 65) val = code - 65 + 10 // A
447464
else if (code >= 48 && code <= 57) val = code - 48 // 0-9
448465
else val = Infinity
449466
if (val >= radix) break
450-
++this.pos
467+
lastCode = code
451468
total = total * radix + val
452469
}
470+
471+
if (allowSeparators && lastCode === 95) this.raiseRecoverable(this.pos - 1, "Numeric separator is not allowed at the last of digits")
453472
if (this.pos === start || len != null && this.pos - start !== len) return null
454473

455474
return total
456475
}
457476

477+
function stringToNumber(str, isLegacyOctalNumericLiteral) {
478+
if (isLegacyOctalNumericLiteral) {
479+
return parseInt(str, 8)
480+
}
481+
482+
// `parseFloat(value)` stops parsing at the first numeric separator then returns a wrong value.
483+
return parseFloat(str.replace(/_/g, ""))
484+
}
485+
486+
function stringToBigInt(str) {
487+
if (typeof BigInt !== "function") {
488+
return null
489+
}
490+
491+
// `BigInt(value)` throws syntax error if the string contains numeric separators.
492+
return BigInt(str.replace(/_/g, ""))
493+
}
494+
458495
pp.readRadixNumber = function(radix) {
459496
let start = this.pos
460497
this.pos += 2 // 0x
461498
let val = this.readInt(radix)
462499
if (val == null) this.raise(this.start + 2, "Expected number in radix " + radix)
463500
if (this.options.ecmaVersion >= 11 && this.input.charCodeAt(this.pos) === 110) {
464-
val = typeof BigInt !== "undefined" ? BigInt(this.input.slice(start, this.pos)) : null
501+
val = stringToBigInt(this.input.slice(start, this.pos))
465502
++this.pos
466503
} else if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
467504
return this.finishToken(tt.num, val)
@@ -471,13 +508,12 @@ pp.readRadixNumber = function(radix) {
471508

472509
pp.readNumber = function(startsWithDot) {
473510
let start = this.pos
474-
if (!startsWithDot && this.readInt(10) === null) this.raise(start, "Invalid number")
511+
if (!startsWithDot && this.readInt(10, undefined, true) === null) this.raise(start, "Invalid number")
475512
let octal = this.pos - start >= 2 && this.input.charCodeAt(start) === 48
476513
if (octal && this.strict) this.raise(start, "Invalid number")
477514
let next = this.input.charCodeAt(this.pos)
478515
if (!octal && !startsWithDot && this.options.ecmaVersion >= 11 && next === 110) {
479-
let str = this.input.slice(start, this.pos)
480-
let val = typeof BigInt !== "undefined" ? BigInt(str) : null
516+
let val = stringToBigInt(this.input.slice(start, this.pos))
481517
++this.pos
482518
if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
483519
return this.finishToken(tt.num, val)
@@ -495,8 +531,7 @@ pp.readNumber = function(startsWithDot) {
495531
}
496532
if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
497533

498-
let str = this.input.slice(start, this.pos)
499-
let val = octal ? parseInt(str, 8) : parseFloat(str)
534+
let val = stringToNumber(this.input.slice(start, this.pos), octal)
500535
return this.finishToken(tt.num, val)
501536
}
502537

‎bin/run_test262.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ const unsupportedFeatures = [
1010
"class-static-fields-private",
1111
"class-static-fields-public",
1212
"class-static-methods-private",
13-
"numeric-separator-literal",
1413
"logical-assignment-operators",
1514
];
1615

‎test/run.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
require("./tests-nullish-coalescing.js");
2323
require("./tests-optional-chaining.js");
2424
require("./tests-logical-assignment-operators.js");
25+
require("./tests-numeric-separators.js");
2526
var acorn = require("../acorn")
2627
var acorn_loose = require("../acorn-loose")
2728

‎test/tests-numeric-separators.js‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
// Tests for ECMAScript 2021 Numeric Separators
2+
3+
if (typeof exports != 'undefined') {
4+
var test = require('./driver.js').test;
5+
var testFail = require('./driver.js').testFail;
6+
}
7+
8+
function bigint(str) {
9+
if (typeof BigInt !== "function") {
10+
return null
11+
}
12+
return BigInt(str)
13+
}
14+
15+
test(
16+
"123_456",
17+
{
18+
"type": "Program",
19+
"start": 0,
20+
"end": 7,
21+
"body": [
22+
{
23+
"type": "ExpressionStatement",
24+
"start": 0,
25+
"end": 7,
26+
"expression": {
27+
"type": "Literal",
28+
"start": 0,
29+
"end": 7,
30+
"value": 123456,
31+
"raw": "123_456"
32+
}
33+
}
34+
],
35+
"sourceType": "script"
36+
},
37+
{ ecmaVersion: 12 }
38+
);
39+
40+
test(
41+
"123_456.123_456e+123_456",
42+
{
43+
"type": "Program",
44+
"start": 0,
45+
"end": 24,
46+
"body": [
47+
{
48+
"type": "ExpressionStatement",
49+
"start": 0,
50+
"end": 24,
51+
"expression": {
52+
"type": "Literal",
53+
"start": 0,
54+
"end": 24,
55+
"value": 123456.123456e+123456,
56+
"raw": "123_456.123_456e+123_456"
57+
}
58+
}
59+
],
60+
"sourceType": "script"
61+
},
62+
{ ecmaVersion: 12 }
63+
);
64+
65+
test(
66+
"0b1010_0001",
67+
{
68+
"type": "Program",
69+
"start": 0,
70+
"end": 11,
71+
"body": [
72+
{
73+
"type": "ExpressionStatement",
74+
"start": 0,
75+
"end": 11,
76+
"expression": {
77+
"type": "Literal",
78+
"start": 0,
79+
"end": 11,
80+
"value": 0b10100001,
81+
"raw": "0b1010_0001"
82+
}
83+
}
84+
],
85+
"sourceType": "script"
86+
},
87+
{ ecmaVersion: 12 }
88+
);
89+
90+
test(
91+
"0xDEAD_BEAF",
92+
{
93+
"type": "Program",
94+
"start": 0,
95+
"end": 11,
96+
"body": [
97+
{
98+
"type": "ExpressionStatement",
99+
"start": 0,
100+
"end": 11,
101+
"expression": {
102+
"type": "Literal",
103+
"start": 0,
104+
"end": 11,
105+
"value": 0xDEADBEAF,
106+
"raw": "0xDEAD_BEAF"
107+
}
108+
}
109+
],
110+
"sourceType": "script"
111+
},
112+
{ ecmaVersion: 12 }
113+
);
114+
115+
test(
116+
"0o755_666",
117+
{
118+
"type": "Program",
119+
"start": 0,
120+
"end": 9,
121+
"body": [
122+
{
123+
"type": "ExpressionStatement",
124+
"start": 0,
125+
"end": 9,
126+
"expression": {
127+
"type": "Literal",
128+
"start": 0,
129+
"end": 9,
130+
"value": 0o755666,
131+
"raw": "0o755_666"
132+
}
133+
}
134+
],
135+
"sourceType": "script"
136+
},
137+
{ ecmaVersion: 12 }
138+
);
139+
140+
test(
141+
"123_456n",
142+
{
143+
"type": "Program",
144+
"start": 0,
145+
"end": 8,
146+
"body": [
147+
{
148+
"type": "ExpressionStatement",
149+
"start": 0,
150+
"end": 8,
151+
"expression": {
152+
"type": "Literal",
153+
"start": 0,
154+
"end": 8,
155+
"value": bigint("123456"),
156+
"raw": "123_456n",
157+
"bigint": "123456"
158+
}
159+
}
160+
],
161+
"sourceType": "script"
162+
},
163+
{ ecmaVersion: 12 }
164+
);
165+
166+
test(
167+
".012_345",
168+
{
169+
"type": "Program",
170+
"start": 0,
171+
"end": 8,
172+
"body": [
173+
{
174+
"type": "ExpressionStatement",
175+
"start": 0,
176+
"end": 8,
177+
"expression": {
178+
"type": "Literal",
179+
"start": 0,
180+
"end": 8,
181+
"value": 0.012345,
182+
"raw": ".012_345"
183+
}
184+
}
185+
],
186+
"sourceType": "script"
187+
},
188+
{ ecmaVersion: 12 }
189+
);
190+
191+
testFail("123_456", "Identifier directly after number (1:3)", { ecmaVersion: 11 });
192+
testFail("123__456", "Numeric separator must be exactly one underscore (1:4)", { ecmaVersion: 12 });
193+
testFail("0._123456", "Numeric separator is not allowed at the first of digits (1:2)", { ecmaVersion: 12 });
194+
testFail("123456_", "Numeric separator is not allowed at the last of digits (1:6)", { ecmaVersion: 12 });
195+
testFail("012_345", "Numeric separator is not allowed in legacy octal numeric literals (1:3)", { ecmaVersion: 12 });
196+
197+
testFail("'\\x2_0'", "Bad character escape sequence (1:3)", { ecmaVersion: 12 });
198+
testFail("'\\u00_20'", "Bad character escape sequence (1:3)", { ecmaVersion: 12 });
199+
testFail("'\\u{2_0}'", "Bad character escape sequence (1:4)", { ecmaVersion: 12 });

0 commit comments

Comments
 (0)