-import {isIdentifierStart, isIdentifierChar} from "./identifier"
-import {types as tt, keywords as keywordTypes} from "./tokentype"
-import {Parser} from "./state"
-import {SourceLocation} from "./locutil"
-import {RegExpValidationState} from "./regexp"
-import {lineBreak, lineBreakG, isNewLine, nonASCIIwhitespace} from "./whitespace"
+import {isIdentifierStart, isIdentifierChar} from "./identifier.js"
+import {types as tt, keywords as keywordTypes} from "./tokentype.js"
+import {Parser} from "./state.js"
+import {SourceLocation} from "./locutil.js"
+import {RegExpValidationState} from "./regexp.js"
+import {lineBreak, nextLineBreak, isNewLine, nonASCIIwhitespace} from "./whitespace.js"
// Object type used to represent tokens. Note that normally, tokens
// simply exist as properties on the parser object. This is only
// Move to the next token
-pp.next = function() {
+pp.next = function(ignoreEscapeSequenceInKeyword) {
+ if (!ignoreEscapeSequenceInKeyword && this.type.keyword && this.containsEsc)
+ this.raiseRecoverable(this.start, "Escape sequence in keyword " + this.type.keyword)
if (this.options.onToken)
this.options.onToken(new Token(this))
// Toggle strict mode. Re-reads the next number or string to please
// pedantic tests (`"use strict"; 010;` should fail).
-pp.curContext = function() {
- return this.context[this.context.length - 1]
-}
-
// Read a single token, updating the parser object's token-related
// properties.
pp.fullCharCodeAtPos = function() {
let code = this.input.charCodeAt(this.pos)
- if (code <= 0xd7ff || code >= 0xe000) return code
+ if (code <= 0xd7ff || code >= 0xdc00) return code
let next = this.input.charCodeAt(this.pos + 1)
- return (code << 10) + next - 0x35fdc00
+ return next <= 0xdbff || next >= 0xe000 ? code : (code << 10) + next - 0x35fdc00
}
pp.skipBlockComment = function() {
if (end === -1) this.raise(this.pos - 2, "Unterminated comment")
this.pos = end + 2
if (this.options.locations) {
- lineBreakG.lastIndex = start
- let match
- while ((match = lineBreakG.exec(this.input)) && match.index < this.pos) {
+ for (let nextBreak, pos = start; (nextBreak = nextLineBreak(this.input, pos, this.pos)) > -1;) {
++this.curLine
- this.lineStart = match.index + match[0].length
+ pos = this.lineStart = nextBreak
}
}
if (this.options.onComment)
pp.readToken_pipe_amp = function(code) { // '|&'
let next = this.input.charCodeAt(this.pos + 1)
- if (next === code) return this.finishOp(code === 124 ? tt.logicalOR : tt.logicalAND, 2)
+ if (next === code) {
+ if (this.options.ecmaVersion >= 12) {
+ let next2 = this.input.charCodeAt(this.pos + 2)
+ if (next2 === 61) return this.finishOp(tt.assign, 3)
+ }
+ return this.finishOp(code === 124 ? tt.logicalOR : tt.logicalAND, 2)
+ }
if (next === 61) return this.finishOp(tt.assign, 2)
return this.finishOp(code === 124 ? tt.bitwiseOR : tt.bitwiseAND, 1)
}
return this.finishOp(code === 61 ? tt.eq : tt.prefix, 1)
}
+pp.readToken_question = function() { // '?'
+ const ecmaVersion = this.options.ecmaVersion
+ if (ecmaVersion >= 11) {
+ let next = this.input.charCodeAt(this.pos + 1)
+ if (next === 46) {
+ let next2 = this.input.charCodeAt(this.pos + 2)
+ if (next2 < 48 || next2 > 57) return this.finishOp(tt.questionDot, 2)
+ }
+ if (next === 63) {
+ if (ecmaVersion >= 12) {
+ let next2 = this.input.charCodeAt(this.pos + 2)
+ if (next2 === 61) return this.finishOp(tt.assign, 3)
+ }
+ return this.finishOp(tt.coalesce, 2)
+ }
+ }
+ return this.finishOp(tt.question, 1)
+}
+
+pp.readToken_numberSign = function() { // '#'
+ const ecmaVersion = this.options.ecmaVersion
+ let code = 35 // '#'
+ if (ecmaVersion >= 13) {
+ ++this.pos
+ code = this.fullCharCodeAtPos()
+ if (isIdentifierStart(code, true) || code === 92 /* '\' */) {
+ return this.finishToken(tt.privateId, this.readWord1())
+ }
+ }
+
+ this.raise(this.pos, "Unexpected character '" + codePointToString(code) + "'")
+}
+
pp.getTokenFromCode = function(code) {
switch (code) {
// The interpretation of a dot depends on whether it is followed
case 123: ++this.pos; return this.finishToken(tt.braceL)
case 125: ++this.pos; return this.finishToken(tt.braceR)
case 58: ++this.pos; return this.finishToken(tt.colon)
- case 63: ++this.pos; return this.finishToken(tt.question)
case 96: // '`'
if (this.options.ecmaVersion < 6) break
// often referred to. `finishOp` simply skips the amount of
// characters it is given as second argument, and returns a token
// of the type given by its first argument.
-
case 47: // '/'
return this.readToken_slash()
case 61: case 33: // '=!'
return this.readToken_eq_excl(code)
+ case 63: // '?'
+ return this.readToken_question()
+
case 126: // '~'
return this.finishOp(tt.prefix, 1)
+
+ case 35: // '#'
+ return this.readToken_numberSign()
}
this.raise(this.pos, "Unexpected character '" + codePointToString(code) + "'")
// were read, the integer value otherwise. When `len` is given, this
// will return `null` unless the integer has exactly `len` digits.
-pp.readInt = function(radix, len) {
- let start = this.pos, total = 0
- for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
+pp.readInt = function(radix, len, maybeLegacyOctalNumericLiteral) {
+ // `len` is used for character escape sequences. In that case, disallow separators.
+ const allowSeparators = this.options.ecmaVersion >= 12 && len === undefined
+
+ // `maybeLegacyOctalNumericLiteral` is true if it doesn't have prefix (0x,0o,0b)
+ // and isn't fraction part nor exponent part. In that case, if the first digit
+ // is zero then disallow separators.
+ const isLegacyOctalNumericLiteral = maybeLegacyOctalNumericLiteral && this.input.charCodeAt(this.pos) === 48
+
+ let start = this.pos, total = 0, lastCode = 0
+ for (let i = 0, e = len == null ? Infinity : len; i < e; ++i, ++this.pos) {
let code = this.input.charCodeAt(this.pos), val
+
+ if (allowSeparators && code === 95) {
+ if (isLegacyOctalNumericLiteral) this.raiseRecoverable(this.pos, "Numeric separator is not allowed in legacy octal numeric literals")
+ if (lastCode === 95) this.raiseRecoverable(this.pos, "Numeric separator must be exactly one underscore")
+ if (i === 0) this.raiseRecoverable(this.pos, "Numeric separator is not allowed at the first of digits")
+ lastCode = code
+ continue
+ }
+
if (code >= 97) val = code - 97 + 10 // a
else if (code >= 65) val = code - 65 + 10 // A
else if (code >= 48 && code <= 57) val = code - 48 // 0-9
else val = Infinity
if (val >= radix) break
- ++this.pos
+ lastCode = code
total = total * radix + val
}
+
+ if (allowSeparators && lastCode === 95) this.raiseRecoverable(this.pos - 1, "Numeric separator is not allowed at the last of digits")
if (this.pos === start || len != null && this.pos - start !== len) return null
return total
}
+function stringToNumber(str, isLegacyOctalNumericLiteral) {
+ if (isLegacyOctalNumericLiteral) {
+ return parseInt(str, 8)
+ }
+
+ // `parseFloat(value)` stops parsing at the first numeric separator then returns a wrong value.
+ return parseFloat(str.replace(/_/g, ""))
+}
+
+function stringToBigInt(str) {
+ if (typeof BigInt !== "function") {
+ return null
+ }
+
+ // `BigInt(value)` throws syntax error if the string contains numeric separators.
+ return BigInt(str.replace(/_/g, ""))
+}
+
pp.readRadixNumber = function(radix) {
+ let start = this.pos
this.pos += 2 // 0x
let val = this.readInt(radix)
if (val == null) this.raise(this.start + 2, "Expected number in radix " + radix)
- if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
+ if (this.options.ecmaVersion >= 11 && this.input.charCodeAt(this.pos) === 110) {
+ val = stringToBigInt(this.input.slice(start, this.pos))
+ ++this.pos
+ } else if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
return this.finishToken(tt.num, val)
}
pp.readNumber = function(startsWithDot) {
let start = this.pos
- if (!startsWithDot && this.readInt(10) === null) this.raise(start, "Invalid number")
+ if (!startsWithDot && this.readInt(10, undefined, true) === null) this.raise(start, "Invalid number")
let octal = this.pos - start >= 2 && this.input.charCodeAt(start) === 48
if (octal && this.strict) this.raise(start, "Invalid number")
- if (octal && /[89]/.test(this.input.slice(start, this.pos))) octal = false
let next = this.input.charCodeAt(this.pos)
+ if (!octal && !startsWithDot && this.options.ecmaVersion >= 11 && next === 110) {
+ let val = stringToBigInt(this.input.slice(start, this.pos))
+ ++this.pos
+ if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
+ return this.finishToken(tt.num, val)
+ }
+ if (octal && /[89]/.test(this.input.slice(start, this.pos))) octal = false
if (next === 46 && !octal) { // '.'
++this.pos
this.readInt(10)
}
if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
- let str = this.input.slice(start, this.pos)
- let val = octal ? parseInt(str, 8) : parseFloat(str)
+ let val = stringToNumber(this.input.slice(start, this.pos), octal)
return this.finishToken(tt.num, val)
}
out += this.input.slice(chunkStart, this.pos)
out += this.readEscapedChar(false)
chunkStart = this.pos
+ } else if (ch === 0x2028 || ch === 0x2029) {
+ if (this.options.ecmaVersion < 10) this.raise(this.start, "Unterminated string constant")
+ ++this.pos
+ if (this.options.locations) {
+ this.curLine++
+ this.lineStart = this.pos
+ }
} else {
- if (isNewLine(ch, this.options.ecmaVersion >= 10)) this.raise(this.start, "Unterminated string constant")
+ if (isNewLine(ch)) this.raise(this.start, "Unterminated string constant")
++this.pos
}
}
if (this.input[this.pos + 1] !== "{") {
break
}
- // falls through
+ // falls through
case "`":
return this.finishToken(tt.invalidTemplate, this.input.slice(this.start, this.pos))
case 10: // ' \n'
if (this.options.locations) { this.lineStart = this.pos; ++this.curLine }
return ""
+ case 56:
+ case 57:
+ if (this.strict) {
+ this.invalidStringToken(
+ this.pos - 1,
+ "Invalid escape sequence"
+ )
+ }
+ if (inTemplate) {
+ const codePos = this.pos - 1
+
+ this.invalidStringToken(
+ codePos,
+ "Invalid escape sequence in template string"
+ )
+
+ return null
+ }
default:
if (ch >= 48 && ch <= 55) {
let octalStr = this.input.substr(this.pos - 1, 3).match(/^[0-7]+/)[0]
}
return String.fromCharCode(octal)
}
+ if (isNewLine(ch)) {
+ // Unicode new line characters after \ get removed from output in both
+ // template literals and strings
+ return ""
+ }
return String.fromCharCode(ch)
}
}
let word = this.readWord1()
let type = tt.name
if (this.keywords.test(word)) {
- if (this.containsEsc) this.raiseRecoverable(this.start, "Escape sequence in keyword " + word)
type = keywordTypes[word]
}
return this.finishToken(type, word)
public / jst.git / blobdiff