From d4685640a00a0c998041c96ec197e613bd67b7b3 Mon Sep 17 00:00:00 2001 From: "Alex Lam S.L" Date: Mon, 1 Feb 2021 02:36:45 +0000 Subject: [PATCH] support template literals (#4601) --- lib/ast.js | 28 +++++++++++++ lib/compress.js | 10 ++++- lib/output.js | 13 ++++++ lib/parse.js | 54 ++++++++++++++++++++++++- lib/transform.js | 4 ++ lib/utils.js | 2 + test/compress/templates.js | 82 ++++++++++++++++++++++++++++++++++++++ test/mocha/templates.js | 64 +++++++++++++++++++++++++++++ test/ufuzz/index.js | 24 +++++++++++ 9 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 test/compress/templates.js create mode 100644 test/mocha/templates.js diff --git a/lib/ast.js b/lib/ast.js index 1a09d4de..5bfe7be1 100644 --- a/lib/ast.js +++ b/lib/ast.js @@ -1418,6 +1418,34 @@ var AST_This = DEFNODE("This", null, { }, }, AST_Symbol); +var AST_Template = DEFNODE("Template", "expressions strings tag", { + $documentation: "A template literal, i.e. tag`str1${expr1}...strN${exprN}strN+1`", + $propdoc: { + expressions: "[AST_Node*] the placeholder expressions", + strings: "[string*] the interpolating text segments", + tag: "[AST_Node] tag function, or null if absent", + }, + walk: function(visitor) { + var node = this; + visitor.visit(node, function() { + if (node.tag) node.tag.walk(visitor); + node.expressions.forEach(function(expr) { + expr.walk(visitor); + }); + }); + }, + _validate: function() { + if (this.expressions.length + 1 != this.strings.length) { + throw new Error("malformed template with " + this.expressions.length + " placeholder(s) but " + this.strings.length + " text segment(s)"); + } + must_be_expressions(this, "expressions"); + this.strings.forEach(function(string) { + if (typeof string != "string") throw new Error("strings must contain string"); + }); + if (this.tag != null) must_be_expression(this, "tag"); + }, +}); + var AST_Constant = DEFNODE("Constant", null, { $documentation: "Base class for all constants", }); diff --git a/lib/compress.js b/lib/compress.js index d65861a6..700f3748 100644 --- a/lib/compress.js +++ b/lib/compress.js @@ -4663,6 +4663,9 @@ merge(Compressor.prototype, { def(AST_SymbolRef, function(compressor) { return !this.is_declared(compressor) || !can_drop_symbol(this); }); + def(AST_Template, function(compressor) { + return any(this.expressions, compressor); + }); def(AST_This, return_false); def(AST_Try, function(compressor) { return any(this.body, compressor) @@ -4673,7 +4676,7 @@ merge(Compressor.prototype, { return unary_side_effects[this.operator] || this.expression.has_side_effects(compressor); }); - def(AST_VarDef, function(compressor) { + def(AST_VarDef, function() { return this.value; }); })(function(node, func) { @@ -7015,6 +7018,11 @@ merge(Compressor.prototype, { def(AST_SymbolRef, function(compressor) { return this.is_declared(compressor) && can_drop_symbol(this) ? null : this; }); + def(AST_Template, function(compressor, first_in_statement) { + var expressions = this.expressions; + if (expressions.length == 0) return null; + return make_sequence(this, expressions).drop_side_effect_free(compressor, first_in_statement); + }); def(AST_This, return_null); def(AST_Unary, function(compressor, first_in_statement) { var exp = this.expression; diff --git a/lib/output.js b/lib/output.js index 8521d7ef..73a63c7d 100644 --- a/lib/output.js +++ b/lib/output.js @@ -1486,6 +1486,19 @@ function OutputStream(options) { DEFPRINT(AST_This, function(output) { output.print("this"); }); + DEFPRINT(AST_Template, function(output) { + var self = this; + if (self.tag) self.tag.print(output); + output.print("`"); + for (var i = 0; i < self.expressions.length; i++) { + output.print(self.strings[i]); + output.print("${"); + self.expressions[i].print(output); + output.print("}"); + } + output.print(self.strings[i]); + output.print("`"); + }); DEFPRINT(AST_Constant, function(output) { output.print(this.value); }); diff --git a/lib/parse.js b/lib/parse.js index 38c56d8f..5b77d675 100644 --- a/lib/parse.js +++ b/lib/parse.js @@ -113,7 +113,7 @@ var OPERATORS = makePredicate([ var NEWLINE_CHARS = "\n\r\u2028\u2029"; var OPERATOR_CHARS = "+-*&%=<>!?|~^"; var PUNC_BEFORE_EXPRESSION = "[{(,;:"; -var PUNC_CHARS = PUNC_BEFORE_EXPRESSION + ")}]"; +var PUNC_CHARS = PUNC_BEFORE_EXPRESSION + "`)}]"; var WHITESPACE_CHARS = NEWLINE_CHARS + " \u00a0\t\f\u000b\u200b\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000\uFEFF"; var NON_IDENTIFIER_CHARS = makePredicate(characters("./'\"" + OPERATOR_CHARS + PUNC_CHARS + WHITESPACE_CHARS)); @@ -191,7 +191,28 @@ function tokenizer($TEXT, filename, html5_comments, shebang) { regex_allowed : false, comments_before : [], directives : {}, - directive_stack : [] + directive_stack : [], + read_template : with_eof_error("Unterminated template literal", function(strings) { + var s = ""; + for (;;) { + var ch = next(true, true); + switch (ch) { + case "\\": + ch += next(true, true); + break; + case "`": + strings.push(s); + return; + case "$": + if (peek() == "{") { + next(); + strings.push(s); + return true; + } + } + s += ch; + } + }), }; var prev_was_dot = false; @@ -816,6 +837,7 @@ function parse($TEXT, options) { }); case "[": case "(": + case "`": return simple_statement(); case ";": S.in_directives = false; @@ -1401,6 +1423,11 @@ function parse($TEXT, options) { var start = S.token; if (is("punc")) { switch (start.value) { + case "`": + var tmpl = template(null); + tmpl.start = start; + tmpl.end = prev(); + return subscripts(tmpl, allow_calls); case "(": next(); if (is("punc", ")")) { @@ -1771,6 +1798,23 @@ function parse($TEXT, options) { } } + function template(tag) { + var read = S.input.context().read_template; + var strings = []; + var expressions = []; + while (read(strings)) { + next(); + expressions.push(expression()); + if (!is("punc", "}")) unexpected(); + } + next(); + return new AST_Template({ + expressions: expressions, + strings: strings, + tag: tag, + }); + } + var subscripts = function(expr, allow_calls) { var start = expr.start; if (is("punc", ".")) { @@ -1804,6 +1848,12 @@ function parse($TEXT, options) { mark_pure(call); return subscripts(call, true); } + if (is("punc", "`")) { + var tmpl = template(expr); + tmpl.start = expr.start; + tmpl.end = prev(); + return subscripts(tmpl, allow_calls); + } return expr; }; diff --git a/lib/transform.js b/lib/transform.js index f011cba9..b84bf0b9 100644 --- a/lib/transform.js +++ b/lib/transform.js @@ -201,6 +201,10 @@ TreeTransformer.prototype = new TreeWalker; if (self.key instanceof AST_Node) self.key = self.key.transform(tw); self.value = self.value.transform(tw); }); + DEF(AST_Template, function(self, tw) { + if (self.tag) self.tag = self.tag.transform(tw); + self.expressions = do_list(self.expressions, tw); + }); })(function(node, descend) { node.DEFMETHOD("transform", function(tw, in_list) { var x, y; diff --git a/lib/utils.js b/lib/utils.js index c3b67a6f..81ddfa63 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -255,6 +255,8 @@ function first_in_statement(stack, arrow) { if (p.expressions[0] === node) continue; } else if (p instanceof AST_Statement) { return p.body === node; + } else if (p instanceof AST_Template) { + if (p.tag === node) continue; } else if (p instanceof AST_UnaryPostfix) { if (p.expression === node) continue; } diff --git a/test/compress/templates.js b/test/compress/templates.js new file mode 100644 index 00000000..07aff0c5 --- /dev/null +++ b/test/compress/templates.js @@ -0,0 +1,82 @@ +simple: { + input: { + console.log(`foo + bar\nbaz`); + } + expect_exact: "console.log(`foo\n bar\\nbaz`);" + expect_stdout: [ + "foo", + " bar", + "baz", + ] + node_version: ">=4" +} + +placeholder: { + input: { + console.log(`foo ${ function(a, b) { + return a * b; + }(6, 7) }`); + } + expect_exact: "console.log(`foo ${function(a,b){return a*b}(6,7)}`);" + expect_stdout: "foo 42" + node_version: ">=4" +} + +nested: { + input: { + console.log(`P${`A${"S"}`}S`); + } + expect_exact: 'console.log(`P${`A${"S"}`}S`);' + expect_stdout: "PASS" + node_version: ">=4" +} + +tagged: { + input: { + console.log(String.raw`foo\nbar`); + } + expect_exact: "console.log(String.raw`foo\\nbar`);" + expect_stdout: "foo\\nbar" + node_version: ">=4" +} + +tagged_chain: { + input: { + function f(strings) { + return strings.join("") || f; + } + console.log(f```${42}``pass`.toUpperCase()); + } + expect_exact: 'function f(strings){return strings.join("")||f}console.log(f```${42}``pass`.toUpperCase());' + expect_stdout: "PASS" + node_version: ">=4" +} + +malformed_escape: { + input: { + (function(s) { + s.forEach((c, i) => console.log(i, c, s.raw[i])); + return () => console.log(arguments); + })`\uFo${42}`(); + } + expect_exact: "(function(s){s.forEach((c,i)=>console.log(i,c,s.raw[i]));return()=>console.log(arguments)})`\\uFo${42}`();" + expect_stdout: true + node_version: ">=4" +} + +evaluate: { + options = { + evaluate: true, + } + input: { + console.log(`foo ${ function(a, b) { + return a * b; + }(6, 7) }`); + } + expect: { + console.log(`foo ${42}`); + } + expect_stdout: "foo 42" + node_version: ">=4" +} diff --git a/test/mocha/templates.js b/test/mocha/templates.js new file mode 100644 index 00000000..7036be3d --- /dev/null +++ b/test/mocha/templates.js @@ -0,0 +1,64 @@ +var assert = require("assert"); +var run_code = require("../sandbox").run_code; +var semver = require("semver"); +var UglifyJS = require("../node"); + +describe("Template literals", function() { + it("Should reject invalid literal", function() { + [ + "`foo\\`", + "`foo${bar`", + "`foo${bar}", + ].forEach(function(input) { + assert.throws(function() { + UglifyJS.parse(input); + }, function(e) { + return e instanceof UglifyJS.JS_Parse_Error + && e.message === "Unterminated template literal"; + }, input); + }); + }); + it("Should reject invalid expression", function() { + [ + "`foo${bar;}`", + "`foo${42bar}`", + ].forEach(function(input) { + assert.throws(function() { + UglifyJS.parse(input); + }, function(e) { + return e instanceof UglifyJS.JS_Parse_Error; + }, input); + }); + }); + it("Should process line-break characters correctly", function() { + [ + // native line breaks + [ "`foo\nbar`", "`foo\nbar`" ], + [ "`foo\rbar`", "`foo\rbar`" ], + [ "`foo\r\nbar`", "`foo\nbar`" ], + [ "`foo\r\n\rbar`", "`foo\n\rbar`" ], + // escaped line breaks + [ "`foo\\nbar`", "`foo\\nbar`" ], + [ "`foo\\rbar`", "`foo\\rbar`" ], + [ "`foo\r\\nbar`", "`foo\r\\nbar`" ], + [ "`foo\\r\nbar`", "`foo\\r\nbar`" ], + [ "`foo\\r\\nbar`", "`foo\\r\\nbar`" ], + // continuation + [ "`foo\\\nbar`", "`foo\\\nbar`" ], + [ "`foo\\\rbar`", "`foo\\\rbar`" ], + [ "`foo\\\r\nbar`", "`foo\\\nbar`" ], + [ "`foo\\\r\n\rbar`", "`foo\\\n\rbar`" ], + [ "`foo\\\\nbar`", "`foo\\\\nbar`" ], + [ "`foo\\\\rbar`", "`foo\\\\rbar`" ], + [ "`foo\\\\r\nbar`", "`foo\\\\r\nbar`" ], + ].forEach(function(test) { + var input = "console.log(" + test[0] + ");"; + var result = UglifyJS.minify(input); + if (result.error) throw result.error; + var expected = "console.log(" + test[1] + ");"; + assert.strictEqual(result.code, expected, test[0]); + if (semver.satisfies(process.version, "<4")) return; + assert.strictEqual(run_code(result.code), run_code(input), test[0]); + }); + }); +}); diff --git a/test/ufuzz/index.js b/test/ufuzz/index.js index bee4c399..86badf3a 100644 --- a/test/ufuzz/index.js +++ b/test/ufuzz/index.js @@ -146,6 +146,7 @@ var SUPPORT = function(matrix) { rest_object: "var {...a} = {};", spread: "[...[]];", spread_object: "({...0});", + template: "``", trailing_comma: "function f(a,) {}", }); @@ -1038,6 +1039,7 @@ function _createExpression(recurmax, noComma, stmtDepth, canThrow) { case p++: return rng(2) + " === 1 ? a : b"; case p++: + if (SUPPORT.template && rng(20) == 0) return createTemplateLiteral(recurmax, stmtDepth, canThrow); case p++: return createValue(); case p++: @@ -1298,6 +1300,28 @@ function createArrayLiteral(recurmax, stmtDepth, canThrow) { return "[" + arr.join(", ") + "]"; } +function createTemplateLiteral(recurmax, stmtDepth, canThrow) { + recurmax--; + var s = []; + addText(); + for (var i = rng(6); --i >= 0;) { + s.push("${", createExpression(recurmax, COMMA_OK, stmtDepth, canThrow), "}"); + addText(); + } + return (rng(10) ? "`" : "String.raw`") + s.join(rng(5) ? "" : "\n") + "`"; + + function addText() { + while (rng(5) == 0) s.push([ + " ", + "$", + "}", + "\\`", + "\\\\", + "tmpl", + ][rng(6)]); + } +} + var SAFE_KEYS = [ "length", "foo", -- 2.34.1