From: Jakub Pawlowicz Date: Sun, 7 Dec 2014 09:52:26 +0000 (+0000) Subject: Fixes handling escaped comments in tokenizer. X-Git-Url: https://git.ndcode.org/public/gitweb.cgi?a=commitdiff_plain;h=de94cbfe527dd03fc828d62fcde0a9ebf3da3c78;p=clean-css.git Fixes handling escaped comments in tokenizer. * That should be much easier with #395 which will get rid of them sooner. --- diff --git a/lib/properties/optimizer.js b/lib/properties/optimizer.js index 12fb1286..12926a36 100644 --- a/lib/properties/optimizer.js +++ b/lib/properties/optimizer.js @@ -249,7 +249,10 @@ module.exports = function Optimizer(options, context) { if (!eligibleForCompacting && processableInfo.implementedFor.test(tokens[i][0])) eligibleForCompacting = true; - var property = tokens[i][0] + ':' + tokens[i][1]; + // FIXME: the check should be gone with #396 + var property = tokens[i][0].indexOf('__ESCAPED_') === 0 ? + tokens[i][0] : + tokens[i][0] + ':' + tokens[i][1]; tokenized.push({ value: property, metadata: tokens[i][4] }); list.push(property); } diff --git a/lib/properties/token.js b/lib/properties/token.js index 02604767..19fb8772 100644 --- a/lib/properties/token.js +++ b/lib/properties/token.js @@ -128,7 +128,10 @@ module.exports = (function() { continue; } - var property = t.prop + ':' + t.value + (t.isImportant ? important : ''); + // FIXME: the check should be gone with #396 + var property = t.prop === '' && t.value.indexOf('__ESCAPED_') === 0 ? + t.value : + t.prop + ':' + t.value + (t.isImportant ? important : ''); tokenized.push({ value: property, metadata: t.metadata || {} }); list.push(property); } diff --git a/lib/selectors/optimizers/simple.js b/lib/selectors/optimizers/simple.js index 6dee63a8..bf1131a4 100644 --- a/lib/selectors/optimizers/simple.js +++ b/lib/selectors/optimizers/simple.js @@ -184,6 +184,14 @@ function reduce(body, options) { for (var i = 0, l = body.length; i < l; i++) { var token = body[i]; + + // FIXME: the check should be gone with #396 + if (token.value.indexOf('__ESCAPED_') === 0) { + reduced.push(token); + properties.push(token.value); + continue; + } + var firstColon = token.value.indexOf(':'); var property = token.value.substring(0, firstColon); var value = token.value.substring(firstColon + 1); diff --git a/lib/selectors/tokenizer.js b/lib/selectors/tokenizer.js index 6348106c..d8368a2c 100644 --- a/lib/selectors/tokenizer.js +++ b/lib/selectors/tokenizer.js @@ -93,7 +93,7 @@ function tokenize(context) { var next = whatsNext(context); if (!next) { var whatsLeft = context.chunk.substring(context.cursor); - if (whatsLeft.length > 0) { + if (whatsLeft.trim().length > 0) { tokenized.push({ kind: 'text', value: whatsLeft }); context.cursor += whatsLeft.length; } @@ -191,7 +191,8 @@ function tokenize(context) { if (addSourceMap) SourceMaps.track(escaped, context); } else { - tokenized.push({ kind: 'text', value: escaped }); + if (escaped.indexOf('__ESCAPED_COMMENT_SPECIAL') === 0) + tokenized.push({ kind: 'text', value: escaped }); if (addSourceMap) SourceMaps.track(escaped, context); diff --git a/lib/text/comments-processor.js b/lib/text/comments-processor.js index 8aac101e..4d4f4e65 100644 --- a/lib/text/comments-processor.js +++ b/lib/text/comments-processor.js @@ -9,6 +9,8 @@ var lineBreak = require('os').EOL; var CommentsProcessor = function CommentsProcessor(context, keepSpecialComments, keepBreaks, saveWaypoints) { this.comments = new EscapeStore('COMMENT'); + this.specialComments = new EscapeStore('COMMENT_SPECIAL'); + this.context = context; this.keepAll = keepSpecialComments == '*'; this.keepOne = keepSpecialComments == '1' || keepSpecialComments === 1; @@ -64,6 +66,7 @@ CommentsProcessor.prototype.escape = function (data) { tempData.push(data.substring(cursor, nextStart)); var comment = data.substring(nextStart, nextEnd + COMMENT_SUFFIX.length); + var isSpecialComment = comment.indexOf(SPECIAL_COMMENT_PREFIX) === 0; if (saveWaypoints) { breaksCount = comment.split(lineBreak).length - 1; @@ -73,9 +76,11 @@ CommentsProcessor.prototype.escape = function (data) { indent + comment.length; } - if (saveWaypoints || comment.indexOf(SPECIAL_COMMENT_PREFIX) === 0) { + if (saveWaypoints || isSpecialComment) { var metadata = saveWaypoints ? [breaksCount, newIndent] : null; - var placeholder = this.comments.store(comment, metadata); + var placeholder = isSpecialComment ? + this.specialComments.store(comment, metadata) : + this.comments.store(comment, metadata); tempData.push(placeholder); } @@ -89,26 +94,26 @@ CommentsProcessor.prototype.escape = function (data) { data; }; -CommentsProcessor.prototype.restore = function (data) { +function restore(context, data, from, isSpecial) { var tempData = []; var restored = 0; var cursor = 0; var addBreak; for (; cursor < data.length;) { - var nextMatch = this.comments.nextMatch(data, cursor); + var nextMatch = from.nextMatch(data, cursor); if (nextMatch.start < 0) break; tempData.push(data.substring(cursor, nextMatch.start)); - var comment = this.comments.restore(nextMatch.match); + var comment = from.restore(nextMatch.match); - if (comment.indexOf(SPECIAL_COMMENT_PREFIX) === 0 && (this.keepAll || (this.keepOne && restored === 0))) { + if (isSpecial && (context.keepAll || (context.keepOne && restored === 0))) { restored++; - addBreak = this.keepBreaks && data[nextMatch.end] != '\n' && data.lastIndexOf('\r\n', nextMatch.end + 1) != nextMatch.end; + addBreak = context.keepBreaks && data[nextMatch.end] != '\n' && data.lastIndexOf('\r\n', nextMatch.end + 1) != nextMatch.end; tempData.push(comment, addBreak ? lineBreak : ''); } else { - nextMatch.end += this.keepBreaks ? lineBreak.length : 0; + nextMatch.end += context.keepBreaks ? lineBreak.length : 0; } cursor = nextMatch.end; @@ -117,6 +122,12 @@ CommentsProcessor.prototype.restore = function (data) { return tempData.length > 0 ? tempData.join('') + data.substring(cursor, data.length) : data; +} + +CommentsProcessor.prototype.restore = function (data) { + data = restore(this, data, this.comments, false); + data = restore(this, data, this.specialComments, true); + return data; }; module.exports = CommentsProcessor; diff --git a/lib/utils/extractors.js b/lib/utils/extractors.js index 7ce9f070..de4f5446 100644 --- a/lib/utils/extractors.js +++ b/lib/utils/extractors.js @@ -8,6 +8,7 @@ var Extractors = { var buffer = []; var all = []; var property; + var isPropertyEnd; var isWhitespace; var wasWhitespace; var isSpecial; @@ -20,15 +21,30 @@ var Extractors = { for (var i = 0, l = string.length; i < l; i++) { current = string[i]; + isPropertyEnd = current === ';'; - isEscape = current == '_' && buffer.length < 2 && string.indexOf('__ESCAPED_', i) === 0; + isEscape = !isPropertyEnd && current == '_' && string.indexOf('__ESCAPED_COMMENT', i) === i; if (isEscape) { - var endOfEscape = string.indexOf('__', i + 1) + 2; - buffer = all = [string.substring(i, endOfEscape)]; - i = endOfEscape - 1; + if (buffer.length > 0) { + i--; + isPropertyEnd = true; + } else { + var endOfEscape = string.indexOf('__', i + 1) + 2; + var comment = string.substring(i, endOfEscape); + i = endOfEscape - 1; + + if (comment.indexOf('__ESCAPED_COMMENT_SPECIAL') === -1) { + if (addSourceMap) + SourceMaps.track(comment, context, true); + continue; + } + else { + buffer = all = [comment]; + } + } } - if (current === ';' || isEscape) { + if (isPropertyEnd || isEscape) { if (wasWhitespace && buffer[buffer.length - 1] === ' ') buffer.pop(); if (buffer.length > 0) { diff --git a/test/selectors/optimizers/simple-test.js b/test/selectors/optimizers/simple-test.js index fceefdba..9ddae424 100644 --- a/test/selectors/optimizers/simple-test.js +++ b/test/selectors/optimizers/simple-test.js @@ -447,4 +447,12 @@ vows.describe(SimpleOptimizer) ], }) ) + .addBatch( + propertyContext('comments', { + 'comment': [ + 'a{__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__color:red__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__}', + ['__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__', 'color:red', '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__'] + ] + }) + ) .export(module); diff --git a/test/selectors/tokenizer-source-maps-test.js b/test/selectors/tokenizer-source-maps-test.js index 4725436c..a7db7bf9 100644 --- a/test/selectors/tokenizer-source-maps-test.js +++ b/test/selectors/tokenizer-source-maps-test.js @@ -345,10 +345,6 @@ vows.describe('source-maps/analyzer') 'top-level': [ '__ESCAPED_COMMENT_CLEAN_CSS0(0, 5)__a{}', [ - { - kind: 'text', - value: '__ESCAPED_COMMENT_CLEAN_CSS0(0, 5)__' - }, { kind: 'selector', value: [{ value: 'a', metadata: { line: 1, column: 5, source: undefined } }], @@ -359,10 +355,6 @@ vows.describe('source-maps/analyzer') 'top-level with line breaks': [ '__ESCAPED_COMMENT_CLEAN_CSS0(2, 5)__a{}', [ - { - kind: 'text', - value: '__ESCAPED_COMMENT_CLEAN_CSS0(2, 5)__' - }, { kind: 'selector', value: [{ value: 'a', metadata: { line: 3, column: 5, source: undefined } }], @@ -382,13 +374,13 @@ vows.describe('source-maps/analyzer') }] ], 'in properties': [ - 'div{__ESCAPED_COMMENT_CLEAN_CSS0(2,5)__background:url(__ESCAPED_URL_CLEAN_CSS0(0,20)__);color:blue}a{font-family:__ESCAPED_FREE_TEXT_CLEAN_CSS0(1,3)__;color:red}', + 'div{__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(2,5)__background:url(__ESCAPED_URL_CLEAN_CSS0(0,20)__);color:blue}a{font-family:__ESCAPED_FREE_TEXT_CLEAN_CSS0(1,3)__;color:red}', [ { kind: 'selector', value: [{ value: 'div', metadata: { line: 1, column: 0, source: undefined } }], body: [ - { value: '__ESCAPED_COMMENT_CLEAN_CSS0(2,5)__', metadata: { line: 1, column: 4, source: undefined }}, + { value: '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(2,5)__', metadata: { line: 1, column: 4, source: undefined }}, { value: 'background:url(__ESCAPED_URL_CLEAN_CSS0(0,20)__)', metadata: { line: 3, column: 5, source: undefined } }, { value: 'color:blue', metadata: { line: 3, column: 42, source: undefined } } ] diff --git a/test/selectors/tokenizer-test.js b/test/selectors/tokenizer-test.js index ed718721..dc43460b 100644 --- a/test/selectors/tokenizer-test.js +++ b/test/selectors/tokenizer-test.js @@ -31,23 +31,11 @@ vows.describe(Tokenizer) ], 'an escaped content': [ '__ESCAPED_COMMENT_CLEAN_CSS0__', - [{ - kind: 'text', - value: '__ESCAPED_COMMENT_CLEAN_CSS0__' - }] + [] ], 'an escaped content followed by a break': [ '__ESCAPED_COMMENT_CLEAN_CSS0__\n', - [ - { - kind: 'text', - value: '__ESCAPED_COMMENT_CLEAN_CSS0__' - }, - { - kind: 'text', - value: '\n' - } - ] + [] ], 'an empty selector': [ 'a{}', @@ -141,14 +129,6 @@ vows.describe(Tokenizer) 'two comments and a selector separated by newline': [ '__ESCAPED_COMMENT_CLEAN_CSS0__\n__ESCAPED_COMMENT_CLEAN_CSS1__\ndiv{}', [ - { - kind: 'text', - value: '__ESCAPED_COMMENT_CLEAN_CSS0__' - }, - { - kind: 'text', - value: '__ESCAPED_COMMENT_CLEAN_CSS1__' - }, { kind: 'selector', value: [{ value: 'div' }], @@ -156,6 +136,18 @@ vows.describe(Tokenizer) } ] ], + 'two properties wrapped between comments': [ + 'div{__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__color:red__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__}', + [{ + kind: 'selector', + value: [{ value: 'div' }], + body: [ + { value: '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__' }, + { value: 'color:red' }, + { value: '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__' } + ] + }] + ], 'media query': [ '@media (min-width:980px){}', [{ @@ -243,9 +235,13 @@ vows.describe(Tokenizer) '', [] ], - 'an escaped content': [ + 'an escaped comment': [ '__ESCAPED_COMMENT_CLEAN_CSS0__', - [{ kind: 'text', value: '__ESCAPED_COMMENT_CLEAN_CSS0__' }] + [] + ], + 'an escaped special comment': [ + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__', + [{ kind: 'text', value: '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__' }] ], 'an empty selector': [ 'a{}', @@ -301,6 +297,40 @@ vows.describe(Tokenizer) } } ] + ], + 'two properties wrapped between comments': [ + 'div{__ESCAPED_COMMENT_CLEAN_CSS0(0, 5)__color:red__ESCAPED_COMMENT_CLEAN_CSS1(0, 5)__}', + [{ + kind: 'selector', + value: [{ value: 'div' }], + body: [ + { value: 'color:red' } + ], + metadata: { + body: 'color:red', + bodiesList: ['color:red'], + selector: 'div', + selectorsList: ['div'] + } + }] + ], + 'two properties wrapped between special comments': [ + 'div{__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(0, 5)__color:red__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1(0, 5)__}', + [{ + kind: 'selector', + value: [{ value: 'div' }], + body: [ + { value: '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(0, 5)__' }, + { value: 'color:red' }, + { value: '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1(0, 5)__' } + ], + metadata: { + body: '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(0, 5)__,color:red,__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1(0, 5)__', + bodiesList: ['__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(0, 5)__', 'color:red', '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1(0, 5)__'], + selector: 'div', + selectorsList: ['div'] + } + }] ] }, true) ) diff --git a/test/source-map-test.js b/test/source-map-test.js index 772a77db..860fb5d9 100644 --- a/test/source-map-test.js +++ b/test/source-map-test.js @@ -15,6 +15,14 @@ var http = require('http'); var port = 24682; vows.describe('source-map') + .addBatch({ + 'vendor prefix with comments': { + 'topic': new CleanCSS({ sourceMap: true }).minify('html{font-family:sans-serif;/* 1 */-ms-text-size-adjust:100%;/* 2 */-webkit-text-size-adjust:100%/* 3 */}'), + 'gets right output': function (minified) { + assert.equal(minified.styles, 'html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}'); + } + } + }) .addBatch({ 'module #1': { 'topic': new CleanCSS({ sourceMap: true }).minify('/*! a */div[data-id=" abc "] { color:red; }'), diff --git a/test/text/comments-processor-test.js b/test/text/comments-processor-test.js index ba0144d8..80c0b355 100644 --- a/test/text/comments-processor-test.js +++ b/test/text/comments-processor-test.js @@ -49,7 +49,7 @@ vows.describe(CommentsProcessor) ], 'one special comment': [ '/*! some text */', - '__ESCAPED_COMMENT_CLEAN_CSS0__', + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__', '/*! some text */' ], 'two comments': [ @@ -64,7 +64,7 @@ vows.describe(CommentsProcessor) ], 'two special comments': [ '/*! one text *//*! another text */', - '__ESCAPED_COMMENT_CLEAN_CSS0____ESCAPED_COMMENT_CLEAN_CSS1__', + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0____ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__', '/*! one text *//*! another text */' ], 'commented selector': [ @@ -88,12 +88,12 @@ vows.describe(CommentsProcessor) ], 'one special comment': [ '/*! some text */', - '__ESCAPED_COMMENT_CLEAN_CSS0__', + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__', '/*! some text */' ], 'two special comments': [ '/*! one text *//*! another text */', - '__ESCAPED_COMMENT_CLEAN_CSS0____ESCAPED_COMMENT_CLEAN_CSS1__', + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0____ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__', '/*! one text */' ] }, '1') @@ -107,12 +107,12 @@ vows.describe(CommentsProcessor) ], 'one special comment': [ '/*! some text */', - '__ESCAPED_COMMENT_CLEAN_CSS0__', + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__', '' ], 'two special comments': [ '/*! one text *//*! another text */', - '__ESCAPED_COMMENT_CLEAN_CSS0____ESCAPED_COMMENT_CLEAN_CSS1__', + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0____ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1__', '' ] }, '0') @@ -121,7 +121,7 @@ vows.describe(CommentsProcessor) processorContext('zero with breaks', { 'content and special comments': [ 'a{}' + lineBreak + '/*! some text */' + lineBreak + 'p{}', - 'a{}' + lineBreak + '__ESCAPED_COMMENT_CLEAN_CSS0__' + lineBreak + 'p{}', + 'a{}' + lineBreak + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__' + lineBreak + 'p{}', 'a{}' + lineBreak + 'p{}' ] }, '0', true) @@ -130,17 +130,17 @@ vows.describe(CommentsProcessor) processorContext('one with breaks', { 'forces break after comments': [ 'a{}/*! some text */p{}', - 'a{}__ESCAPED_COMMENT_CLEAN_CSS0__p{}', + 'a{}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__p{}', 'a{}/*! some text */' + lineBreak + 'p{}' ], 'if not given already comments': [ 'a{}/*! some text */' + lineBreak + 'p{}', - 'a{}__ESCAPED_COMMENT_CLEAN_CSS0__' + lineBreak + 'p{}', + 'a{}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__' + lineBreak + 'p{}', 'a{}/*! some text */' + lineBreak + 'p{}' ], 'if given an other platform break already': [ 'a{}/*! some text */' + otherLineBreak + 'p{}', - 'a{}__ESCAPED_COMMENT_CLEAN_CSS0__' + otherLineBreak + 'p{}', + 'a{}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0__' + otherLineBreak + 'p{}', 'a{}/*! some text */' + otherLineBreak + 'p{}' ] }, '1', true) @@ -154,7 +154,7 @@ vows.describe(CommentsProcessor) ], 'one special comment': [ '/*! some text */', - '__ESCAPED_COMMENT_CLEAN_CSS0(0,16)__', + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(0,16)__', '/*! some text */' ], 'two comments': [ @@ -169,17 +169,17 @@ vows.describe(CommentsProcessor) ], 'two special comments': [ '/*! one text *//*! another text */', - '__ESCAPED_COMMENT_CLEAN_CSS0(0,15)____ESCAPED_COMMENT_CLEAN_CSS1(0,35)__', + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(0,15)____ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1(0,35)__', '/*! one text *//*! another text */' ], 'two special comments with line breaks': [ '/*! one text' + lineBreak + '*//*! another' + lineBreak + ' text' + lineBreak + ' */', - '__ESCAPED_COMMENT_CLEAN_CSS0(1,2)____ESCAPED_COMMENT_CLEAN_CSS1(2,3)__', + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(1,2)____ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1(2,3)__', '/*! one text' + lineBreak + '*//*! another' + lineBreak + ' text' + lineBreak + ' */' ], 'three special comments with line breaks and content in between': [ '/*! one text' + lineBreak + '*/a{}/*! ' + lineBreak + 'test */p{color:red}/*! another' + lineBreak + ' text' + lineBreak + lineBreak + ' */', - '__ESCAPED_COMMENT_CLEAN_CSS0(1,2)__a{}__ESCAPED_COMMENT_CLEAN_CSS1(1,7)__p{color:red}__ESCAPED_COMMENT_CLEAN_CSS2(3,4)__', + '__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS0(1,2)__a{}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS1(1,7)__p{color:red}__ESCAPED_COMMENT_SPECIAL_CLEAN_CSS2(3,4)__', '/*! one text' + lineBreak + '*/a{}/*! ' + lineBreak + 'test */p{color:red}/*! another' + lineBreak + ' text' + lineBreak + lineBreak + ' */' ] }, '*', false, true)