From b06879dcbc3d32d19fb2bedc23af90b1473d109a Mon Sep 17 00:00:00 2001 From: Jakub Pawlowicz Date: Mon, 27 Oct 2014 20:46:17 +0000 Subject: [PATCH] Moves cleaning up tokens out of tokenizer. * Pulls out extracting logic into an utility class. --- lib/selectors/optimizer.js | 2 +- lib/selectors/optimizers/clean-up.js | 14 +++- lib/selectors/optimizers/simple.js | 25 ++++--- lib/selectors/tokenizer.js | 98 ++-------------------------- lib/utils/extractors.js | 78 ++++++++++++++++++++++ test/selectors/tokenizer-test.js | 16 ++--- 6 files changed, 118 insertions(+), 115 deletions(-) create mode 100644 lib/utils/extractors.js diff --git a/lib/selectors/optimizer.js b/lib/selectors/optimizer.js index 0454bd8f..98d5e0a2 100644 --- a/lib/selectors/optimizer.js +++ b/lib/selectors/optimizer.js @@ -27,7 +27,7 @@ function rebuild(tokens, keepBreaks, isFlatBlock) { for (var i = 0, l = tokens.length; i < l; i++) { var token = tokens[i]; - if (token.kind === 'text') { + if (token.kind === 'text' || token.kind == 'at-rule') { parts.push(token.value); continue; } diff --git a/lib/selectors/optimizers/clean-up.js b/lib/selectors/optimizers/clean-up.js index 55ef9e8b..0e3781da 100644 --- a/lib/selectors/optimizers/clean-up.js +++ b/lib/selectors/optimizers/clean-up.js @@ -8,7 +8,12 @@ var CleanUp = { for (var i = 0, l = selectors.length; i < l; i++) { var selector = selectors[i].value; - var reduced = selector.replace(/\s*([>\+\~])\s*/g, '$1'); + var reduced = selector + .replace(/\s/g, ' ') + .replace(/\s{2,}/g, ' ') + .replace(/ ?, ?/g, ',') + .replace(/\s*([>\+\~])\s*/g, '$1') + .trim(); if (selector.indexOf('*') > -1) { reduced = reduced @@ -36,6 +41,13 @@ var CleanUp = { .replace(/(\s{2,}|\s)/g, ' ') .replace(/(,|:|\() /g, '$1') .replace(/ \)/g, ')'); + }, + + atRule: function (block) { + return block + .replace(/\s/g, ' ') + .replace(/\s{2,}/g, ' ') + .trim(); } }; diff --git a/lib/selectors/optimizers/simple.js b/lib/selectors/optimizers/simple.js index a4a66104..a29ca931 100644 --- a/lib/selectors/optimizers/simple.js +++ b/lib/selectors/optimizers/simple.js @@ -29,12 +29,12 @@ function SimpleOptimizer(options) { function removeUnsupported(selectors, options) { if (options.compatibility.selectors.ie7Hack) - return false; + return selectors; var supported = []; var values = []; - for (var i = 0, l = selectors.length; i < l; i++) { - var selector = selectors[i]; + for (var i = 0, l = selectors.tokenized.length; i < l; i++) { + var selector = selectors.tokenized[i]; if (selector.value.indexOf('*+html ') === -1 && selector.value.indexOf('*:first-child+html ') === -1) { supported.push(selector); @@ -233,9 +233,8 @@ SimpleOptimizer.prototype.optimize = function(tokens) { break; if (token.kind == 'selector') { - var newSelectors = removeUnsupported(CleanUp.selectors(token.value).tokenized, options); - if (newSelectors) - token.value = newSelectors.tokenized; + var newSelectors = removeUnsupported(CleanUp.selectors(token.value), options); + token.value = newSelectors.tokenized; if (token.value.length === 0) { tokens.splice(i, 1); @@ -248,10 +247,8 @@ SimpleOptimizer.prototype.optimize = function(tokens) { if (options.updateMetadata) { token.metadata.body = newBody.list.join(';'); token.metadata.bodiesList = newBody.list; - if (newSelectors) { - token.metadata.selector = newSelectors.list.join(','); - token.metadata.selectorsList = newSelectors.list; - } + token.metadata.selector = newSelectors.list.join(','); + token.metadata.selectorsList = newSelectors.list; } } else if (token.kind == 'block') { token.value = CleanUp.block(token.value); @@ -259,15 +256,17 @@ SimpleOptimizer.prototype.optimize = function(tokens) { token.body = reduce(token.body, self.options).tokenized; else _optimize(token.body); - } else if (token.kind == 'text') { + } else if (token.kind == 'at-rule') { + token.value = CleanUp.atRule(token.value); + if (CHARSET_REGEXP.test(token.value)) { if (hasCharset || token.value.indexOf(CHARSET_TOKEN) == -1) { tokens.splice(i, 1); - i++; + i--; } else { hasCharset = true; tokens.splice(i, 1); - tokens.unshift({ kind: 'text', value: token.value.replace(CHARSET_REGEXP, CHARSET_TOKEN) }); + tokens.unshift({ kind: 'at-rule', value: token.value.replace(CHARSET_REGEXP, CHARSET_TOKEN) }); } } } diff --git a/lib/selectors/tokenizer.js b/lib/selectors/tokenizer.js index 6eeb75ab..879470b9 100644 --- a/lib/selectors/tokenizer.js +++ b/lib/selectors/tokenizer.js @@ -1,10 +1,7 @@ var Chunker = require('../utils/chunker'); -var Splitter = require('../utils/splitter'); +var Extract = require('../utils/extractors'); var flatBlock = /(^@(font\-face|page|\-ms\-viewport|\-o\-viewport|viewport|counter\-style)|\\@.+?)/; -var WHITESPACE = /\s/g; -var MULTI_WHITESPACE = /\s{2,}/g; -var WHITESPACE_COMMA = / ?, ?/g; function Tokenizer(minifyContext, addMetadata) { this.minifyContext = minifyContext; @@ -30,89 +27,6 @@ Tokenizer.prototype.toTokens = function (data) { return tokenize(context); }; -function valueMapper(property) { return { value: property }; } - -function extractProperties(string) { - var tokenized = []; - var list = []; - var buffer = []; - var property; - var isWhitespace; - var wasWhitespace; - var isSpecial; - var wasSpecial; - var current; - var wasCloseParenthesis; - - for (var i = 0, l = string.length; i < l; i++) { - current = string[i]; - - if (current === ';') { - if (wasWhitespace && buffer[buffer.length - 1] === ' ') - buffer.pop(); - if (buffer.length > 0) { - property = buffer.join(''); - tokenized.push({ value: property }); - list.push(property); - } - buffer = []; - } else { - isWhitespace = current === ' ' || current === '\t' || current === '\n'; - isSpecial = current === ':' || current === '[' || current === ']' || current === ',' || current === '(' || current === ')'; - - if (wasWhitespace && isSpecial) { - buffer.pop(); - buffer.push(current); - } else if (isWhitespace && wasSpecial && !wasCloseParenthesis) { - } else if (isWhitespace && !wasWhitespace && buffer.length > 0) { - buffer.push(' '); - } else if (isWhitespace && buffer.length === 0) { - } else if (isWhitespace && wasWhitespace) { - } else { - buffer.push(isWhitespace ? ' ' : current); - } - } - - wasSpecial = isSpecial; - wasWhitespace = isWhitespace; - wasCloseParenthesis = current === ')'; - } - - if (wasWhitespace && buffer[buffer.length - 1] === ' ') - buffer.pop(); - if (buffer.length > 0) { - property = buffer.join(''); - tokenized.push({ value: property }); - list.push(property); - } - - return { - list: list, - tokenized: tokenized - }; -} - -function extractSelectors(string) { - var extracted = string - .replace(WHITESPACE, ' ') - .replace(MULTI_WHITESPACE, ' ') - .replace(WHITESPACE_COMMA, ',') - .trim(); - - var selectors = new Splitter(',').split(extracted); - return { - list: selectors, - tokenized: selectors.map(valueMapper) - }; -} - -function extractBlock(string) { - return string - .replace(WHITESPACE, ' ') - .replace(MULTI_WHITESPACE, ' ') - .trim(); -} - function whatsNext(context) { var mode = context.mode; var chunk = context.chunk; @@ -194,8 +108,8 @@ function tokenize(context) { } else if (isSingle) { nextEnd = chunk.indexOf(';', nextSpecial + 1); - var single = extractBlock(chunk.substring(context.cursor, nextEnd + 1)); - tokenized.push({ kind: 'text', value: single }); + var single = chunk.substring(context.cursor, nextEnd + 1); + tokenized.push({ kind: 'at-rule', value: single }); context.cursor = nextEnd + 1; } else { @@ -209,7 +123,7 @@ function tokenize(context) { var specialBody = tokenize(context); if (typeof specialBody == 'string') - specialBody = extractProperties(specialBody).tokenized; + specialBody = Extract.properties(specialBody).tokenized; context.mode = oldMode; @@ -222,12 +136,12 @@ function tokenize(context) { context.cursor = nextEnd + 2; } else if (what == 'bodyStart') { - var selectorData = extractSelectors(chunk.substring(context.cursor, nextSpecial)); + var selectorData = Extract.selectors(chunk.substring(context.cursor, nextSpecial)); oldMode = context.mode; context.cursor = nextSpecial + 1; context.mode = 'body'; - var bodyData = extractProperties(tokenize(context)); + var bodyData = Extract.properties(tokenize(context)); context.mode = oldMode; diff --git a/lib/utils/extractors.js b/lib/utils/extractors.js new file mode 100644 index 00000000..4067fbcb --- /dev/null +++ b/lib/utils/extractors.js @@ -0,0 +1,78 @@ +var Splitter = require('./splitter'); + +function valueMapper(property) { + return { value: property }; +} + +var Extractors = { + properties: function (string) { + var tokenized = []; + var list = []; + var buffer = []; + var property; + var isWhitespace; + var wasWhitespace; + var isSpecial; + var wasSpecial; + var current; + var wasCloseParenthesis; + + for (var i = 0, l = string.length; i < l; i++) { + current = string[i]; + + if (current === ';') { + if (wasWhitespace && buffer[buffer.length - 1] === ' ') + buffer.pop(); + if (buffer.length > 0) { + property = buffer.join(''); + tokenized.push({ value: property }); + list.push(property); + } + buffer = []; + } else { + isWhitespace = current === ' ' || current === '\t' || current === '\n'; + isSpecial = current === ':' || current === '[' || current === ']' || current === ',' || current === '(' || current === ')'; + + if (wasWhitespace && isSpecial) { + buffer.pop(); + buffer.push(current); + } else if (isWhitespace && wasSpecial && !wasCloseParenthesis) { + } else if (isWhitespace && !wasWhitespace && buffer.length > 0) { + buffer.push(' '); + } else if (isWhitespace && buffer.length === 0) { + } else if (isWhitespace && wasWhitespace) { + } else { + buffer.push(isWhitespace ? ' ' : current); + } + } + + wasSpecial = isSpecial; + wasWhitespace = isWhitespace; + wasCloseParenthesis = current === ')'; + } + + if (wasWhitespace && buffer[buffer.length - 1] === ' ') + buffer.pop(); + if (buffer.length > 0) { + property = buffer.join(''); + tokenized.push({ value: property }); + list.push(property); + } + + return { + list: list, + tokenized: tokenized + }; + }, + + selectors: function (string) { + var selectors = new Splitter(',').split(string); + + return { + list: selectors, + tokenized: selectors.map(valueMapper) + }; + } +}; + +module.exports = Extractors; diff --git a/test/selectors/tokenizer-test.js b/test/selectors/tokenizer-test.js index fc3e57ff..0d4df9eb 100644 --- a/test/selectors/tokenizer-test.js +++ b/test/selectors/tokenizer-test.js @@ -64,7 +64,7 @@ vows.describe(Tokenizer) 'a {color:red;\n\ndisplay :\r\n block }', [{ kind: 'selector', - value: [{ value: 'a' }], + value: [{ value: 'a ' }], body: [ { value: 'color:red' }, { value: 'display:block' @@ -105,7 +105,7 @@ vows.describe(Tokenizer) kind: 'selector', value: [ { value: 'a' }, - { value: 'div.class > p' } + { value: '\n\ndiv.class > p ' } ], body: [{ value: 'color:red' }] }] @@ -176,7 +176,7 @@ vows.describe(Tokenizer) '@charset \'utf-8\';a{color:red}', [ { - kind: 'text', + kind: 'at-rule', value: '@charset \'utf-8\';' }, { @@ -189,8 +189,8 @@ vows.describe(Tokenizer) 'charset after a line break': [ '\n@charset \n\'utf-8\';', [{ - kind: 'text', - value: '@charset \'utf-8\';' + kind: 'at-rule', + value: '\n@charset \n\'utf-8\';' }] ], 'keyframes with quoted attribute': [ @@ -232,13 +232,13 @@ vows.describe(Tokenizer) 'a,\n\ndiv.class > p {color:red}', [{ kind: 'selector', - value: [{ value: 'a' }, { value: 'div.class > p' }], + value: [{ value: 'a' }, { value: '\n\ndiv.class > p ' }], body: [{ value: 'color:red' }], metadata: { body: 'color:red', bodiesList: ['color:red'], - selector: 'a,div.class > p', - selectorsList: ['a', 'div.class > p'] + selector: 'a,\n\ndiv.class > p ', + selectorsList: ['a', '\n\ndiv.class > p '] } }], ], -- 2.34.1