Moves cleaning up tokens out of tokenizer.

* Pulls out extracting logic into an utility class.

diff --git a/lib/selectors/optimizer.js b/lib/selectors/optimizer.js
index 0454bd8..98d5e0a 100644 (file)
--- a/lib/selectors/optimizer.js
+++ b/lib/selectors/optimizer.js
@@ -27,7 +27,7 @@ function rebuild(tokens, keepBreaks, isFlatBlock) {
   for (var i = 0, l = tokens.length; i < l; i++) {
     var token = tokens[i];
 
-    if (token.kind === 'text') {
+    if (token.kind === 'text' || token.kind == 'at-rule') {
       parts.push(token.value);
       continue;
     }

diff --git a/lib/selectors/optimizers/clean-up.js b/lib/selectors/optimizers/clean-up.js
index 55ef9e8..0e3781d 100644 (file)
--- a/lib/selectors/optimizers/clean-up.js
+++ b/lib/selectors/optimizers/clean-up.js
@@ -8,7 +8,12 @@ var CleanUp = {
 
     for (var i = 0, l = selectors.length; i < l; i++) {
       var selector = selectors[i].value;
-      var reduced = selector.replace(/\s*([>\+\~])\s*/g, '$1');
+      var reduced = selector
+        .replace(/\s/g, ' ')
+        .replace(/\s{2,}/g, ' ')
+        .replace(/ ?, ?/g, ',')
+        .replace(/\s*([>\+\~])\s*/g, '$1')
+        .trim();
 
       if (selector.indexOf('*') > -1) {
         reduced = reduced
@@ -36,6 +41,13 @@ var CleanUp = {
       .replace(/(\s{2,}|\s)/g, ' ')
       .replace(/(,|:|\() /g, '$1')
       .replace(/ \)/g, ')');
+  },
+
+  atRule: function (block) {
+    return block
+      .replace(/\s/g, ' ')
+      .replace(/\s{2,}/g, ' ')
+      .trim();
   }
 };
 

diff --git a/lib/selectors/optimizers/simple.js b/lib/selectors/optimizers/simple.js
index a4a6610..a29ca93 100644 (file)
--- a/lib/selectors/optimizers/simple.js
+++ b/lib/selectors/optimizers/simple.js
@@ -29,12 +29,12 @@ function SimpleOptimizer(options) {
 
 function removeUnsupported(selectors, options) {
   if (options.compatibility.selectors.ie7Hack)
-    return false;
+    return selectors;
 
   var supported = [];
   var values = [];
-  for (var i = 0, l = selectors.length; i < l; i++) {
-    var selector = selectors[i];
+  for (var i = 0, l = selectors.tokenized.length; i < l; i++) {
+    var selector = selectors.tokenized[i];
 
     if (selector.value.indexOf('*+html ') === -1 && selector.value.indexOf('*:first-child+html ') === -1) {
       supported.push(selector);
@@ -233,9 +233,8 @@ SimpleOptimizer.prototype.optimize = function(tokens) {
         break;
 
       if (token.kind == 'selector') {
-        var newSelectors = removeUnsupported(CleanUp.selectors(token.value).tokenized, options);
-        if (newSelectors)
-          token.value = newSelectors.tokenized;
+        var newSelectors = removeUnsupported(CleanUp.selectors(token.value), options);
+        token.value = newSelectors.tokenized;
 
         if (token.value.length === 0) {
           tokens.splice(i, 1);
@@ -248,10 +247,8 @@ SimpleOptimizer.prototype.optimize = function(tokens) {
         if (options.updateMetadata) {
           token.metadata.body = newBody.list.join(';');
           token.metadata.bodiesList = newBody.list;
-          if (newSelectors) {
-            token.metadata.selector = newSelectors.list.join(',');
-            token.metadata.selectorsList = newSelectors.list;
-          }
+          token.metadata.selector = newSelectors.list.join(',');
+          token.metadata.selectorsList = newSelectors.list;
         }
       } else if (token.kind == 'block') {
         token.value = CleanUp.block(token.value);
@@ -259,15 +256,17 @@ SimpleOptimizer.prototype.optimize = function(tokens) {
           token.body = reduce(token.body, self.options).tokenized;
         else
           _optimize(token.body);
-      } else if (token.kind == 'text') {
+      } else if (token.kind == 'at-rule') {
+        token.value = CleanUp.atRule(token.value);
+
         if (CHARSET_REGEXP.test(token.value)) {
           if (hasCharset || token.value.indexOf(CHARSET_TOKEN) == -1) {
             tokens.splice(i, 1);
-            i++;
+            i--;
           } else {
             hasCharset = true;
             tokens.splice(i, 1);
-            tokens.unshift({ kind: 'text', value: token.value.replace(CHARSET_REGEXP, CHARSET_TOKEN) });
+            tokens.unshift({ kind: 'at-rule', value: token.value.replace(CHARSET_REGEXP, CHARSET_TOKEN) });
           }
         }
       }

diff --git a/lib/selectors/tokenizer.js b/lib/selectors/tokenizer.js
index 6eeb75a..879470b 100644 (file)
--- a/lib/selectors/tokenizer.js
+++ b/lib/selectors/tokenizer.js
@@ -1,10 +1,7 @@
 var Chunker = require('../utils/chunker');
-var Splitter = require('../utils/splitter');
+var Extract = require('../utils/extractors');
 
 var flatBlock = /(^@(font\-face|page|\-ms\-viewport|\-o\-viewport|viewport|counter\-style)|\\@.+?)/;
-var WHITESPACE = /\s/g;
-var MULTI_WHITESPACE = /\s{2,}/g;
-var WHITESPACE_COMMA = / ?, ?/g;
 
 function Tokenizer(minifyContext, addMetadata) {
   this.minifyContext = minifyContext;
@@ -30,89 +27,6 @@ Tokenizer.prototype.toTokens = function (data) {
   return tokenize(context);
 };
 
-function valueMapper(property) { return { value: property }; }
-
-function extractProperties(string) {
-  var tokenized = [];
-  var list = [];
-  var buffer = [];
-  var property;
-  var isWhitespace;
-  var wasWhitespace;
-  var isSpecial;
-  var wasSpecial;
-  var current;
-  var wasCloseParenthesis;
-
-  for (var i = 0, l = string.length; i < l; i++) {
-    current = string[i];
-
-    if (current === ';') {
-      if (wasWhitespace && buffer[buffer.length - 1] === ' ')
-        buffer.pop();
-      if (buffer.length > 0) {
-        property = buffer.join('');
-        tokenized.push({ value: property });
-        list.push(property);
-      }
-      buffer = [];
-    } else {
-      isWhitespace = current === ' ' || current === '\t' || current === '\n';
-      isSpecial = current === ':' || current === '[' || current === ']' || current === ',' || current === '(' || current === ')';
-
-      if (wasWhitespace && isSpecial) {
-        buffer.pop();
-        buffer.push(current);
-      } else if (isWhitespace && wasSpecial && !wasCloseParenthesis) {
-      } else if (isWhitespace && !wasWhitespace && buffer.length > 0) {
-        buffer.push(' ');
-      } else if (isWhitespace && buffer.length === 0) {
-      } else if (isWhitespace && wasWhitespace) {
-      } else {
-        buffer.push(isWhitespace ? ' ' : current);
-      }
-    }
-
-    wasSpecial = isSpecial;
-    wasWhitespace = isWhitespace;
-    wasCloseParenthesis = current === ')';
-  }
-
-  if (wasWhitespace && buffer[buffer.length - 1] === ' ')
-    buffer.pop();
-  if (buffer.length > 0) {
-    property = buffer.join('');
-    tokenized.push({ value: property });
-    list.push(property);
-  }
-
-  return {
-    list: list,
-    tokenized: tokenized
-  };
-}
-
-function extractSelectors(string) {
-  var extracted = string
-    .replace(WHITESPACE, ' ')
-    .replace(MULTI_WHITESPACE, ' ')
-    .replace(WHITESPACE_COMMA, ',')
-    .trim();
-
-  var selectors = new Splitter(',').split(extracted);
-  return {
-    list: selectors,
-    tokenized: selectors.map(valueMapper)
-  };
-}
-
-function extractBlock(string) {
-  return string
-    .replace(WHITESPACE, ' ')
-    .replace(MULTI_WHITESPACE, ' ')
-    .trim();
-}
-
 function whatsNext(context) {
   var mode = context.mode;
   var chunk = context.chunk;
@@ -194,8 +108,8 @@ function tokenize(context) {
       } else if (isSingle) {
         nextEnd = chunk.indexOf(';', nextSpecial + 1);
 
-        var single = extractBlock(chunk.substring(context.cursor, nextEnd + 1));
-        tokenized.push({ kind: 'text', value: single });
+        var single = chunk.substring(context.cursor, nextEnd + 1);
+        tokenized.push({ kind: 'at-rule', value: single });
 
         context.cursor = nextEnd + 1;
       } else {
@@ -209,7 +123,7 @@ function tokenize(context) {
         var specialBody = tokenize(context);
 
         if (typeof specialBody == 'string')
-          specialBody = extractProperties(specialBody).tokenized;
+          specialBody = Extract.properties(specialBody).tokenized;
 
         context.mode = oldMode;
 
@@ -222,12 +136,12 @@ function tokenize(context) {
 
       context.cursor = nextEnd + 2;
     } else if (what == 'bodyStart') {
-      var selectorData = extractSelectors(chunk.substring(context.cursor, nextSpecial));
+      var selectorData = Extract.selectors(chunk.substring(context.cursor, nextSpecial));
 
       oldMode = context.mode;
       context.cursor = nextSpecial + 1;
       context.mode = 'body';
-      var bodyData = extractProperties(tokenize(context));
+      var bodyData = Extract.properties(tokenize(context));
 
       context.mode = oldMode;
 

diff --git a/lib/utils/extractors.js b/lib/utils/extractors.js
new file mode 100644 (file)
index 0000000..4067fbc
--- /dev/null
+++ b/lib/utils/extractors.js
@@ -0,0 +1,78 @@
+var Splitter = require('./splitter');
+
+function valueMapper(property) {
+  return { value: property };
+}
+
+var Extractors = {
+  properties: function (string) {
+    var tokenized = [];
+    var list = [];
+    var buffer = [];
+    var property;
+    var isWhitespace;
+    var wasWhitespace;
+    var isSpecial;
+    var wasSpecial;
+    var current;
+    var wasCloseParenthesis;
+
+    for (var i = 0, l = string.length; i < l; i++) {
+      current = string[i];
+
+      if (current === ';') {
+        if (wasWhitespace && buffer[buffer.length - 1] === ' ')
+          buffer.pop();
+        if (buffer.length > 0) {
+          property = buffer.join('');
+          tokenized.push({ value: property });
+          list.push(property);
+        }
+        buffer = [];
+      } else {
+        isWhitespace = current === ' ' || current === '\t' || current === '\n';
+        isSpecial = current === ':' || current === '[' || current === ']' || current === ',' || current === '(' || current === ')';
+
+        if (wasWhitespace && isSpecial) {
+          buffer.pop();
+          buffer.push(current);
+        } else if (isWhitespace && wasSpecial && !wasCloseParenthesis) {
+        } else if (isWhitespace && !wasWhitespace && buffer.length > 0) {
+          buffer.push(' ');
+        } else if (isWhitespace && buffer.length === 0) {
+        } else if (isWhitespace && wasWhitespace) {
+        } else {
+          buffer.push(isWhitespace ? ' ' : current);
+        }
+      }
+
+      wasSpecial = isSpecial;
+      wasWhitespace = isWhitespace;
+      wasCloseParenthesis = current === ')';
+    }
+
+    if (wasWhitespace && buffer[buffer.length - 1] === ' ')
+      buffer.pop();
+    if (buffer.length > 0) {
+      property = buffer.join('');
+      tokenized.push({ value: property });
+      list.push(property);
+    }
+
+    return {
+      list: list,
+      tokenized: tokenized
+    };
+  },
+
+  selectors: function (string) {
+    var selectors = new Splitter(',').split(string);
+
+    return {
+      list: selectors,
+      tokenized: selectors.map(valueMapper)
+    };
+  }
+};
+
+module.exports = Extractors;

diff --git a/test/selectors/tokenizer-test.js b/test/selectors/tokenizer-test.js
index fc3e57f..0d4df9e 100644 (file)
--- a/test/selectors/tokenizer-test.js
+++ b/test/selectors/tokenizer-test.js
@@ -64,7 +64,7 @@ vows.describe(Tokenizer)
         'a {color:red;\n\ndisplay :\r\n  block }',
         [{
           kind: 'selector',
-          value: [{ value: 'a' }],
+          value: [{ value: 'a ' }],
           body: [
             { value: 'color:red' },
             { value: 'display:block'
@@ -105,7 +105,7 @@ vows.describe(Tokenizer)
           kind: 'selector',
           value: [
             { value: 'a' },
-            { value: 'div.class > p' }
+            { value: '\n\ndiv.class > p ' }
           ],
           body: [{ value: 'color:red' }]
         }]
@@ -176,7 +176,7 @@ vows.describe(Tokenizer)
         '@charset \'utf-8\';a{color:red}',
         [
           {
-            kind: 'text',
+            kind: 'at-rule',
             value: '@charset \'utf-8\';'
           },
           {
@@ -189,8 +189,8 @@ vows.describe(Tokenizer)
       'charset after a line break': [
         '\n@charset \n\'utf-8\';',
         [{
-          kind: 'text',
-          value: '@charset \'utf-8\';'
+          kind: 'at-rule',
+          value: '\n@charset \n\'utf-8\';'
         }]
       ],
       'keyframes with quoted attribute': [
@@ -232,13 +232,13 @@ vows.describe(Tokenizer)
         'a,\n\ndiv.class > p {color:red}',
         [{
           kind: 'selector',
-          value: [{ value: 'a' }, { value: 'div.class > p' }],
+          value: [{ value: 'a' }, { value: '\n\ndiv.class > p ' }],
           body: [{ value: 'color:red' }],
           metadata: {
             body: 'color:red',
             bodiesList: ['color:red'],
-            selector: 'a,div.class > p',
-            selectorsList: ['a', 'div.class > p']
+            selector: 'a,\n\ndiv.class > p ',
+            selectorsList: ['a', '\n\ndiv.class > p ']
           }
         }],
       ],