Standardizes parsed tokens.

This commit unifies tokenized values' structure, i.e.

```
[
  <TOKEN_NAME>,
  <TOKEN_VALUE>,
  <METADATA>
]
```

Why:

* So it's easier to work with those values further down the
  optimizing pipeline and when restoring back to text content.

diff --git a/lib/optimizer/advanced.js b/lib/optimizer/advanced.js
index fb5d9ec..d809147 100644 (file)
--- a/lib/optimizer/advanced.js
+++ b/lib/optimizer/advanced.js
@@ -38,7 +38,7 @@ function recursivelyOptimizeBlocks(tokens, context) {
     var token = tokens[i];
 
     if (token[0] == Token.BLOCK) {
-      var isKeyframes = /@(-moz-|-o-|-webkit-)?keyframes/.test(token[1][0][0]);
+      var isKeyframes = /@(-moz-|-o-|-webkit-)?keyframes/.test(token[1][0][1]);
       optimize(token[2], context, !isKeyframes);
     }
   }

diff --git a/lib/optimizer/basic.js b/lib/optimizer/basic.js
index 0e2dc20..ed78d98 100644 (file)
--- a/lib/optimizer/basic.js
+++ b/lib/optimizer/basic.js
@@ -446,7 +446,7 @@ function removeComments(tokens, options) {
 }
 
 function optimizeComment(token, options) {
-  if (token[1][0][2] == Marker.EXCLAMATION && (options.keepSpecialComments == '*' || options.commentsKept < options.keepSpecialComments)) {
+  if (token[1][2] == Marker.EXCLAMATION && (options.keepSpecialComments == '*' || options.commentsKept < options.keepSpecialComments)) {
     options.commentsKept++;
     return;
   }

diff --git a/lib/optimizer/reduce-non-adjacent.js b/lib/optimizer/reduce-non-adjacent.js
index 267231a..caa93ba 100644 (file)
--- a/lib/optimizer/reduce-non-adjacent.js
+++ b/lib/optimizer/reduce-non-adjacent.js
@@ -52,7 +52,7 @@ function wrappedSelectorsFrom(list) {
   var wrapped = [];
 
   for (var i = 0; i < list.length; i++) {
-    wrapped.push([list[i][0]]);
+    wrapped.push([list[i][1]]);
   }
 
   return wrapped;

diff --git a/lib/optimizer/reorderable.js b/lib/optimizer/reorderable.js
index 3c49a3b..f6d1a63 100644 (file)
--- a/lib/optimizer/reorderable.js
+++ b/lib/optimizer/reorderable.js
@@ -79,7 +79,7 @@ function withDifferentVendorPrefix(value1, value2) {
 function selectorsDoNotOverlap(s1, s2) {
   for (var i = 0, l = s1.length; i < l; i++) {
     for (var j = 0, m = s2.length; j < m; j++) {
-      if (s1[i][0] == s2[j][0])
+      if (s1[i][1] == s2[j][1])
         return false;
     }
   }

diff --git a/lib/optimizer/tidy-block.js b/lib/optimizer/tidy-block.js
index 10b437c..f5cd6c6 100644 (file)
--- a/lib/optimizer/tidy-block.js
+++ b/lib/optimizer/tidy-block.js
@@ -2,7 +2,7 @@ function tidyBlock(values, spaceAfterClosingBrace) {
   var i;
 
   for (i = values.length - 1; i >= 0; i--) {
-    values[i][0] = values[i][0]
+    values[i][1] = values[i][1]
       .replace(/\n|\r\n/g, ' ')
       .replace(/\s+/g, ' ')
       .replace(/(,|:|\() /g, '$1')

diff --git a/lib/optimizer/tidy-rule-duplicates.js b/lib/optimizer/tidy-rule-duplicates.js
index e488d64..30a9c21 100644 (file)
--- a/lib/optimizer/tidy-rule-duplicates.js
+++ b/lib/optimizer/tidy-rule-duplicates.js
@@ -1,5 +1,5 @@
 function ruleSorter(s1, s2) {
-  return s1[0] > s2[0] ? 1 : -1;
+  return s1[1] > s2[1] ? 1 : -1;
 }
 
 function tidyRuleDuplicates(rules) {
@@ -9,8 +9,8 @@ function tidyRuleDuplicates(rules) {
   for (var i = 0, l = rules.length; i < l; i++) {
     var rule = rules[i];
 
-    if (repeated.indexOf(rule[0]) == -1) {
-      repeated.push(rule[0]);
+    if (repeated.indexOf(rule[1]) == -1) {
+      repeated.push(rule[1]);
       list.push(rule);
     }
   }

diff --git a/lib/optimizer/tidy-rules.js b/lib/optimizer/tidy-rules.js
index d5c8a6d..684666b 100644 (file)
--- a/lib/optimizer/tidy-rules.js
+++ b/lib/optimizer/tidy-rules.js
@@ -125,7 +125,7 @@ function removeQuotes(value) {
 }
 
 function ruleSorter(s1, s2) {
-  return s1[0] > s2[0] ? 1 : -1;
+  return s1[1] > s2[1] ? 1 : -1;
 }
 
 function tidyRules(rules, removeUnsupported, adjacentSpace, warnings) {
@@ -134,10 +134,10 @@ function tidyRules(rules, removeUnsupported, adjacentSpace, warnings) {
 
   for (var i = 0, l = rules.length; i < l; i++) {
     var rule = rules[i];
-    var reduced = rule[0];
+    var reduced = rule[1];
 
     if (hasInvalidCharacters(reduced)) {
-      warnings.push('Invalid selector \'' + rule[0] + '\' at line ' + rule[1][0][0] + ', column ' + rule[1][0][1] + '. Ignoring.');
+      warnings.push('Invalid selector \'' + rule[1] + '\' at line ' + rule[2][0][0] + ', column ' + rule[2][0][1] + '. Ignoring.');
       continue;
     }
 
@@ -166,7 +166,7 @@ function tidyRules(rules, removeUnsupported, adjacentSpace, warnings) {
       continue;
     }
 
-    rule[0] = reduced;
+    rule[1] = reduced;
     repeated.push(reduced);
     list.push(rule);
   }

diff --git a/lib/stringifier/helpers.js b/lib/stringifier/helpers.js
index 66aabb4..0cc788d 100644 (file)
--- a/lib/stringifier/helpers.js
+++ b/lib/stringifier/helpers.js
@@ -78,17 +78,14 @@ function property(tokens, position, lastPropertyAt, context) {
 
   switch (token[0]) {
     case Token.AT_RULE:
-      // slicing here is a bit inconvenient but necessary since otherwise source map info
-      // would not be passed in. We should consider changing parsing to output an array as
-      // 2nd element. Unfortunately this also refers to properties... :-/
-      store(token.slice(1), context);
+      store(token, context);
       store(position < lastPropertyAt ? Marker.SEMICOLON : '', context);
       break;
     case Token.COMMENT:
-      store(token[1][0], context);
+      store(token, context);
       break;
     case Token.PROPERTY:
-      store(token[1][1], context);
+      store(token[1], context);
       store(Marker.COLON, context);
       value(token, context);
       store(position < lastPropertyAt || token[2][0] == Token.PROPERTY_BLOCK ? Marker.SEMICOLON : '', context);
@@ -105,7 +102,7 @@ function value(token, context) {
     store(Marker.CLOSE_BRACE, context);
   } else {
     for (j = 2, m = token.length; j < m; j++) {
-      store(token[j][1], context);
+      store(token[j], context);
 
       if (j < m - 1 && (inFilter(token) || !inSpecialContext(token, j, context))) {
         store(Marker.SPACE, context);
@@ -125,7 +122,7 @@ function all(tokens, context) {
 
     switch (token[0]) {
       case Token.AT_RULE:
-        store(token[1], context);
+        store(token, context);
         store(Marker.SEMICOLON, context);
         break;
       case Token.AT_RULE_BLOCK:
@@ -142,7 +139,7 @@ function all(tokens, context) {
         store(Marker.CLOSE_BRACE, context);
         break;
       case Token.COMMENT:
-        store(token[1], context);
+        store(token, context);
         break;
       case Token.RULE:
         rules(token[1], context);

diff --git a/lib/stringifier/one-time.js b/lib/stringifier/one-time.js
index abcf6c5..2f2e767 100644 (file)
--- a/lib/stringifier/one-time.js
+++ b/lib/stringifier/one-time.js
@@ -1,7 +1,7 @@
 var helpers = require('./helpers');
 
 function store(token, context) {
-  context.output.push(typeof token == 'string' ? token : token[0]);
+  context.output.push(typeof token == 'string' ? token : token[1]);
 }
 
 function context() {

diff --git a/lib/stringifier/simple.js b/lib/stringifier/simple.js
index 78f9277..b3a497c 100644 (file)
--- a/lib/stringifier/simple.js
+++ b/lib/stringifier/simple.js
@@ -1,7 +1,7 @@
 var all = require('./helpers').all;
 
 function store(token, stringifyContext) {
-  stringifyContext.output.push(typeof token == 'string' ? token : token[0]);
+  stringifyContext.output.push(typeof token == 'string' ? token : token[1]);
 }
 
 function stringify(tokens, context) {

diff --git a/lib/stringifier/source-maps.js b/lib/stringifier/source-maps.js
index c8a1563..4b62a5b 100644 (file)
--- a/lib/stringifier/source-maps.js
+++ b/lib/stringifier/source-maps.js
@@ -6,7 +6,7 @@ var unknownSource = '$stdin';
 
 function store(element, context) {
   var fromString = typeof element == 'string';
-  var value = fromString ? element : element[0];
+  var value = fromString ? element : element[1];
 
   if (value.indexOf('_') > -1)
     value = context.restore(value, prefixContentFrom(context.output));

diff --git a/lib/tokenizer/token.js b/lib/tokenizer/token.js
index 06a1a13..a4a9d4d 100644 (file)
--- a/lib/tokenizer/token.js
+++ b/lib/tokenizer/token.js
@@ -1,13 +1,16 @@
 var Token = {
   AT_RULE: 'at-rule', // e.g. `@import`, `@charset`
-  AT_RULE_BLOCK: 'at-rule-block', // e.g. `@font-face`
-  BLOCK: 'block', // e.g. `@media`, `@keyframes`
+  AT_RULE_BLOCK: 'at-rule-block', // e.g. `@font-face{...}`
+  AT_RULE_BLOCK_SCOPE: 'at-rule-block-scope', // e.g. `@font-face`
+  BLOCK: 'block', // e.g. `@media screen{...}`, `@keyframes animation {...}`
+  BLOCK_SCOPE: 'block-scope', // e.g. `@media`, `@keyframes`
   COMMENT: 'comment', // e.g. `/* comment */`
   PROPERTY: 'property', // e.g. `color:red`
   PROPERTY_BLOCK: 'property-block', // e.g. `--var:{color:red}`
   PROPERTY_NAME: 'property-name', // e.g. `color`
   PROPERTY_VALUE: 'property-value', // e.g. `red`
-  RULE: 'rule' // e.g `div > a`
+  RULE: 'rule', // e.g `div > a{...}`
+  RULE_SCOPE: 'rule-scope' // e.g `div > a`
 };
 
 module.exports = Token;

diff --git a/lib/tokenizer/tokenize.js b/lib/tokenizer/tokenize.js
index d614647..ac8a4cb 100644 (file)
--- a/lib/tokenizer/tokenize.js
+++ b/lib/tokenizer/tokenize.js
@@ -105,7 +105,7 @@ function intoTokens(source, externalContext, internalContext, isNested) {
       buffer.push(character);
     } else if (isCommentEnd) {
       // comment end, e.g. /* comment */<--
-      lastToken = [Token.COMMENT, [buffer.join('').trim() + character, [metadata]]];
+      lastToken = [Token.COMMENT, buffer.join('').trim() + character, [metadata]];
       newTokens.push(lastToken);
 
       level = levels.pop();
@@ -149,7 +149,7 @@ function intoTokens(source, externalContext, internalContext, isNested) {
       buffer = [];
     } else if (character == Marker.COMMA && level == Level.BLOCK && ruleToken) {
       // comma separator at block level, e.g. a,div,<--
-      ruleToken[1].push([buffer.join('').trim(), [metadata]]);
+      ruleToken[1].push([tokenScopeFrom(ruleToken[0]), buffer.join('').trim(), [metadata]]);
 
       buffer = [];
     } else if (character == Marker.COMMA && level == Level.BLOCK && tokenTypeFrom(buffer) == Token.AT_RULE) {
@@ -158,12 +158,13 @@ function intoTokens(source, externalContext, internalContext, isNested) {
       buffer.push(character);
     } else if (character == Marker.COMMA && level == Level.BLOCK) {
       // comma separator at block level, e.g. a,<--
-      ruleToken = [tokenTypeFrom(buffer), [[buffer.join('').trim(), [metadata]]], []];
+      ruleToken = [tokenTypeFrom(buffer), [], []];
+      ruleToken[1].push([tokenScopeFrom(ruleToken[0]), buffer.join('').trim(), [metadata]]);
 
       buffer = [];
     } else if (character == Marker.OPEN_BRACE && level == Level.BLOCK && ruleToken && ruleToken[0] == Token.BLOCK) {
       // open brace opening at-rule at block level, e.g. @media{<--
-      ruleToken[1].push([buffer.join('').trim(), [metadata]]);
+      ruleToken[1].push([Token.BLOCK_SCOPE, buffer.join('').trim(), [metadata]]);
       allTokens.push(ruleToken);
 
       levels.push(level);
@@ -176,7 +177,7 @@ function intoTokens(source, externalContext, internalContext, isNested) {
     } else if (character == Marker.OPEN_BRACE && level == Level.BLOCK && tokenTypeFrom(buffer) == Token.BLOCK) {
       // open brace opening at-rule at block level, e.g. @media{<--
       ruleToken = ruleToken || [Token.BLOCK, [], []];
-      ruleToken[1].push([buffer.join('').trim(), [metadata]]);
+      ruleToken[1].push([Token.BLOCK_SCOPE, buffer.join('').trim(), [metadata]]);
       allTokens.push(ruleToken);
 
       levels.push(level);
@@ -189,7 +190,7 @@ function intoTokens(source, externalContext, internalContext, isNested) {
     } else if (character == Marker.OPEN_BRACE && level == Level.BLOCK) {
       // open brace opening rule at block level, e.g. div{<--
       ruleToken = ruleToken || [tokenTypeFrom(buffer), [], []];
-      ruleToken[1].push([buffer.join('').trim(), [metadata]]);
+      ruleToken[1].push([tokenScopeFrom(ruleToken[0]), buffer.join('').trim(), [metadata]]);
       newTokens = ruleToken[2];
       allTokens.push(ruleToken);
 
@@ -398,4 +399,14 @@ function tokenTypeFrom(buffer) {
   }
 }
 
+function tokenScopeFrom(tokenType) {
+  if (tokenType == Token.RULE) {
+    return Token.RULE_SCOPE;
+  } else if (tokenType == Token.BLOCK) {
+    return Token.BLOCK_SCOPE;
+  } else if (tokenType == Token.AT_RULE_BLOCK) {
+    return Token.AT_RULE_BLOCK_SCOPE;
+  }
+}
+
 module.exports = tokenize;

diff --git a/lib/utils/read-sources.js b/lib/utils/read-sources.js
index 30b35d6..24da9dd 100644 (file)
--- a/lib/utils/read-sources.js
+++ b/lib/utils/read-sources.js
@@ -467,7 +467,7 @@ function inlineLocalStylesheet(uri, mediaQuery, inlinerContext) {
 
 function wrapInMedia(tokens, mediaQuery) {
   if (mediaQuery) {
-    return [[Token.BLOCK, [['@media ' + mediaQuery]], tokens]];
+    return [[Token.BLOCK, [[Token.BLOCK_SCOPE, '@media ' + mediaQuery]], tokens]];
   } else {
     return tokens;
   }

diff --git a/test/module-test.js b/test/module-test.js
index 4795339..b353784 100644 (file)
--- a/test/module-test.js
+++ b/test/module-test.js
@@ -603,7 +603,7 @@ vows.describe('module tests').addBatch({
       }
     },
     'with remote paths': {
-      'topic': new CleanCSS({ sourceMap: true }).minify({
+      'topic': new CleanCSS().minify({
         'http://127.0.0.1/styles.css': {
           styles: 'div{background-image:url(image.png)}'
         }

diff --git a/test/optimizer/extract-properties-test.js b/test/optimizer/extract-properties-test.js
index 9bef4d6..209f929 100644 (file)
--- a/test/optimizer/extract-properties-test.js
+++ b/test/optimizer/extract-properties-test.js
@@ -33,7 +33,7 @@ vows.describe(extractProperties)
             'color',
             ['property', ['property-name', 'color', [[1, 2, undefined]]], ['property-value', 'red', [[1, 8, undefined]]]],
             'color:red',
-            [['a', [[1, 0, undefined]]]],
+            [['rule-scope', 'a', [[1, 0, undefined]]]],
             true
           ]
         ]);
@@ -51,7 +51,7 @@ vows.describe(extractProperties)
             'color',
             ['property', ['property-name', 'color', [[1, 2, undefined]]], ['property-value', 'red!important', [[1, 8, undefined]]]],
             'color:red!important',
-            [['a', [[1, 0, undefined]]]],
+            [['rule-scope', 'a', [[1, 0, undefined]]]],
             true
           ]
         ]);
@@ -69,7 +69,7 @@ vows.describe(extractProperties)
             'color',
             ['property', ['property-name', 'color', [[1, 10, undefined]]], ['property-value', 'red', [[1, 16, undefined]]]],
             'color:red',
-            [['#one span', [[1, 0, undefined]]]],
+            [['rule-scope', '#one span', [[1, 0, undefined]]]],
             true
           ]
         ]);
@@ -103,7 +103,7 @@ vows.describe(extractProperties)
             'color',
             ['property', ['property-name', 'color', [[1, 5, undefined]]], ['property-value', 'red', [[1, 11, undefined]]]],
             'color:red',
-            [['.one', [[1, 0, undefined]]]],
+            [['rule-scope', '.one', [[1, 0, undefined]]]],
             false
           ]
         ]);
@@ -121,7 +121,7 @@ vows.describe(extractProperties)
             'color',
             ['property', ['property-name', 'color', [[1, 2, undefined]]], ['property-value', 'red', [[1, 8, undefined]]]],
             'color:red',
-            [['a', [[1, 0, undefined]]]],
+            [['rule-scope', 'a', [[1, 0, undefined]]]],
             true
           ],
           [
@@ -130,7 +130,7 @@ vows.describe(extractProperties)
             'display',
             ['property', ['property-name', 'display', [[1, 12, undefined]]], ['property-value', 'block', [[1, 20, undefined]]]],
             'display:block',
-            [['a', [[1, 0, undefined]]]],
+            [['rule-scope', 'a', [[1, 0, undefined]]]],
             true
           ]
         ]);
@@ -148,7 +148,7 @@ vows.describe(extractProperties)
             'color',
             ['property', ['property-name', 'color', [[1, 9, undefined]]], ['property-value', 'red', [[1, 15, undefined]]]],
             'color:red',
-            [['a', [[1, 7, undefined]]]],
+            [['rule-scope', 'a', [[1, 7, undefined]]]],
             true
           ],
           [
@@ -157,7 +157,7 @@ vows.describe(extractProperties)
             'display',
             ['property', ['property-name', 'display', [[1, 19, undefined]]], ['property-value', 'block', [[1, 27, undefined]]]],
             'display:block',
-            [['a', [[1, 7, undefined]]]],
+            [['rule-scope', 'a', [[1, 7, undefined]]]],
             true
           ],
           [
@@ -166,7 +166,7 @@ vows.describe(extractProperties)
             'color',
             ['property', ['property-name', 'color', [[1, 35, undefined]]], ['property-value', 'red', [[1, 41, undefined]]]],
             'color:red',
-            [['p', [[1, 33, undefined]]]],
+            [['rule-scope', 'p', [[1, 33, undefined]]]],
             true
           ]
         ]);
@@ -187,7 +187,7 @@ vows.describe(extractProperties)
               'transform',
               ['property', ['property-name', '-moz-transform', [[1, 2, undefined]]], ['property-value', 'none', [[1, 17, undefined]]]],
               '-moz-transform:none',
-              [['a', [[1, 0, undefined]]]],
+              [['rule-scope', 'a', [[1, 0, undefined]]]],
               true
             ]
           ]);
@@ -205,7 +205,7 @@ vows.describe(extractProperties)
               'list-style',
               ['property', ['property-name', 'list-style', [[1, 2, undefined]]], ['property-value', 'none', [[1, 13, undefined]]]],
               'list-style:none',
-              [['a', [[1, 0, undefined]]]],
+              [['rule-scope', 'a', [[1, 0, undefined]]]],
               true
             ]
           ]);
@@ -223,7 +223,7 @@ vows.describe(extractProperties)
               'border-radius',
               ['property', ['property-name', 'border-top-left-radius', [[1, 2, undefined]]], ['property-value', 'none', [[1, 25, undefined]]]],
               'border-top-left-radius:none',
-              [['a', [[1, 0, undefined]]]],
+              [['rule-scope', 'a', [[1, 0, undefined]]]],
               true
             ]
           ]);
@@ -241,7 +241,7 @@ vows.describe(extractProperties)
               'border-radius',
               ['property', ['property-name', '-webkit-border-top-left-radius', [[1, 2, undefined]]], ['property-value', 'none', [[1, 33, undefined]]]],
               '-webkit-border-top-left-radius:none',
-              [['a', [[1, 0, undefined]]]],
+              [['rule-scope', 'a', [[1, 0, undefined]]]],
               true
             ]
           ]);
@@ -259,7 +259,7 @@ vows.describe(extractProperties)
               'border-image',
               ['property', ['property-name', 'border-image-width', [[1, 2, undefined]]], ['property-value', '2px', [[1, 21, undefined]]]],
               'border-image-width:2px',
-              [['a', [[1, 0, undefined]]]],
+              [['rule-scope', 'a', [[1, 0, undefined]]]],
               true
             ]
           ]);
@@ -277,7 +277,7 @@ vows.describe(extractProperties)
               'border',
               ['property', ['property-name', 'border-color', [[1, 2, undefined]]], ['property-value', 'red', [[1, 15, undefined]]]],
               'border-color:red',
-              [['a', [[1, 0, undefined]]]],
+              [['rule-scope', 'a', [[1, 0, undefined]]]],
               true
             ]
           ]);
@@ -295,7 +295,7 @@ vows.describe(extractProperties)
               'border-top',
               ['property', ['property-name', 'border-top-style', [[1, 2, undefined]]], ['property-value', 'none', [[1, 19, undefined]]]],
               'border-top-style:none',
-              [['a', [[1, 0, undefined]]]],
+              [['rule-scope', 'a', [[1, 0, undefined]]]],
               true
             ]
           ]);
@@ -313,7 +313,7 @@ vows.describe(extractProperties)
               'border',
               ['property', ['property-name', 'border-top', [[1, 2, undefined]]], ['property-value', 'none', [[1, 13, undefined]]]],
               'border-top:none',
-              [['a', [[1, 0, undefined]]]],
+              [['rule-scope', 'a', [[1, 0, undefined]]]],
               true
             ]
           ]);
@@ -331,7 +331,7 @@ vows.describe(extractProperties)
               'border-collapse',
               ['property', ['property-name', 'border-collapse', [[1, 2, undefined]]], ['property-value', 'collapse', [[1, 18, undefined]]]],
               'border-collapse:collapse',
-              [['a', [[1, 0, undefined]]]],
+              [['rule-scope', 'a', [[1, 0, undefined]]]],
               true
             ]
           ]);
@@ -349,7 +349,7 @@ vows.describe(extractProperties)
               'text-shadow',
               ['property', ['property-name', 'text-shadow', [[1, 2, undefined]]], ['property-value', 'none', [[1, 14, undefined]]]],
               'text-shadow:none',
-              [['a', [[1, 0, undefined]]]],
+              [['rule-scope', 'a', [[1, 0, undefined]]]],
               true
             ]
           ]);

diff --git a/test/properties/wrap-for-optimizing-test.js b/test/properties/wrap-for-optimizing-test.js
index 22e76a5..dc3f3fd 100644 (file)
--- a/test/properties/wrap-for-optimizing-test.js
+++ b/test/properties/wrap-for-optimizing-test.js
@@ -75,7 +75,7 @@ vows.describe(wrapForOptimizing)
         return wrapForOptimizing([
           [
             'comment',
-            ['/* comment */']
+            '/* comment */'
           ],
           [
             'property',

diff --git a/test/tokenizer/tokenize-test.js b/test/tokenizer/tokenize-test.js
index 72c4c82..14c175b 100644 (file)
--- a/test/tokenizer/tokenize-test.js
+++ b/test/tokenizer/tokenize-test.js
@@ -50,11 +50,9 @@ vows.describe(tokenize)
         [
           [
             'comment',
+            '/* comment */',
             [
-              '/* comment */',
-              [
-                [1, 0, undefined]
-              ]
+              [1, 0, undefined]
             ]
           ]
         ]
@@ -64,11 +62,9 @@ vows.describe(tokenize)
         [
           [
             'comment',
+            '/* comment */',
             [
-              '/* comment */',
-              [
-                [1, 0, undefined]
-              ]
+              [1, 0, undefined]
             ]
           ]
         ]
@@ -78,11 +74,9 @@ vows.describe(tokenize)
         [
           [
             'comment',
+            '/*/ comment */',
             [
-              '/*/ comment */',
-              [
-                [1, 0, undefined]
-              ]
+              [1, 0, undefined]
             ]
           ]
         ]
@@ -92,26 +86,23 @@ vows.describe(tokenize)
         [
           [
             'comment',
+            '/* comment 1 */',
             [
-              '/* comment 1 */',
-              [
-                [1, 0, undefined]
-              ]
+              [1, 0, undefined]
             ]
           ],
           [
             'comment',
+            '/* comment 2 */',
             [
-              '/* comment 2 */',
-              [
-                [1, 16, undefined]
-              ]
+              [1, 16, undefined]
             ]
           ],
           [
             'rule',
             [
               [
+                'rule-scope',
                 '*',
                 [
                   [1, 15, undefined]
@@ -129,6 +120,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -144,17 +136,16 @@ vows.describe(tokenize)
         [
           [
             'comment',
+            '/* comment \n\n */',
             [
-              '/* comment \n\n */',
-              [
-                [1, 0, undefined]
-              ]
+              [1, 0, undefined]
             ]
           ],
           [
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [3, 3, undefined]
@@ -172,6 +163,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -189,6 +181,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -224,6 +217,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -273,6 +267,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -308,6 +303,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -360,6 +356,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'div a',
                 [
                   [1, 0, undefined]
@@ -395,6 +392,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -430,6 +428,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -486,6 +485,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -542,6 +542,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -594,6 +595,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -629,6 +631,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a[data-kind="one two"]',
                 [
                   [1, 0, undefined]
@@ -664,6 +667,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 '.this-class\\\'s-got-an-apostrophe',
                 [
                   [1, 0, undefined]
@@ -699,6 +703,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -734,6 +739,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -769,12 +775,14 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
                 ]
               ],
               [
+                'rule-scope',
                 'div.class > p',
                 [
                   [3, 0, undefined]
@@ -810,18 +818,21 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'b',
                 [
                   [1, 0, undefined]
                 ]
               ],
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 2, undefined]
                 ]
               ],
               [
+                'rule-scope',
                 'div',
                 [
                   [1, 4, undefined]
@@ -839,6 +850,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -869,6 +881,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'div',
                 [
                   [2, 1, undefined]
@@ -902,26 +915,23 @@ vows.describe(tokenize)
         [
           [
             'comment',
+            '/* comment 1 */',
             [
-              '/* comment 1 */',
-              [
-                [1, 0, undefined]
-              ]
+              [1, 0, undefined]
             ]
           ],
           [
             'comment',
+            '/* comment 2 */',
             [
-              '/* comment 2 */',
-              [
-                [2, 0, undefined]
-              ]
+              [2, 0, undefined]
             ]
           ],
           [
             'rule',
             [
               [
+                'rule-scope',
                 'div',
                 [
                   [3, 0, undefined]
@@ -937,26 +947,23 @@ vows.describe(tokenize)
         [
           [
             'comment',
+            '/* comment 1 */',
             [
-              '/* comment 1 */',
-              [
-                [1, 0, undefined]
-              ]
+              [1, 0, undefined]
             ]
           ],
           [
             'comment',
+            '/* comment 2 */',
             [
-              '/* comment 2 */',
-              [
-                [1, 18, undefined]
-              ]
+              [1, 18, undefined]
             ]
           ],
           [
             'rule',
             [
               [
+                'rule-scope',
                 'div',
                 [
                   [1, 15, undefined]
@@ -992,6 +999,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'div',
                 [
                   [1, 0, undefined]
@@ -1001,11 +1009,9 @@ vows.describe(tokenize)
             [
               [
                 'comment',
+                '/* comment 1 */',
                 [
-                  '/* comment 1 */',
-                  [
-                    [1, 4, undefined]
-                  ]
+                  [1, 4, undefined]
                 ]
               ],
               [
@@ -1027,11 +1033,9 @@ vows.describe(tokenize)
               ],
               [
                 'comment',
+                '/* comment 2 */',
                 [
-                  '/* comment 2 */',
-                  [
-                    [1, 28, undefined]
-                  ]
+                  [1, 28, undefined]
                 ]
               ]
             ]
@@ -1045,6 +1049,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'div',
                 [
                   [1, 0, undefined]
@@ -1085,11 +1090,9 @@ vows.describe(tokenize)
               ],
               [
                 'comment',
+                '/* comment */',
                 [
-                  '/* comment */',
-                  [
-                    [1, 30, undefined]
-                  ]
+                  [1, 30, undefined]
                 ]
               ]
             ]
@@ -1103,6 +1106,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'div',
                 [
                   [1, 0, undefined]
@@ -1143,11 +1147,9 @@ vows.describe(tokenize)
               ],
               [
                 'comment',
+                '/* comment */',
                 [
-                  '/* comment */',
-                  [
-                    [1, 33, undefined]
-                  ]
+                  [1, 33, undefined]
                 ]
               ]
             ]
@@ -1161,6 +1163,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'div',
                 [
                   [1, 0, undefined]
@@ -1201,13 +1204,11 @@ vows.describe(tokenize)
               ],
               [
                 'comment',
+                '/* comment */',
                 [
-                  '/* comment */',
-                  [
-                    [1, 41, undefined]
-                  ]
+                  [1, 41, undefined]
                 ]
-              ],
+              ]
             ]
           ]
         ]
@@ -1219,6 +1220,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'div:nth-child(2n):not(.test)',
                 [
                   [1, 0, undefined]
@@ -1254,6 +1256,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -1296,6 +1299,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -1338,6 +1342,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -1387,6 +1392,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -1429,6 +1435,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -1454,6 +1461,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 ':root',
                 [
                   [1, 0, undefined]
@@ -1494,6 +1502,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 ':root',
                 [
                   [1, 0, undefined]
@@ -1568,6 +1577,7 @@ vows.describe(tokenize)
             'block',
             [
               [
+                'block-scope',
                 '@media (min-width:980px)',
                 [
                   [1, 0, undefined]
@@ -1585,12 +1595,14 @@ vows.describe(tokenize)
             'block',
             [
               [
+                'block-scope',
                 '@media print',
                 [
                   [1, 0, undefined]
                 ]
               ],
               [
+                'block-scope',
                 '(min-width:980px)',
                 [
                   [1, 13, undefined]
@@ -1602,6 +1614,7 @@ vows.describe(tokenize)
                 'rule',
                 [
                   [
+                    'rule-scope',
                     'a',
                     [
                       [1, 31, undefined]
@@ -1639,6 +1652,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -1669,6 +1683,7 @@ vows.describe(tokenize)
             'block',
             [
               [
+                'block-scope',
                 '@media (min-width:980px)',
                 [
                   [1, 12, undefined]
@@ -1681,6 +1696,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'p',
                 [
                   [1, 38, undefined]
@@ -1716,6 +1732,7 @@ vows.describe(tokenize)
             'block',
             [
               [
+                'block-scope',
                 '@media (min-width:980px)',
                 [
                   [1, 0, undefined]
@@ -1727,6 +1744,7 @@ vows.describe(tokenize)
                 'rule',
                 [
                   [
+                    'rule-scope',
                     'a',
                     [
                       [1, 25, undefined]
@@ -1764,6 +1782,7 @@ vows.describe(tokenize)
             'block',
             [
               [
+                'block-scope',
                 '@media only screen and (max-width:1319px)',
                 [
                   [1, 0, undefined]
@@ -1775,6 +1794,7 @@ vows.describe(tokenize)
                 'block',
                 [
                   [
+                    'block-scope',
                     '@media print',
                     [
                       [2, 0, undefined]
@@ -1786,6 +1806,7 @@ vows.describe(tokenize)
                     'rule',
                     [
                       [
+                        'rule-scope',
                         'a',
                         [
                           [3, 0, undefined]
@@ -1818,6 +1839,7 @@ vows.describe(tokenize)
                 'rule',
                 [
                   [
+                    'rule-scope',
                     'a',
                     [
                       [5, 0, undefined]
@@ -1855,6 +1877,7 @@ vows.describe(tokenize)
             'block',
             [
               [
+                'block-scope',
                 '@media ( min-width:980px )',
                 [
                   [1, 0, undefined]
@@ -1867,6 +1890,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'p',
                 [
                   [1, 29, undefined]
@@ -1902,6 +1926,7 @@ vows.describe(tokenize)
             'block',
             [
               [
+                'block-scope',
                 '@media (min-width:980px)',
                 [
                   [1, 0, undefined]
@@ -1912,17 +1937,16 @@ vows.describe(tokenize)
           ],
           [
             'comment',
+            '/*! comment */',
             [
-              '/*! comment */',
-              [
-                [1, 26, undefined]
-              ]
+              [1, 26, undefined]
             ]
           ],
           [
             'block',
             [
               [
+                'block-scope',
                 '@media screen',
                 [
                   [1, 40, undefined]
@@ -1938,17 +1962,16 @@ vows.describe(tokenize)
         [
           [
             'comment',
+            '/* comment \n */',
             [
-              '/* comment \n */',
-              [
-                [1, 8, undefined]
-              ]
+              [1, 8, undefined]
             ]
           ],
           [
             'block',
             [
               [
+                'block-scope',
                 '@media (min-width:980px)',
                 [
                   [1, 0, undefined]
@@ -1960,6 +1983,7 @@ vows.describe(tokenize)
                 'rule',
                 [
                   [
+                    'rule-scope',
                     'a',
                     [
                       [2, 20, undefined]
@@ -1997,6 +2021,7 @@ vows.describe(tokenize)
             'at-rule-block',
             [
               [
+                'at-rule-block-scope',
                 '@font-face',
                 [
                   [1, 0, undefined]
@@ -2056,6 +2081,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 17, undefined]
@@ -2103,6 +2129,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2122,6 +2149,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [4, 0, undefined]
@@ -2177,6 +2205,7 @@ vows.describe(tokenize)
             'block',
             [
               [
+                'block-scope',
                 '@keyframes "test"',
                 [
                   [1, 0, undefined]
@@ -2188,6 +2217,7 @@ vows.describe(tokenize)
                 'rule',
                 [
                   [
+                    'rule-scope',
                     '0%',
                     [
                       [1, 18, undefined]
@@ -2225,6 +2255,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2274,6 +2305,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 ':root',
                 [
                   [1, 0, undefined]
@@ -2309,6 +2341,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'div',
                 [
                   [1, 0, undefined]
@@ -2391,6 +2424,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2458,12 +2492,14 @@ vows.describe(tokenize)
             'at-rule-block',
             [
               [
+                'at-rule-block-scope',
                 '_:-ms-lang(x)',
                 [
                   [1, 0, undefined]
                 ]
               ],
               [
+                'at-rule-block-scope',
                 '@-ms-viewport',
                 [
                   [1, 14, undefined]
@@ -2503,6 +2539,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2538,6 +2575,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2605,6 +2643,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2710,6 +2749,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2759,6 +2799,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2808,6 +2849,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2857,6 +2899,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2892,6 +2935,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2941,6 +2985,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -2990,6 +3035,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -3039,6 +3085,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -3074,6 +3121,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -3109,6 +3157,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -3144,6 +3193,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -3183,6 +3233,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -3218,6 +3269,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, undefined]
@@ -3253,6 +3305,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'body',
                 [
                   [1, 0, undefined]
@@ -3305,6 +3358,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'body',
                 [
                   [1, 0, undefined]
@@ -3335,6 +3389,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 '}a',
                 [
                   [1, 15, undefined]
@@ -3398,6 +3453,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'a',
                 [
                   [1, 0, 'one.css']
@@ -3450,6 +3506,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'div > a',
                 [
                   [1, 0, 'styles.less']
@@ -3500,6 +3557,7 @@ vows.describe(tokenize)
             'rule',
             [
               [
+                'rule-scope',
                 'div > a',
                 [
                   [1, 0, 'styles.less']